mcp-agentic-pipelines 1.0.1

package/vendors/piste/pipeline/stage4/__init__.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+# Stage 4 — Verdict
+#   4a: Criticality Gate [C3]          — pipeline/stage4/criticality_gate.py
+#   4b: Verdict Aggregator [J5]        — pipeline/stage4/verdict_aggregator.py
+#   4c: Editorial Review Panel [C3]    — pipeline/stage4/orchestrator.py
+#   Orchestrator: pipeline/stage4/orchestrator.py
+
+from pipeline.stage4.criticality_gate import (
+    CriticalityGate, CriticalityAssessment, criticality_gate,
+    CRITICAL_TOPICS, HIGH_PROFILE_INDICATORS,
+)
+from pipeline.stage4.verdict_aggregator import (
+    VerdictAggregator, verdict_aggregator, VERDICT_LABELS,
+)
+from pipeline.stage4.orchestrator import (
+    Stage4Orchestrator, Stage4Result, stage4_orchestrator,
+)
+
+__all__ = [
+    "CriticalityGate",
+    "CriticalityAssessment",
+    "criticality_gate",
+    "CRITICAL_TOPICS",
+    "HIGH_PROFILE_INDICATORS",
+    "VerdictAggregator",
+    "verdict_aggregator",
+    "VERDICT_LABELS",
+    "Stage4Orchestrator",
+    "Stage4Result",
+    "stage4_orchestrator",
+]

package/vendors/piste/pipeline/stage4/criticality_gate.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 4a: Criticality Gate [C3]
+=================================
+Routes high-stakes claims to human Editorial Review Panel.
+Low-stakes claims go to automated Verdict Aggregator.
+
+Inspired by PolitiFact's three-editor panel:
+  - Elections, public health, legal, high-profile figures → HUMAN review
+  - Everything else → AUTOMATED verdict
+
+The gate checks:
+  1. Keyword match against critical topics list
+  2. Embedding similarity to critical topic centroids
+  3. Source reputation (high-profile figure detection)
+"""
+
+from typing import List, Optional
+from dataclasses import dataclass
+
+
+# Critical topics that trigger human review — Canada-focused bilingual (EN/FR)
+# Covers: Canadian federal elections + Quebec provincial elections
+CRITICAL_TOPICS = [
+    # ── Elections (EN/FR) ────────────────────────────────────
+    "election", "élection", "vote", "voter", "candidate", "candidat",
+    "ballot", "bulletin", "democracy", "démocratie",
+    "federal election", "élection fédérale",
+    "provincial election", "élection provinciale",
+    "referendum", "référendum",
+    "riding", "circonscription", "mp", "député", "députée",
+    "minority government", "gouvernement minoritaire",
+    "majority government", "gouvernement majoritaire",
+    "coalition", "confidence vote", "vote de confiance",
+    # ── Quebec Politics (FR/EN) ──────────────────────────────
+    "quebec", "québec", "québécois", "quebecois",
+    "national assembly", "assemblée nationale",
+    "sovereignty", "souveraineté", "separatist", "séparatiste",
+    "federalism", "fédéralisme", "federalist", "fédéraliste",
+    "language law", "loi linguistique", "bill 101", "loi 101",
+    "bill 96", "loi 96", "bill 21", "loi 21",
+    "secularism", "laïcité", "religious symbols", "signes religieux",
+    "distinct society", "société distincte",
+    "notwithstanding clause", "clause dérogatoire",
+    # ── Canadian Federal Politics ────────────────────────────
+    "parliament", "parlement", "house of commons", "chambre des communes",
+    "senate", "sénat", "governor general", "gouverneur général",
+    "throne speech", "discours du trône",
+    "first past the post", "scrutin uninominal",
+    "electoral reform", "réforme électorale",
+    "equalization", "péréquation", "transfer payments", "paiements de transfert",
+    "carbon tax", "taxe carbone", "carbon pricing", "tarification du carbone",
+    "pipeline", "oil sands", "sables bitumineux",
+    "indigenous", "autochtone", "first nations", "premières nations",
+    "reconciliation", "réconciliation", "treaty", "traité",
+    "indian act", "loi sur les indiens",
+    # ── Public Health (EN/FR) ─────────────────────────────────
+    "public health", "santé publique", "pandemic", "pandémie",
+    "vaccine", "vaccin", "covid", "disease", "maladie",
+    "health transfer", "transfert en santé", "healthcare", "soins de santé",
+    # ── Economy / Budget (EN/FR) ──────────────────────────────
+    "economy", "économie", "inflation", "recession", "récession",
+    "deficit", "déficit", "budget", "tax", "impôt", "taxes", "impôts",
+    "debt", "dette", "spending", "dépenses", "austerity", "austérité",
+    "housing", "logement", "affordable housing", "logement abordable",
+    "interest rate", "taux d'intérêt", "bank of canada", "banque du canada",
+    # ── Rights / Immigration (EN/FR) ──────────────────────────
+    "abortion", "avortement", "civil rights", "droits civils",
+    "human rights", "droits humains", "charter", "charte",
+    "immigration", "refugee", "réfugié", "asylum", "asile",
+    "multiculturalism", "multiculturalisme",
+]
+
+# High-profile figure indicators — Canada-focused bilingual
+HIGH_PROFILE_INDICATORS = [
+    # ── Canadian Federal ──────────────────────────────────────
+    "prime minister", "premier ministre",
+    "pm", "trudeau", "justin trudeau",
+    "poilievre", "pierre poilievre",
+    "singh", "jagmeet singh",
+    "liberal party", "parti libéral",
+    "conservative party", "parti conservateur",
+    "ndp", "nouveau parti démocratique",
+    "bloc", "bloc québécois",
+    "green party", "parti vert",
+    "minister", "ministre", "cabinet",
+    "governor general", "gouverneur général",
+    "senator", "sénateur", "mp", "member of parliament",
+    # ── Quebec Provincial ─────────────────────────────────────
+    "premier", "première ministre",
+    "quebec premier", "premier du québec",
+    "legault", "françois legault",
+    "caq", "coalition avenir québec",
+    "parti québécois", "pq",
+    "québec solidaire", "qs",
+    "liberal party of quebec", "parti libéral du québec",
+    "mna", "député", "députée",
+    "national assembly", "assemblée nationale",
+    # ── Provincial Premiers (other provinces) ─────────────────
+    "ontario premier", "premier ontarien", "ford", "doug ford",
+    "alberta premier", "smith", "danielle smith",
+    "bc premier", "eby", "david eby",
+]
+
+
+@dataclass
+class CriticalityAssessment:
+    """Result of the Criticality Gate check."""
+    is_critical: bool
+    matched_topics: List[str]
+    is_high_profile: bool
+    recommendation: str  # "automated" or "human_review"
+    reason: str
+
+
+class CriticalityGate:
+    """
+    Routes claims based on criticality.
+
+    Jewel [C3] — Human-in-the-loop breakpoint:
+    Automated for scale, human-reviewed for stakes.
+    """
+
+    def assess(self, claim_text: str) -> CriticalityAssessment:
+        """
+        Determine if a claim requires human review.
+
+        Args:
+            claim_text: The claim text to assess.
+
+        Returns:
+            CriticalityAssessment with routing recommendation.
+        """
+        claim_lower = claim_text.lower()
+
+        # Check critical topics
+        matched_topics = [
+            topic for topic in CRITICAL_TOPICS
+            if topic in claim_lower
+        ]
+
+        # Check high-profile figures
+        is_high_profile = any(
+            indicator in claim_lower
+            for indicator in HIGH_PROFILE_INDICATORS
+        )
+
+        # Determine routing
+        is_critical = bool(matched_topics) or is_high_profile
+
+        if is_critical:
+            reason_parts = []
+            if matched_topics:
+                reason_parts.append(
+                    f"matches critical topics: {', '.join(matched_topics[:3])}"
+                )
+            if is_high_profile:
+                reason_parts.append("involves high-profile figure")
+            reason = "; ".join(reason_parts)
+            recommendation = "human_review"
+        else:
+            reason = "No critical topics or high-profile indicators detected."
+            recommendation = "automated"
+
+        return CriticalityAssessment(
+            is_critical=is_critical,
+            matched_topics=matched_topics,
+            is_high_profile=is_high_profile,
+            recommendation=recommendation,
+            reason=reason,
+        )
+
+
+# Singleton
+criticality_gate = CriticalityGate()

package/vendors/piste/pipeline/stage4/orchestrator.py

@@ -0,0 +1,269 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 4 Orchestrator — Verdict
+================================
+Coordinates:
+  4a: Criticality Gate [C3] — route to automated vs human review
+  4b: Verdict Aggregator [J5] — 7-way PolitiFact-aligned verdict
+  4c: Editorial Review Panel [C3] — human-in-the-loop for critical claims
+
+Writes APPEND-ONLY verdict records to PostgreSQL [C5].
+Emits SSE events: criticality_determined, verdict_complete.
+"""
+
+import time
+import uuid
+from typing import Optional, Dict
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import StageRecord, Verdict
+from pipeline.stage4.criticality_gate import criticality_gate
+from pipeline.stage4.verdict_aggregator import (
+    verdict_aggregator, VERDICT_LABELS,
+)
+from pipeline.stage3.orchestrator import Stage3Result, ClassificationResult
+
+
+@dataclass
+class Stage4Result:
+    """Output of Stage 4 — Verdict."""
+    atomic_claim: str
+
+    # Criticality Gate
+    is_critical: bool
+    review_route: str  # "automated" or "human_review"
+    criticality_reason: str
+
+    # Verdict
+    verdict: str  # One of 7 labels
+    confidence: float
+    explanation: str
+    distribution: Dict[str, float]
+
+    # Human review (if applicable)
+    human_review: Optional[dict] = None
+
+    # Stage 3 summary for context
+    support_count: int = 0
+    refute_count: int = 0
+    unrelated_count: int = 0
+
+
+class Stage4Orchestrator:
+    """
+    Orchestrates Stage 4 of the fact-checking pipeline.
+
+    Flow:
+      1. Criticality Gate: assess if human review needed [C3]
+      2a. If automated → VerdictAggregator synthesizes 7-way verdict [J5]
+      2b. If critical → queue for Editorial Review Panel [C3]
+      3. Write verdict record to PostgreSQL (append-only)
+      4. Emit SSE events
+    """
+
+    def __init__(self, sse_callback: Optional[callable] = None):
+        self.sse_callback = sse_callback
+
+    async def process(
+        self,
+        atomic_claim: str,
+        stage3_result: Stage3Result,
+        db: Optional[AsyncSession] = None,
+        run_id: Optional[uuid.UUID] = None,
+        locale: str = "en",
+    ) -> Stage4Result:
+        """
+        Run Stage 4 processing for one atomic claim.
+
+        Args:
+            atomic_claim: The atomic claim to verdict.
+            stage3_result: Per-source classifications from Stage 3.
+            db: Optional DB session for audit ledger writes.
+
+        Returns:
+            Stage4Result with final verdict.
+        """
+        # --- 4a: Criticality Gate ---
+        await self._emit("stage_4a_start", {
+            "atomic_claim": atomic_claim,
+        })
+
+        assessment = criticality_gate.assess(atomic_claim)
+
+        await self._emit("criticality_determined", {
+            "atomic_claim": atomic_claim,
+            "is_critical": assessment.is_critical,
+            "matched_topics": assessment.matched_topics,
+            "recommendation": assessment.recommendation,
+        })
+
+        # --- 4b: Verdict Aggregation ---
+        if assessment.recommendation == "automated":
+            result = await self._automated_verdict(
+                atomic_claim, stage3_result, db, run_id, locale
+            )
+        else:
+            result = await self._human_review_verdict(
+                atomic_claim, stage3_result, assessment, db, run_id, locale
+            )
+
+        # NOTE: Verdict creation moved to pipeline_service.py —
+        # one Verdict per run_id, aggregated from all atomic claims.
+        # Stage 4 only returns per-atomic-claim results.
+        return result
+
+    async def _automated_verdict(
+        self,
+        atomic_claim: str,
+        stage3_result: Stage3Result,
+        db: Optional[AsyncSession],
+        run_id: Optional[uuid.UUID] = None,
+        locale: str = "en",
+    ) -> Stage4Result:
+        """Run automated VerdictAggregator [J5]."""
+        t0 = time.monotonic()
+
+        verdict, confidence, explanation, distribution = verdict_aggregator(
+            claim=atomic_claim,
+            classifications=stage3_result.classifications,
+            locale=locale,
+        )
+
+        latency_ms = (time.monotonic() - t0) * 1000
+
+        # Write stage record
+        if db:
+            db.add(StageRecord(
+                run_id=run_id or uuid.uuid4(),
+                stage_name="stage_4b",
+                input_snapshot={
+                    "atomic_claim": atomic_claim,
+                    "classifications_count": len(stage3_result.classifications),
+                },
+                output_snapshot={
+                    "verdict": verdict,
+                    "confidence": confidence,
+                    "distribution": distribution,
+                },
+                model_used="dspy/verdict_aggregator",
+                latency_ms=latency_ms,
+                retry_attempt=0,
+            ))
+
+        await self._emit("atomic_verdict", {
+            "atomic_claim": atomic_claim,
+            "verdict": verdict,
+            "confidence": confidence,
+            "explanation": explanation,
+            "distribution": distribution,
+            "route": "automated",
+        })
+
+        return Stage4Result(
+            atomic_claim=atomic_claim,
+            is_critical=False,
+            review_route="automated",
+            criticality_reason="Low-stakes claim — automated verdict.",
+            verdict=verdict,
+            confidence=confidence,
+            explanation=explanation,
+            distribution=distribution,
+            support_count=stage3_result.support_count,
+            refute_count=stage3_result.refute_count,
+            unrelated_count=stage3_result.unrelated_count,
+        )
+
+    async def _human_review_verdict(
+        self,
+        atomic_claim: str,
+        stage3_result: Stage3Result,
+        assessment,
+        db: Optional[AsyncSession],
+        run_id: Optional[uuid.UUID] = None,
+        locale: str = "en",
+    ) -> Stage4Result:
+        """
+        Queue claim for human Editorial Review Panel [C3].
+
+        In production: claim added to review_queue table.
+        Human reviewer sees: claim + evidence + per-source classifications.
+        Panel votes on final verdict (PolitiFact-style democratic vote).
+
+        MVP: fall back to automated verdict with human_review flag.
+        """
+        # For MVP, run automated verdict but flag for human review
+        t0 = time.monotonic()
+
+        verdict, confidence, explanation, distribution = verdict_aggregator(
+            claim=atomic_claim,
+            classifications=stage3_result.classifications,
+            locale=locale,
+        )
+
+        latency_ms = (time.monotonic() - t0) * 1000
+
+        human_review_payload = {
+            "status": "pending_review",
+            "matched_topics": assessment.matched_topics,
+            "auto_verdict": verdict,
+            "auto_confidence": confidence,
+            "queued_at": datetime.utcnow().isoformat(),
+            "reviewer_id": None,
+            "final_verdict": None,
+            "reason": None,
+        }
+
+        if db:
+            db.add(StageRecord(
+                run_id=run_id or uuid.uuid4(),
+                stage_name="stage_4c",
+                input_snapshot={
+                    "atomic_claim": atomic_claim,
+                    "assessment": {
+                        "is_critical": assessment.is_critical,
+                        "matched_topics": assessment.matched_topics,
+                    },
+                },
+                output_snapshot=human_review_payload,
+                model_used="dspy/verdict_aggregator+human_review",
+                latency_ms=latency_ms,
+                retry_attempt=0,
+            ))
+
+        await self._emit("verdict_complete", {
+            "atomic_claim": atomic_claim,
+            "verdict": verdict,
+            "confidence": confidence,
+            "explanation": explanation,
+            "distribution": distribution,
+            "route": "human_review_pending",
+            "human_review": human_review_payload,
+        })
+
+        return Stage4Result(
+            atomic_claim=atomic_claim,
+            is_critical=True,
+            review_route="human_review",
+            criticality_reason=assessment.reason,
+            verdict=verdict,
+            confidence=confidence,
+            explanation=explanation,
+            distribution=distribution,
+            human_review=human_review_payload,
+            support_count=stage3_result.support_count,
+            refute_count=stage3_result.refute_count,
+            unrelated_count=stage3_result.unrelated_count,
+        )
+
+    async def _emit(self, event_type: str, data: dict):
+        if self.sse_callback:
+            await self.sse_callback(event_type, data)
+
+
+# Singleton
+stage4_orchestrator = Stage4Orchestrator()

package/vendors/piste/pipeline/stage4/test_stage4.py

@@ -0,0 +1,192 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Unit Tests — Stage 4: Verdict
+===============================
+Tests CriticalityGate [C3], VerdictAggregator [J5],
+7-way verdict labels, Stage4Result.
+
+Run: pytest pipeline/stage4/test_stage4.py -v
+"""
+
+import pytest
+import json
+
+
+# ============================================================
+# Criticality Gate Tests [C3]
+# ============================================================
+
+class TestCriticalityGate:
+    """Test routing of claims to automated vs human review."""
+
+    @pytest.fixture
+    def gate(self):
+        from pipeline.stage4.criticality_gate import CriticalityGate
+        return CriticalityGate()
+
+    def test_election_claim_is_critical(self, gate):
+        """Claims about elections → human review."""
+        result = gate.assess("The election was rigged and the votes were tampered with.")
+        assert result.is_critical is True
+        assert result.recommendation == "human_review"
+        assert "election" in result.matched_topics
+        assert "vote" in result.matched_topics
+
+    def test_public_health_claim_is_critical(self, gate):
+        """Claims about vaccines/pandemics → human review."""
+        result = gate.assess("The vaccine causes severe side effects in children.")
+        assert result.is_critical is True
+        assert "vaccine" in result.matched_topics
+
+    def test_high_profile_figure_is_critical(self, gate):
+        """Claims about presidents → human review."""
+        result = gate.assess("The president signed the executive order yesterday.")
+        assert result.is_critical is True
+        assert result.is_high_profile is True
+
+    def test_neutral_claim_is_automated(self, gate):
+        """Non-critical claims → automated verdict."""
+        result = gate.assess("Water boils at 100 degrees Celsius at sea level.")
+        assert result.is_critical is False
+        assert result.recommendation == "automated"
+
+    def test_empty_claim_is_automated(self, gate):
+        """Empty claims → automated."""
+        result = gate.assess("")
+        assert result.is_critical is False
+
+
+# ============================================================
+# Verdict Aggregator Tests [J5]
+# ============================================================
+
+class TestVerdictAggregator:
+    """Test 7-way verdict normalization and distribution."""
+
+    @pytest.fixture
+    def aggregator(self):
+        from pipeline.stage4.verdict_aggregator import VerdictAggregator
+        return VerdictAggregator()
+
+    def test_normalize_direct_match(self, aggregator):
+        """Direct verdict labels pass through unchanged."""
+        assert aggregator._normalize_verdict("TRUE") == "TRUE"
+        assert aggregator._normalize_verdict("FALSE") == "FALSE"
+        assert aggregator._normalize_verdict("UNVERIFIABLE") == "UNVERIFIABLE"
+        assert aggregator._normalize_verdict("PANTS_ON_FIRE") == "PANTS_ON_FIRE"
+
+    def test_normalize_with_spaces(self, aggregator):
+        """Labels with spaces are normalized."""
+        assert aggregator._normalize_verdict("MOSTLY TRUE") == "MOSTLY_TRUE"
+        assert aggregator._normalize_verdict("HALF TRUE") == "HALF_TRUE"
+        assert aggregator._normalize_verdict("MOSTLY FALSE") == "MOSTLY_FALSE"
+
+    def test_normalize_without_spaces(self, aggregator):
+        """Labels without spaces are normalized."""
+        assert aggregator._normalize_verdict("MOSTLYTRUE") == "MOSTLY_TRUE"
+        assert aggregator._normalize_verdict("HALFTRUE") == "HALF_TRUE"
+
+    def test_normalize_nei(self, aggregator):
+        """NEI → UNVERIFIABLE."""
+        assert aggregator._normalize_verdict("NEI") == "UNVERIFIABLE"
+        assert aggregator._normalize_verdict("NOT ENOUGH INFORMATION") == "UNVERIFIABLE"
+
+    def test_normalize_unknown_fallback(self, aggregator):
+        """Unknown labels → UNVERIFIABLE (safe default)."""
+        assert aggregator._normalize_verdict("SOMETHING WEIRD") == "UNVERIFIABLE"
+
+    def test_default_distribution_centered(self, aggregator):
+        """Default distribution puts all weight on the given verdict."""
+        dist = aggregator._default_distribution("TRUE")
+        assert dist["TRUE"] == 1.0
+        assert dist["FALSE"] == 0.0
+        assert sum(dist.values()) == 1.0
+
+    def test_all_verdict_labels_present(self, aggregator):
+        """Default distribution includes all 7 labels."""
+        from pipeline.stage4.verdict_aggregator import VERDICT_LABELS
+        dist = aggregator._default_distribution("MOSTLY_TRUE")
+        for label in VERDICT_LABELS:
+            assert label in dist
+        assert len(dist) == 7
+
+    def test_build_classifications_payload(self, aggregator):
+        """Classification payload is valid JSON."""
+        from pipeline.stage3.orchestrator import ClassificationResult
+        classifications = [
+            ClassificationResult(0, "a.com", "a.com", "SUPPORTS", 0.9, "Good", 0.9),
+            ClassificationResult(1, "b.com", "b.com", "REFUTES", 0.7, "Bad", 0.5),
+        ]
+        payload = aggregator._build_classifications_payload("Test", classifications)
+        data = json.loads(payload)
+        assert len(data) == 2
+        assert data[0]["label"] == "SUPPORTS"
+        assert data[1]["label"] == "REFUTES"
+
+
+# ============================================================
+# Stage4Result Tests
+# ============================================================
+
+class TestStage4Result:
+    """Test Stage4Result dataclass."""
+
+    def test_automated_result(self):
+        from pipeline.stage4.orchestrator import Stage4Result
+        result = Stage4Result(
+            atomic_claim="Water boils at 100°C.",
+            is_critical=False,
+            review_route="automated",
+            criticality_reason="Low-stakes claim.",
+            verdict="TRUE",
+            confidence=0.95,
+            explanation="Well-established scientific fact.",
+            distribution={"TRUE": 0.95, "MOSTLY_TRUE": 0.05},
+            support_count=3,
+            refute_count=0,
+            unrelated_count=0,
+        )
+        assert result.verdict == "TRUE"
+        assert result.review_route == "automated"
+        assert result.human_review is None
+
+    def test_human_review_result(self):
+        from pipeline.stage4.orchestrator import Stage4Result
+        result = Stage4Result(
+            atomic_claim="The election was fraudulent.",
+            is_critical=True,
+            review_route="human_review",
+            criticality_reason="Matches critical topic: election.",
+            verdict="UNVERIFIABLE",
+            confidence=0.45,
+            explanation="Insufficient credible evidence.",
+            distribution={"UNVERIFIABLE": 1.0},
+            human_review={"status": "pending_review"},
+            support_count=0,
+            refute_count=0,
+            unrelated_count=5,
+        )
+        assert result.is_critical is True
+        assert result.review_route == "human_review"
+        assert result.human_review is not None
+
+
+# ============================================================
+# Verdict Labels Tests
+# ============================================================
+
+class TestVerdictLabels:
+    """Test the 7-way PolitiFact-aligned verdict labels."""
+
+    def test_all_seven_labels(self):
+        from pipeline.stage4.verdict_aggregator import VERDICT_LABELS
+        assert len(VERDICT_LABELS) == 7
+        assert "TRUE" in VERDICT_LABELS
+        assert "MOSTLY_TRUE" in VERDICT_LABELS
+        assert "HALF_TRUE" in VERDICT_LABELS
+        assert "MOSTLY_FALSE" in VERDICT_LABELS
+        assert "FALSE" in VERDICT_LABELS
+        assert "PANTS_ON_FIRE" in VERDICT_LABELS
+        assert "UNVERIFIABLE" in VERDICT_LABELS