mcp-agentic-pipelines 1.0.1

package/vendors/piste/pipeline/offline/verifaid_pipeline.py

@@ -0,0 +1,247 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+VERIFAID Offline Dataset Pipeline [J7]
+=========================================
+M1: Generate diverse factual claims (LLM, multilingual)
+M2: Enrich + Label + FAISS Index
+
+Jewel [J7] — VERIFAID's dataset creation as first-class module:
+  - No data bottleneck: system generates its own training data
+  - Self-improving: verified claims enrich the knowledge base
+  - Freshness: claims about current events generated and indexed immediately
+  - Domain expansion: generate claims on any topic, in any language
+
+Scheduled weekly to refresh the FAISS Tier-1 evidence cache.
+Loop 2: newly verified claims automatically enrich the offline index.
+"""
+
+import dspy
+import json
+import asyncio
+from datetime import datetime
+from typing import List
+from dataclasses import dataclass, field
+
+import numpy as np
+from app.services.vector_store import faiss_store
+
+
+# ============================================================
+# DSPy Signatures
+# ============================================================
+
+class ClaimGenerationSignature(dspy.Signature):
+    """Generate diverse factual claims for dataset creation."""
+    topic: str = dspy.InputField(desc="Topic domain to generate claims about")
+    locale: str = dspy.InputField(desc="Language locale for generated claims")
+    count: int = dspy.InputField(desc="Number of claims to generate")
+
+    claims: list[str] = dspy.OutputField(desc="Generated factual claims")
+
+
+class EvidenceLabelingSignature(dspy.Signature):
+    """Generate evidence and labels for a claim."""
+    claim: str = dspy.InputField(desc="The claim to generate evidence for")
+
+    evidence_text: str = dspy.OutputField(desc="Synthetic evidence text supporting or refuting the claim")
+    label: str = dspy.OutputField(desc="TRUE, FALSE, or UNVERIFIABLE")
+    explanation: str = dspy.OutputField(desc="Explanation of the label")
+
+
+# ============================================================
+# M1: Claim Generator
+# ============================================================
+
+TOPICS = [
+    "science", "technology", "health", "politics", "economics",
+    "environment", "education", "sports", "entertainment", "history",
+]
+LOCALES = ["en", "fr", "es"]
+
+
+class ClaimGenerator(dspy.Module):
+    """
+    M1: Generate diverse factual claims across topics and languages.
+
+    Uses DSPy to generate claims that cover a wide range of domains,
+    ensuring the offline index has broad coverage.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.generate = dspy.ChainOfThought(ClaimGenerationSignature)
+
+    def forward(self, topic: str, locale: str = "en", count: int = 10) -> List[str]:
+        result = self.generate(topic=topic, locale=locale, count=count)
+        return [c.strip() for c in result.claims if c.strip()]
+
+    def generate_all(self, topics: List[str] = None, locales: List[str] = None,
+                     per_topic: int = 10) -> List[dict]:
+        """Generate claims across all topics and locales."""
+        topics = topics or TOPICS
+        locales = locales or LOCALES
+        all_claims = []
+
+        for topic in topics:
+            for locale in locales:
+                claims = self(topic=topic, locale=locale, count=per_topic)
+                for claim_text in claims:
+                    all_claims.append({
+                        "topic": topic,
+                        "locale": locale,
+                        "claim_text": claim_text,
+                        "generated_at": datetime.utcnow().isoformat(),
+                    })
+
+        return all_claims
+
+
+# ============================================================
+# M2: Evidence Enricher
+# ============================================================
+
+class EvidenceEnricher(dspy.Module):
+    """
+    M2: Generate evidence, labels, and FAISS vectors for claims.
+
+    For each generated claim, this module:
+      1. Produces synthetic evidence text
+      2. Assigns a label (TRUE/FALSE/UNVERIFIABLE)
+      3. Creates a FAISS vector for semantic retrieval
+      4. Stores metadata in the vector index
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.label = dspy.ChainOfThought(EvidenceLabelingSignature)
+
+    def forward(self, claim: str) -> dict:
+        """Generate evidence and label for a single claim."""
+        result = self.label(claim=claim)
+        return {
+            "claim": claim,
+            "evidence": result.evidence_text,
+            "label": result.label.strip().upper(),
+            "explanation": result.explanation,
+            "enriched_at": datetime.utcnow().isoformat(),
+        }
+
+    async def enrich_batch(self, claims: List[dict], embed_fn=None) -> int:
+        """
+        Enrich a batch of claims and index them in FAISS.
+
+        Args:
+            claims: List of claim dicts from ClaimGenerator.
+            embed_fn: Function to convert text → embedding vector (default: random for demo).
+
+        Returns:
+            Number of claims indexed.
+        """
+        indexed = 0
+        vectors = []
+        metadata_list = []
+
+        for item in claims:
+            try:
+                enriched = self(item["claim_text"])
+
+                # Generate embedding (placeholder — use real embedding model in production)
+                if embed_fn:
+                    vector = embed_fn(enriched["claim"])
+                else:
+                    # Random 1536-dim unit vector as placeholder
+                    vector = np.random.randn(1536).astype("float32")
+                    vector = vector / np.linalg.norm(vector)
+
+                vectors.append(vector)
+                metadata_list.append({
+                    "claim_text": enriched["claim"],
+                    "evidence": enriched["evidence"],
+                    "label": enriched["label"],
+                    "explanation": enriched["explanation"],
+                    "topic": item.get("topic", ""),
+                    "locale": item.get("locale", "en"),
+                    "enriched_at": enriched["enriched_at"],
+                })
+                indexed += 1
+            except Exception:
+                continue  # Skip failed enrichments
+
+        if vectors:
+            faiss_store.add_vectors(
+                np.array(vectors, dtype="float32"),
+                metadata_list,
+            )
+            faiss_store.save()
+
+        return indexed
+
+
+# ============================================================
+# Offline Pipeline Scheduler
+# ============================================================
+
+class OfflinePipelineScheduler:
+    """
+    Scheduled execution of the VERIFAID offline pipeline.
+
+    Runs weekly to:
+      1. Generate new claims across topics/locales (M1)
+      2. Enrich + label + index in FAISS (M2)
+      3. Loop 2: ingest newly verified claims from online pipeline
+    """
+
+    def __init__(self):
+        self.generator = ClaimGenerator()
+        self.enricher = EvidenceEnricher()
+
+    async def run_weekly_job(self):
+        """Execute the full offline pipeline."""
+        print(f"[VERIFAID] Starting weekly dataset pipeline: {datetime.utcnow().isoformat()}")
+
+        # M1: Generate claims
+        claims = self.generator.generate_all()
+        print(f"[VERIFAID] M1 complete: {len(claims)} claims generated")
+
+        # M2: Enrich + index
+        indexed = await self.enricher.enrich_batch(claims)
+        print(f"[VERIFAID] M2 complete: {indexed} claims indexed in FAISS")
+
+        # Loop 2: ingest verified claims
+        # (Phase 8 — reads from PostgreSQL verdicts table)
+        print(f"[VERIFAID] FAISS index size: {len(faiss_store)} vectors")
+
+        return {"claims_generated": len(claims), "claims_indexed": indexed}
+
+    async def ingest_verified_claim(self, claim_text: str, verdict_data: dict):
+        """
+        Loop 2 integration: ingest a newly verified claim into FAISS.
+
+        Called after each successful pipeline verdict.
+        """
+        enriched = self.enricher(claim_text)
+
+        vector = np.random.randn(1536).astype("float32")
+        vector = vector / np.linalg.norm(vector)
+
+        faiss_store.add_vectors(
+            np.array([vector], dtype="float32"),
+            [{
+                "claim_text": claim_text,
+                "evidence": enriched["evidence"],
+                "label": verdict_data.get("verdict", enriched["label"]),
+                "explanation": verdict_data.get("explanation", enriched["explanation"]),
+                "verified": True,
+                "run_id": verdict_data.get("run_id"),
+            }],
+        )
+
+        print(f"[VERIFAID] Loop 2: indexed verified claim in FAISS")
+
+
+# Singletons
+claim_generator = ClaimGenerator()
+evidence_enricher = EvidenceEnricher()
+offline_scheduler = OfflinePipelineScheduler()

package/vendors/piste/pipeline/replay.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Replay Engine [C5] — Re-exports from full implementation.
+==========================================================
+See pipeline/replay_engine.py for the full ReplayEngine class
+with replay_run(), compare_verdicts(), and rollback_pipeline_version().
+"""
+
+from pipeline.replay_engine import (
+    ReplayEngine, ReplayComparison, StageDiff,
+)
+
+__all__ = ["ReplayEngine", "ReplayComparison", "StageDiff"]

package/vendors/piste/pipeline/replay_engine.py

@@ -0,0 +1,249 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Replay Engine [C5] — Full Implementation
+==========================================
+Replay historical claims through the updated pipeline.
+Compare old vs new verdicts. Support rollback to previous pipeline versions.
+
+Reads from the append-only PostgreSQL audit ledger.
+Every replay creates a new immutable ReplayRun record.
+"""
+
+import uuid
+from datetime import datetime
+from typing import Optional, Dict, List
+from dataclasses import dataclass, field
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select
+from app.db.models import AnalysisRun, StageRecord, Verdict, ReplayRun
+
+
+@dataclass
+class StageDiff:
+    """Difference between old and new stage outputs."""
+    stage_name: str
+    old_output: dict
+    new_output: dict
+    changed: bool
+    diff_summary: str
+
+
+@dataclass
+class ReplayComparison:
+    """Full side-by-side comparison of old vs new verdict."""
+    original_run_id: uuid.UUID
+    new_run_id: uuid.UUID
+    original_verdict: str
+    new_verdict: str
+    verdict_changed: bool
+    original_confidence: float
+    new_confidence: float
+    stage_diffs: List[StageDiff]
+    replayed_at: datetime
+
+
+class ReplayEngine:
+    """
+    Replays historical claims through the current pipeline version.
+
+    Jewel [C5] — Append-Only Audit Ledger + Replay:
+    Every pipeline run is immutably recorded. Historical claims can be
+    re-executed through updated pipeline versions to compare verdicts.
+    This enables:
+      - Forensic audit: prove exactly what changed and why
+      - Regression testing: ensure pipeline improvements don't break
+      - Rollback: restore previous pipeline version if needed
+    """
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+
+    async def replay_run(self, original_run_id: uuid.UUID) -> ReplayComparison:
+        """
+        Replay a historical claim through the current pipeline.
+
+        Steps:
+          1. Read original claim text from stage_1a input_snapshot
+          2. Run claim through current pipeline version
+          3. Compare old vs new stage outputs
+          4. Compare old vs new verdict
+          5. Create ReplayRun record (append-only)
+        """
+        # 1. Fetch original run
+        result = await self.db.execute(
+            select(AnalysisRun).where(AnalysisRun.run_id == original_run_id)
+        )
+        original_run = result.scalar()
+        if not original_run:
+            raise ValueError(f"Run not found: {original_run_id}")
+
+        # 2. Fetch original stage records
+        stages_result = await self.db.execute(
+            select(StageRecord)
+            .where(StageRecord.run_id == original_run_id)
+            .order_by(StageRecord.created_at)
+        )
+        original_stages = stages_result.scalars().all()
+
+        # 3. Extract original claim text
+        claim_text = ""
+        for stage in original_stages:
+            if stage.stage_name == "stage_1a":
+                claim_text = stage.input_snapshot.get("claim_text", "")
+
+        if not claim_text:
+            raise ValueError("Could not extract claim text from audit trail")
+
+        # 4. Fetch original verdict
+        verdict_result = await self.db.execute(
+            select(Verdict).where(Verdict.run_id == original_run_id)
+        )
+        original_verdict = verdict_result.scalar()
+
+        # 5. Create new analysis run (replay)
+        new_run_id = uuid.uuid4()
+        new_run = AnalysisRun(
+            claim_id=original_run.claim_id,
+            run_id=new_run_id,
+            status="replaying",
+            pipeline_version="0.1.0",  # Current version
+            started_at=datetime.utcnow(),
+        )
+        self.db.add(new_run)
+
+        # 6. Run pipeline with current version (placeholder — real pipeline in Phase 5)
+        # In production: await pipeline_service.run_pipeline(claim_text, ...)
+        # For now: simulate re-run
+
+        # 7. Fetch new verdict (simulated)
+        new_verdict_result = await self.db.execute(
+            select(Verdict).where(Verdict.run_id == new_run_id)
+        )
+        new_verdict = new_verdict_result.scalar()
+
+        new_run.status = "completed"
+        new_run.completed_at = datetime.utcnow()
+
+        # 8. Compare stage-by-stage
+        stage_diffs = self._compute_stage_diffs(original_stages, [])
+
+        # 9. Compare verdicts
+        verdict_changed = False
+        if original_verdict and new_verdict:
+            verdict_changed = original_verdict.verdict != new_verdict.verdict
+
+        comparison = ReplayComparison(
+            original_run_id=original_run_id,
+            new_run_id=new_run_id,
+            original_verdict=original_verdict.verdict if original_verdict else "N/A",
+            new_verdict=new_verdict.verdict if new_verdict else "N/A",
+            verdict_changed=verdict_changed,
+            original_confidence=original_verdict.confidence if original_verdict else 0.0,
+            new_confidence=new_verdict.confidence if new_verdict else 0.0,
+            stage_diffs=stage_diffs,
+            replayed_at=datetime.utcnow(),
+        )
+
+        # 10. Create ReplayRun record (append-only)
+        replay_record = ReplayRun(
+            original_run_id=original_run.id,
+            new_run_id=new_run.id,
+            pipeline_version="0.1.0",
+            verdict_changed=verdict_changed,
+            verdict_comparison={
+                "original_verdict": comparison.original_verdict,
+                "new_verdict": comparison.new_verdict,
+                "original_confidence": comparison.original_confidence,
+                "new_confidence": comparison.new_confidence,
+                "stage_diffs": [
+                    {
+                        "stage": d.stage_name,
+                        "changed": d.changed,
+                        "summary": d.diff_summary,
+                    }
+                    for d in stage_diffs
+                ],
+            },
+        )
+        self.db.add(replay_record)
+        await self.db.commit()
+
+        return comparison
+
+    def _compute_stage_diffs(
+        self,
+        original_stages: List[StageRecord],
+        new_stages: List[StageRecord],
+    ) -> List[StageDiff]:
+        """Compute per-stage differences between old and new pipeline runs."""
+        diffs = []
+        new_by_stage = {s.stage_name: s for s in new_stages}
+
+        for old_stage in original_stages:
+            new_stage = new_by_stage.get(old_stage.stage_name)
+            if new_stage is None:
+                diffs.append(StageDiff(
+                    stage_name=old_stage.stage_name,
+                    old_output=old_stage.output_snapshot,
+                    new_output={},
+                    changed=True,
+                    diff_summary=f"Stage {old_stage.stage_name} not present in new run.",
+                ))
+                continue
+
+            changed = old_stage.output_snapshot != new_stage.output_snapshot
+            diffs.append(StageDiff(
+                stage_name=old_stage.stage_name,
+                old_output=old_stage.output_snapshot,
+                new_output=new_stage.output_snapshot,
+                changed=changed,
+                diff_summary=(
+                    f"Output changed." if changed
+                    else f"Output identical."
+                ),
+            ))
+
+        return diffs
+
+    async def compare_verdicts(
+        self, original_run_id: uuid.UUID, new_run_id: uuid.UUID
+    ) -> dict:
+        """Side-by-side comparison of two specific verdicts."""
+        orig = await self.db.execute(
+            select(Verdict).where(Verdict.run_id == original_run_id)
+        )
+        new = await self.db.execute(
+            select(Verdict).where(Verdict.run_id == new_run_id)
+        )
+        o = orig.scalar()
+        n = new.scalar()
+
+        return {
+            "original": {
+                "verdict": o.verdict if o else None,
+                "confidence": o.confidence if o else None,
+                "explanation": o.explanation if o else None,
+            },
+            "new": {
+                "verdict": n.verdict if n else None,
+                "confidence": n.confidence if n else None,
+                "explanation": n.explanation if n else None,
+            },
+            "changed": o.verdict != n.verdict if o and n else True,
+        }
+
+    async def rollback_pipeline_version(self, version: str) -> dict:
+        """
+        Flag a pipeline version for rollback.
+
+        In production: loads the DSPy module checkpoints from that version.
+        """
+        return {
+            "status": "rollback_flagged",
+            "version": version,
+            "message": f"Pipeline version {version} flagged for rollback. "
+                       f"Restart backend to apply.",
+        }

package/vendors/piste/pipeline/signatures/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+# DSPy Signatures — typed interfaces for all pipeline modules

package/vendors/piste/pipeline/signatures/signatures.py

@@ -0,0 +1,136 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+DSPy Signatures — Typed Interfaces for the Piste Pipeline
+==========================================================
+Every LLM call in the pipeline is defined as a typed DSPy Signature.
+This makes modules model-agnostic, testable, and compiler-optimizable.
+"""
+
+import dspy
+
+
+# --- Stage 1: Claim Processing ---
+
+class CheckWorthinessSignature(dspy.Signature):
+    """Classify whether a claim is worth fact-checking (CFC/UFC/NFC).
+
+    Jewel [J4] — ClaimBuster's pre-filter: find the needles before examining them.
+    """
+
+    claim_text: str = dspy.InputField(desc="The raw claim text to evaluate")
+    locale: str = dspy.InputField(desc="Language locale of the claim (en, fr, es, ...)")
+
+    label: str = dspy.OutputField(
+        desc="CFC (Check-worthy Factual Claim), UFC (Unimportant Factual Claim), or NFC (Non-Factual Claim)"
+    )
+    confidence: float = dspy.OutputField(desc="Confidence in the classification (0.0–1.0)")
+    rationale: str = dspy.OutputField(desc="Brief explanation of why this classification was assigned")
+
+
+class AtomicClaimDecompositionSignature(dspy.Signature):
+    """Decompose a compound claim into independent atomic claims.
+
+    Jewel [J7] — FACT5's atomization: each atomic claim focuses on one verifiable fact.
+    """
+
+    claim_text: str = dspy.InputField(desc="The claim text to decompose (may be compound)")
+
+    atomic_claims: list[str] = dspy.OutputField(
+        desc="List of independent atomic claims, each a single verifiable fact"
+    )
+
+
+# --- Stage 2: Blind Retrieval ---
+
+class SearchDecisionSignature(dspy.Signature):
+    """Decide whether web search is needed for this claim.
+
+    Jewel [J1] — Veracity's LLM-autonomous search decision:
+    skip search for well-known facts to save cost and latency.
+    """
+
+    atomic_claim: str = dspy.InputField(desc="A single atomic claim to evaluate")
+
+    needs_search: bool = dspy.OutputField(
+        desc="True if external evidence is needed; False if answerable from parametric knowledge"
+    )
+    reasoning: str = dspy.OutputField(desc="Why search is or is not needed")
+
+
+class QueryGenerationSignature(dspy.Signature):
+    """Generate NEUTRAL search queries from a claim.
+
+    Jewel [J2] — Blind Retrieval: queries must be factual and neutral.
+    NEVER include the original claim text — prevents confirmation bias.
+    """
+
+    atomic_claim: str = dspy.InputField(desc="The atomic claim to search evidence for")
+
+    search_queries: list[str] = dspy.OutputField(
+        desc="Neutral, factual search queries. Must NOT contain the claim text or biased framing."
+    )
+
+
+class QueryRefinementSignature(dspy.Signature):
+    """Analyze why previous search was insufficient and generate refined queries.
+
+    Jewel [J8c] — ClaimeAI's feedback-driven iterative query refinement.
+    Loop 1: seconds-scale retry with intelligent query adjustment.
+    """
+
+    original_query: str = dspy.InputField(desc="The query that returned insufficient results")
+    insufficient_reason: str = dspy.InputField(desc="Why the previous results were insufficient")
+
+    refined_queries: list[str] = dspy.OutputField(
+        desc="New, refined search queries targeting the identified gaps"
+    )
+
+
+# --- Stage 3: Per-Source Classification ---
+
+class SourceClassificationSignature(dspy.Signature):
+    """Classify a single evidence source as supporting, refuting, or unrelated to a claim.
+
+    Jewel [J3] — Aletheia's structured per-source classification:
+    each source evaluated independently BEFORE aggregation.
+    """
+
+    claim: str = dspy.InputField(desc="The atomic claim being verified")
+    evidence_title: str = dspy.InputField(desc="Title of the evidence source")
+    evidence_excerpt: str = dspy.InputField(desc="Relevant excerpt from the evidence source")
+    source_domain: str = dspy.InputField(desc="Domain name of the source (e.g., bbc.com)")
+    credibility_score: float = dspy.InputField(desc="Pre-computed domain credibility (0.0–1.0)")
+    locale: str = dspy.InputField(desc="Language locale for the response (en, fr, ...)")
+
+    label: str = dspy.OutputField(desc="SUPPORTS, REFUTES, or UNRELATED")
+    confidence: float = dspy.OutputField(desc="Confidence in the classification (0.0–1.0)")
+    rationale: str = dspy.OutputField(desc="Brief explanation of why this label was assigned")
+
+
+# --- Stage 4: Verdict Aggregation ---
+
+class VerdictAggregationSignature(dspy.Signature):
+    """Synthesize per-source classifications into a final 7-way verdict.
+
+    Jewel [J5] — DSPy-powered aggregation with PolitiFact-aligned granularity.
+    Weighted by source credibility scores.
+    """
+
+    claim: str = dspy.InputField(desc="The atomic claim being verified")
+    classifications_json: str = dspy.InputField(
+        desc="JSON array of per-source classifications with labels, confidences, and rationales"
+    )
+    locale: str = dspy.InputField(desc="Language locale for the response (en, fr, ...)")
+
+    verdict: str = dspy.OutputField(
+        desc="TRUE, MOSTLY_TRUE, HALF_TRUE, MOSTLY_FALSE, FALSE, PANTS_ON_FIRE, or UNVERIFIABLE"
+    )
+    confidence: float = dspy.OutputField(desc="Overall confidence in the verdict (0.0–1.0)")
+    explanation: str = dspy.OutputField(
+        desc="Natural language explanation of the verdict with source citations"
+    )
+    distribution_json: str = dspy.OutputField(
+        desc="JSON object mapping each verdict label to its probability weight"
+    )

package/vendors/piste/pipeline/stage1/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+# Stage 1 — Claim Processing
+#   1a: Check-Worthiness Detector [J4] — pipeline/stage1/check_worthiness.py
+#   1b: Atomic Claim Decomposer [J7] — pipeline/stage1/atomic_decomposer.py
+#   Orchestrator: pipeline/stage1/orchestrator.py
+
+from pipeline.stage1.check_worthiness import CheckWorthinessDetector, check_worthiness_detector
+from pipeline.stage1.atomic_decomposer import AtomicClaimDecomposer, atomic_claim_decomposer
+from pipeline.stage1.orchestrator import Stage1Orchestrator, Stage1Result, stage1_orchestrator
+
+__all__ = [
+    "CheckWorthinessDetector",
+    "check_worthiness_detector",
+    "AtomicClaimDecomposer",
+    "atomic_claim_decomposer",
+    "Stage1Orchestrator",
+    "Stage1Result",
+    "stage1_orchestrator",
+]