mcp-agentic-pipelines 1.0.1

package/vendors/piste/pipeline/stage2/blind_retriever.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 2b: Blind Retriever [J2]
+================================
+Veri-fact.ai's jewel: the retriever executes NEUTRAL search queries
+and NEVER sees the original claim. This prevents confirmation bias
+at the architecture level — not the prompt level.
+
+Executes searches against multiple providers with graceful fallback:
+  Primary: Tavily
+  Fallback: Serper → Google CSE
+
+Also checks FAISS Tier-1 cache before external search [J7].
+"""
+
+import asyncio
+import hashlib
+import time
+from typing import List, Optional
+from dataclasses import dataclass, field
+
+import httpx
+import numpy as np
+from app.core.config import settings
+from app.services.vector_store import faiss_store
+
+
+@dataclass
+class RawSearchResult:
+    """A single raw search result from any provider, before canonical mapping."""
+    provider: str
+    query_used: str
+    title: str
+    url: str
+    snippet: str
+    domain: str = ""
+    raw_data: dict = field(default_factory=dict)
+
+
+class BlindRetriever:
+    """
+    Executes neutral search queries against web search providers.
+
+    CRITICAL ARCHITECTURAL CONSTRAINT [J2]:
+    This module NEVER receives the original claim text.
+    It only sees neutral, factual queries generated by SearchDecisionGenerator.
+    This prevents confirmation bias — the retriever cannot preferentially
+    return sources that support or attack the claim's framing.
+    """
+
+    def __init__(self):
+        self.http_client: Optional[httpx.AsyncClient] = None
+        self._locale: str = "en"  # Default locale
+
+    def set_locale(self, locale: str):
+        """Set locale for region-biased search results [C2]."""
+        self._locale = locale
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self.http_client is None:
+            self.http_client = httpx.AsyncClient(timeout=30.0)
+        return self.http_client
+
+    async def search(
+        self,
+        queries: List[str],
+        top_k: int = 5,
+        locale: str = "en",
+    ) -> List[RawSearchResult]:
+        """
+        Execute searches across all queries with provider fallback.
+
+        Args:
+            queries: Neutral search queries (NEVER the original claim).
+            top_k: Max results per query.
+            locale: Language/region for locale-biased search results [C2].
+
+        Returns:
+            List of RawSearchResult objects from all providers.
+        """
+        self.set_locale(locale)
+        all_results: List[RawSearchResult] = []
+
+        # Check FAISS Tier-1 cache first [J7]
+        cached = await self._check_faiss_cache(queries)
+        if cached:
+            all_results.extend(cached)
+
+        # Execute parallel web searches
+        tasks = [self._search_single_query(q, top_k) for q in queries]
+        results_per_query = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for query, result in zip(queries, results_per_query):
+            if isinstance(result, Exception):
+                continue  # Individual query failure is non-fatal
+            all_results.extend(result)
+
+        # Deduplicate by URL
+        seen_urls = set()
+        unique_results = []
+        for r in all_results:
+            normalized_url = r.url.lower().rstrip("/")
+            if normalized_url not in seen_urls:
+                seen_urls.add(normalized_url)
+                unique_results.append(r)
+
+        return unique_results
+
+    async def _search_single_query(
+        self, query: str, top_k: int
+    ) -> List[RawSearchResult]:
+        """Search across ALL available providers concurrently, then aggregate results."""
+        client = await self._get_client()
+        all_results: List[RawSearchResult] = []
+
+        # Build list of coroutines for each configured provider
+        tasks = []
+        task_names = []
+
+        if settings.TAVILY_API_KEY:
+            tasks.append(self._search_tavily(client, query, top_k))
+            task_names.append("Tavily")
+
+        if settings.SERPER_API_KEY:
+            tasks.append(self._search_serper(client, query, top_k))
+            task_names.append("Serper")
+
+        if settings.GOOGLE_CSE_API_KEY and settings.GOOGLE_CSE_ID:
+            tasks.append(self._search_google_cse(client, query, top_k))
+            task_names.append("Google CSE")
+
+        if not tasks:
+            print(f"[SEARCH] No search providers configured for query: {query[:50]}...", flush=True)
+            return []
+
+        # Execute all providers concurrently
+        gathered = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for name, result in zip(task_names, gathered):
+            if isinstance(result, Exception):
+                print(f"[SEARCH] {name} FAILED: {result}", flush=True)
+            elif isinstance(result, list):
+                print(f"[SEARCH] {name} returned {len(result)} results for query: {query[:50]}...", flush=True)
+                all_results.extend(result)
+            else:
+                print(f"[SEARCH] {name} returned unexpected type: {type(result)}", flush=True)
+
+        return all_results
+
+    async def _search_tavily(
+        self, client: httpx.AsyncClient, query: str, top_k: int
+    ) -> List[RawSearchResult]:
+        """Search via Tavily API with locale-aware region biasing [C2]."""
+        from app.services.locale_service import get_search_region_params
+
+        region_params = get_search_region_params(self._locale)
+        response = await client.post(
+            "https://api.tavily.com/search",
+            json={
+                "query": query,
+                "max_results": top_k,
+                "search_depth": "basic",
+            },
+            headers={
+                "Authorization": f"Bearer {settings.TAVILY_API_KEY}",
+                "Content-Type": "application/json",
+            },
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        results = []
+        for item in data.get("results", [])[:top_k]:
+            domain = self._extract_domain(item.get("url", ""))
+            results.append(RawSearchResult(
+                provider="tavily",
+                query_used=query,
+                title=item.get("title", ""),
+                url=item.get("url", ""),
+                snippet=item.get("content", ""),
+                domain=domain,
+                raw_data=item,
+            ))
+        return results
+
+    async def _search_serper(
+        self, client: httpx.AsyncClient, query: str, top_k: int
+    ) -> List[RawSearchResult]:
+        """Search via Serper.dev (Google Search API) with locale biasing [C2]."""
+        from app.services.locale_service import get_search_region_params
+
+        region_params = get_search_region_params(self._locale)
+        response = await client.post(
+            "https://google.serper.dev/search",
+            json={
+                "q": query,
+                "num": top_k,
+                "gl": region_params.get("gl", "us"),
+                "hl": region_params.get("hl", "en"),
+            },
+            headers={"X-API-KEY": settings.SERPER_API_KEY},
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        results = []
+        for item in data.get("organic", [])[:top_k]:
+            domain = self._extract_domain(item.get("link", ""))
+            results.append(RawSearchResult(
+                provider="serper",
+                query_used=query,
+                title=item.get("title", ""),
+                url=item.get("link", ""),
+                snippet=item.get("snippet", ""),
+                domain=domain,
+                raw_data=item,
+            ))
+        return results
+
+    async def _search_google_cse(
+        self, client: httpx.AsyncClient, query: str, top_k: int
+    ) -> List[RawSearchResult]:
+        """Search via Google Custom Search Engine with Canada/Quebec locale bias [C2]."""
+        # Build locale-aware query with site restrictions
+        locale_query = self._build_locale_query(query)
+        
+        response = await client.get(
+            "https://www.googleapis.com/customsearch/v1",
+            params={
+                "key": settings.GOOGLE_CSE_API_KEY,
+                "cx": settings.GOOGLE_CSE_ID,
+                "q": locale_query,
+                "num": min(top_k, 10),
+                "cr": "countryCA",  # Restrict to Canada
+                "lr": f"lang_{self._locale}",  # Language restriction
+            },
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        results = []
+        for item in data.get("items", [])[:top_k]:
+            domain = self._extract_domain(item.get("link", ""))
+            results.append(RawSearchResult(
+                provider="google_cse",
+                query_used=query,
+                title=item.get("title", ""),
+                url=item.get("link", ""),
+                snippet=item.get("snippet", ""),
+                domain=domain,
+                raw_data=item,
+            ))
+        return results
+
+    def _build_locale_query(self, query: str) -> str:
+        """Add locale-specific site restrictions to focus on Canadian/Quebec sources [C2]."""
+        # Authoritative Canadian sources for political fact-checking
+        ca_sites = [
+            "gc.ca", "canada.ca", "parl.ca", "elections.ca",
+            "cbc.ca", "radio-canada.ca", "ctvnews.ca", "globalnews.ca",
+            "theglobeandmail.com", "macleans.ca", "hilltimes.com",
+            "policyalternatives.ca",
+        ]
+        qc_sites = [
+            "qc.ca", "assnat.qc.ca", "electionsquebec.qc.ca",
+            "lapresse.ca", "ledevoir.com",
+        ]
+        
+        if self._locale == "fr":
+            sites = qc_sites + ca_sites
+        else:
+            sites = ca_sites + qc_sites
+            
+        # Add site: operators (Google CSE supports this)
+        site_filter = " OR ".join(f"site:{s}" for s in sites[:10])
+        return f"({query}) ({site_filter})"
+
+    async def _check_faiss_cache(
+        self, queries: List[str]
+    ) -> List[RawSearchResult]:
+        """Check FAISS Tier-1 cache for semantically similar verified claims."""
+        # In production, embed queries and search FAISS.
+        # For now, return empty — FAISS cache populated in Loop 2 (Phase 8).
+        return []
+
+    @staticmethod
+    def _extract_domain(url: str) -> str:
+        """Extract domain name from URL."""
+        from urllib.parse import urlparse
+        try:
+            parsed = urlparse(url)
+            domain = parsed.netloc.lower()
+            if domain.startswith("www."):
+                domain = domain[4:]
+            return domain
+        except Exception:
+            return ""
+
+
+# Singleton
+blind_retriever = BlindRetriever()

package/vendors/piste/pipeline/stage2/canonical_mapper.py

@@ -0,0 +1,124 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 2e: Canonical Evidence Mapper [C6]
+===========================================
+Normalizes heterogeneous search provider formats into a unified
+CanonicalEvidence schema.
+
+The Replay Architecture's Canonical Mapping pattern applied to evidence:
+  Tavily → CanonicalEvidence
+  Serper → CanonicalEvidence
+  Google CSE → CanonicalEvidence
+  FAISS cache → CanonicalEvidence
+
+All downstream stages (3, 4) operate on CanonicalEvidence ONLY.
+No stage-specific format handling. Adding a new provider requires
+ONLY a new adapter — zero changes to the verification pipeline.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional
+from uuid import UUID
+
+from pipeline.stage2.blind_retriever import RawSearchResult
+
+
+@dataclass
+class CanonicalEvidence:
+    """
+    Unified evidence schema — the single format for all downstream stages.
+
+    Schema [C6]:
+      - title: Source title/headline
+      - url: Full URL to the source
+      - excerpt: Relevant text excerpt from the source
+      - source_domain: Domain name (e.g., bbc.com)
+      - credibility_score: 0.0–1.0 per-domain credibility [J1b]
+      - retrieval_ts: ISO 8601 timestamp of retrieval
+      - query_used: The search query that found this source
+      - provider: Which search provider returned this (tavily, serper, google_cse, faiss)
+      - db_id: PK of the persisted Source row — set by Stage 2 after insert,
+               consumed by Stage 3 to populate classifications.source_id (FK).
+               Ephemeral, not serialized into the JSONB blob.
+    """
+    title: str
+    url: str
+    excerpt: str
+    source_domain: str
+    credibility_score: float
+    retrieval_ts: str = field(default_factory=lambda: datetime.utcnow().isoformat())
+    query_used: str = ""
+    provider: str = ""
+    db_id: Optional[UUID] = None
+
+
+class CanonicalEvidenceMapper:
+    """
+    Normalizes raw search results from any provider into CanonicalEvidence.
+
+    This is the ONLY place that handles provider-specific formats.
+    Downstream stages (classification, verdict) never see raw provider data.
+    """
+
+    async def map_results(
+        self,
+        raw_results: List[RawSearchResult],
+        credibility_scorer=None,  # CredibilityScorer instance
+    ) -> List[CanonicalEvidence]:
+        """
+        Map all raw results to canonical format, attaching credibility scores.
+
+        Args:
+            raw_results: Raw results from BlindRetriever.
+            credibility_scorer: CredibilityScorer for per-domain scoring.
+
+        Returns:
+            List of CanonicalEvidence objects.
+        """
+        canonical_list = []
+
+        for raw in raw_results:
+            # Resolve credibility score
+            cred_score = 0.5  # Default
+            if credibility_scorer:
+                cred_score = await credibility_scorer.score_domain(raw.domain)
+
+            canonical = CanonicalEvidence(
+                title=raw.title,
+                url=raw.url,
+                excerpt=raw.snippet,
+                source_domain=raw.domain,
+                credibility_score=cred_score,
+                query_used=raw.query_used,
+                provider=raw.provider,
+            )
+            canonical_list.append(canonical)
+
+        # Sort by credibility score descending (most credible first)
+        canonical_list.sort(key=lambda x: x.credibility_score, reverse=True)
+
+        return canonical_list
+
+    @staticmethod
+    def to_dict(evidence: CanonicalEvidence) -> dict:
+        """Serialize CanonicalEvidence to dict for JSONB storage."""
+        return {
+            "title": evidence.title,
+            "url": evidence.url,
+            "excerpt": evidence.excerpt,
+            "source_domain": evidence.source_domain,
+            "credibility_score": evidence.credibility_score,
+            "retrieval_ts": evidence.retrieval_ts,
+            "query_used": evidence.query_used,
+            "provider": evidence.provider,
+        }
+
+    @staticmethod
+    def from_dict(data: dict) -> CanonicalEvidence:
+        """Deserialize CanonicalEvidence from JSONB dict."""
+        # db_id is not stored in JSONB (it's the row PK itself); strip if present.
+        clean = {k: v for k, v in data.items() if k != "db_id"}
+        return CanonicalEvidence(**clean)

package/vendors/piste/pipeline/stage2/credibility_scorer.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2026 Jinan Kordab
+# SPDX-License-Identifier: MIT
+
+"""
+Stage 2c: Per-Domain Credibility Scorer [J1b]
+================================================
+Veracity's second jewel: continuous 0–1 credibility scoring per source domain.
+Uses Lin et al. (2023) composite domain quality database.
+
+Unlike Aletheia's binary blacklist, this provides a continuous, auditable metric.
+Users see not just WHICH sources were used, but HOW credible each one is.
+"""
+
+from typing import Optional
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select
+from app.db.models import Domain
+
+
+class CredibilityScorer:
+    """
+    Scores each source domain on a continuous 0–1 credibility scale.
+
+    Jewel [J1b] — Veracity's per-domain credibility:
+    Transforms source transparency from binary (blocked/allowed) into
+    a continuous, auditable metric that educates users about source quality
+    while holding the system accountable.
+    """
+
+    # Domain credibility scores are fetched from PostgreSQL domains table.
+    # Unknown domains get a default score of 0.5 (neutral, flagged for review).
+
+    DEFAULT_SCORE: float = 0.5
+    UNKNOWN_DOMAIN_SCORE: float = 0.5
+
+    def __init__(self, db: Optional[AsyncSession] = None):
+        self.db = db
+        self._cache: dict[str, float] = {}  # In-memory cache per session
+
+    async def score_domain(self, domain: str) -> float:
+        """
+        Get credibility score for a domain.
+
+        Returns:
+            0.0–1.0 score, where 1.0 = highest credibility.
+        """
+        domain = domain.lower().strip()
+
+        # Check in-memory cache
+        if domain in self._cache:
+            return self._cache[domain]
+
+        # Query PostgreSQL domains table
+        if self.db:
+            result = await self.db.execute(
+                select(Domain.credibility_score).where(
+                    Domain.domain_name == domain
+                )
+            )
+            score = result.scalar()
+            if score is not None:
+                self._cache[domain] = float(score)
+                return float(score)
+
+        # Unknown domain → default neutral score
+        self._cache[domain] = self.DEFAULT_SCORE
+        return self.DEFAULT_SCORE
+
+    async def score_domains(
+        self, domains: list[str]
+    ) -> dict[str, float]:
+        """Batch-score multiple domains."""
+        scores = {}
+        for domain in domains:
+            scores[domain] = await self.score_domain(domain)
+        return scores
+
+    async def is_reliable(self, domain: str) -> bool:
+        """Quick binary check: is this domain generally reliable?"""
+        score = await self.score_domain(domain)
+        return score >= 0.6
+
+    def flush_cache(self):
+        """Clear in-memory cache (useful between sessions)."""
+        self._cache.clear()