mcp-agentic-pipelines 1.0.1

package/vendors/precis/backend/agents/radial_interpol.py

@@ -0,0 +1,99 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import numpy as np
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+
+@dataclass
+class TrainingNode:
+    beta: np.ndarray
+    response: float
+    metadata: Dict = field(default_factory=dict)
+
+
+@dataclass
+class PredictionResult:
+    predicted_value: float
+    confidence: float
+    contributing_nodes: List[Dict]
+    metadata: Dict = field(default_factory=dict)
+
+
+class RadialInterpolPredictor:
+    """RBF closed-form predictor. f_pred(x) = Σ ω_k(x)·f(β_k)·exp[-τ·K(x,β_k)] with Σ ω_k·exp[-τ·K] = 1."""
+
+    def __init__(self, tau: float = 500.0, gamma: float = 1.0) -> None:
+        self.tau = tau
+        self.gamma = gamma
+        self.nodes: List[TrainingNode] = []
+        self.n: int = 0
+        self.m: int = 0
+        self._X_min: Optional[np.ndarray] = None
+        self._X_max: Optional[np.ndarray] = None
+        self._node_access_counts: np.ndarray = np.array([])
+        self._total_predictions: int = 0
+
+    def fit(self, X: np.ndarray, y: np.ndarray, metadata: Optional[List[Dict]] = None) -> None:
+        self._X_min = X.min(axis=0)
+        self._X_max = X.max(axis=0)
+        X_t = self._phi_transform(X)
+        meta_list = metadata or [{}] * len(X)
+        self.nodes = [TrainingNode(beta=X_t[i], response=y[i], metadata=meta_list[i]) for i in range(len(X))]
+        self.n = len(self.nodes)
+        self.m = X.shape[1]
+        self._node_access_counts = np.zeros(self.n)
+
+    def predict(self, x: np.ndarray, top_k: int = 20,
+                trace=None) -> Tuple[float, List[Dict]]:
+        if self.n == 0:
+            return 0.0, []
+        x_t = self._phi_transform(x.reshape(1, -1))[0]
+        contributions, w_sum, w_norm = [], 0.0, 0.0
+        for i, node in enumerate(self.nodes):
+            K = self._kernel(x_t, node.beta)
+            w = np.exp(-self.tau * K)
+            if w > 1e-15:
+                w_sum += w * node.response
+                w_norm += w
+                self._node_access_counts[i] += 1
+                contributions.append({"node_idx": i, "weight": float(w), "kernel_distance": float(K),
+                                       "response": float(node.response), "metadata": node.metadata})
+        if w_norm == 0:
+            return 0.0, []
+        f_pred = w_sum / w_norm
+        self._total_predictions += 1
+        contributions.sort(key=lambda c: c["weight"], reverse=True)
+        if trace:
+            trace.event(type("TE", (), {"value": "decision.prediction"})(), agent_name="RadialInterpol",
+                        message=f"Predicted {f_pred:.4f} from {len(contributions)} active nodes",
+                        data={"top_weight": contributions[0]["weight"] if contributions else 0})
+        return f_pred, contributions[:top_k]
+
+    def _kernel(self, x: np.ndarray, beta: np.ndarray) -> float:
+        return self.gamma * float(np.dot(x - beta, x - beta))
+
+    def _phi_transform(self, X: np.ndarray) -> np.ndarray:
+        if self._X_min is None or self._X_max is None:
+            return X
+        denom = self._X_max - self._X_min
+        denom[denom == 0] = 1.0
+        return (X - self._X_min) / denom
+
+    def auto_distill(self, min_access: int = 0) -> int:
+        if self._total_predictions == 0:
+            return 0
+        keep = self._node_access_counts >= min_access
+        removed = self.n - int(np.sum(keep))
+        self.nodes = [n for i, n in enumerate(self.nodes) if keep[i]]
+        self._node_access_counts = self._node_access_counts[keep]
+        self.n = len(self.nodes)
+        return removed
+
+    def get_weights_distribution(self, x: np.ndarray) -> np.ndarray:
+        if self.n == 0:
+            return np.array([])
+        x_t = self._phi_transform(x.reshape(1, -1))[0]
+        weights = np.array([np.exp(-self.tau * self._kernel(x_t, n.beta)) for n in self.nodes])
+        s = weights.sum()
+        return weights / s if s > 0 else weights

package/vendors/precis/backend/agents/report_generator.py

@@ -0,0 +1,92 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class ReportSection:
+    section_id: str = ""
+    title: str = ""
+    content: str = ""
+    citations: List[Dict[str, Any]] = field(default_factory=list)
+    agent_source: str = ""
+    relevance: float = 1.0
+
+
+class ReportGenerator:
+    """Assembles agent results into a structured report with citations."""
+
+    def __init__(self) -> None:
+        pass
+
+    async def generate(self, query: str, agent_results: List[Any],
+                       veriscore_report: Optional[Any] = None,
+                       guardrail_result: Optional[Any] = None) -> dict:
+        sections = []
+        all_citations = []
+
+        for i, result in enumerate(agent_results):
+            if not result.success:
+                sections.append({"title": f"Agent: {result.agent_name}",
+                                 "content": f"Error: {result.error_message}",
+                                 "agent_source": result.agent_name})
+                continue
+
+            data = result.data or {}
+            
+            # Build human-readable content from structured data
+            if isinstance(data, dict):
+                parts = []
+                synth = data.get("synthesis", "")
+                if synth:
+                    parts.append(str(synth))
+                for item in data.get("results", []):
+                    if isinstance(item, dict):
+                        txt = item.get("text", "")
+                        src = item.get("source", "")
+                        score = item.get("score", "")
+                        mt = item.get("match_type", "")
+                        if txt:
+                            line = str(txt)
+                            if src:
+                                line += f"  [{src}]"
+                            if score:
+                                line += f"  [score={score}, {mt}]"
+                            parts.append(line)
+                pred = data.get("prediction")
+                if pred is not None:
+                    parts.append(f"Prediction: {pred}")
+                for flag in data.get("flags", []):
+                    parts.append(str(flag))
+                content = "\n".join(parts) if parts else "(no readable content)"
+            else:
+                content = str(data)[:1000]
+
+            for citation in getattr(result, "citations", []):
+                all_citations.append({**citation, "agent_source": result.agent_name})
+
+            sections.append({"title": f"Findings from {result.agent_name}",
+                             "content": content, "agent_source": result.agent_name,
+                             "citations": getattr(result, "citations", [])})
+
+        evaluation_summary = None
+        if veriscore_report:
+            evaluation_summary = {
+                "relevancy": getattr(veriscore_report, "relevancy_score", 0),
+                "trust": getattr(veriscore_report, "trustworthiness_score", 0),
+                "hallucination_rate": getattr(veriscore_report, "hallucination_rate", 0),
+                "citation_coverage": getattr(veriscore_report, "citation_coverage", 0),
+            }
+
+        guardrail_summary = None
+        if guardrail_result:
+            guardrail_summary = {
+                "action": getattr(guardrail_result, "action", "pass"),
+                "issues": getattr(guardrail_result, "issues_found", []),
+            }
+
+        return {"query": query, "sections": sections, "citations": all_citations,
+                "evaluation": evaluation_summary, "guardrail": guardrail_summary,
+                "generated_at": datetime.now().isoformat()}

package/vendors/precis/backend/agents/semantic_reranker.py

@@ -0,0 +1,135 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT
+
+No embeddings, no vector DB — the LLM reads candidate text and scores relevance 0-100.
+Only passes the best candidates to the synthesis step.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+from backend.llm.base import LLMProvider
+
+
+class SemanticReRanker:
+    """Uses LLM semantic understanding to filter hash-retrieved candidates by true relevance."""
+
+    def __init__(self, llm: LLMProvider) -> None:
+        self.llm = llm
+
+    async def rerank(self, query: str, candidates: List[Dict[str, Any]],
+                     top_k: int = 5) -> List[Dict[str, Any]]:
+        """Score each candidate by semantic relevance to the query. Returns top_k.
+
+        Each candidate: {"text": str, "source": str, "score": float, "page": int, ...}
+        """
+        if not candidates:
+            return []
+        if len(candidates) <= top_k:
+            return candidates
+
+        # Build a scoring prompt with numbered candidates (use surrounding context when available)
+        items = []
+        for i, c in enumerate(candidates[:20]):
+            # Prefer surrounding context (full paragraph) over short n-gram text
+            text = c.get('surrounding', '') or c.get('sentence', '') or c.get('text', '')
+            items.append(f"[{i}] {text[:300]}")
+
+        prompt = f"""You are a precise relevance judge. Score each text chunk below for how well
+it answers this query: "{query}"
+
+For each chunk, give a score from 0-100:
+  90-100: Directly answers the query with specific facts
+  70-89:  Related and useful context
+  40-69:  Tangentially related
+  0-39:   Not relevant
+
+Text chunks:
+{chr(10).join(items)}
+
+Return ONLY a JSON array: [{{"index": 0, "score": 85, "reason": "5 words"}}, ...]
+Score ALL chunks. Be strict — only give high scores for truly relevant content."""
+
+        try:
+            import asyncio, json
+            response = await asyncio.wait_for(
+                self.llm.generate(prompt, max_tokens=200, temperature=0.0),
+                timeout=20
+            )
+            # Parse the JSON array
+            start = response.find("[")
+            end = response.rfind("]") + 1
+            if start >= 0 and end > start:
+                scores = json.loads(response[start:end])
+                # Map scores back to candidates
+                scored = []
+                for s in scores:
+                    idx = s.get("index", 0)
+                    if 0 <= idx < len(candidates):
+                        candidates[idx]["semantic_score"] = s.get("score", 0)
+                        candidates[idx]["relevance_reason"] = s.get("reason", "")
+                        scored.append(candidates[idx])
+                # Sort by semantic score, return top_k
+                scored.sort(key=lambda c: c.get("semantic_score", 0), reverse=True)
+                return scored[:top_k]
+        except Exception:
+            pass
+
+        # Fallback: return top by original hash score
+        return sorted(candidates, key=lambda c: c.get("score", 0), reverse=True)[:top_k]
+
+
+class DirectReader:
+    """When hash search finds nothing, ask the LLM to directly read document text and answer."""
+
+    def __init__(self, llm: LLMProvider) -> None:
+        self.llm = llm
+
+    async def read_and_answer(self, query: str, doc_snippets: List[Dict[str, str]],
+                               index) -> Dict[str, Any]:
+        """Read document snippets directly and attempt to answer the query.
+
+        doc_snippets: [{"text": "...", "source": "file.pdf", "page": 1}, ...]
+        """
+        if not doc_snippets:
+            return {"found": False, "answer": "No documents have been uploaded yet. Please upload a document to search.", "citation": ""}
+
+        doc_names = ", ".join(d.get("source", "unknown") for d in doc_snippets[:5])
+
+        context = "\n\n".join(
+            f"[DOC: {d['source']}, page {d.get('page', 1)}]\n{d['text'][:3000]}"
+            for d in doc_snippets[:5]
+        )
+
+        prompt = f"""You are a precise document analyst. Read the document excerpts below
+and answer this query: "{query}"
+
+Documents available: {doc_names}
+
+DOCUMENTS:
+{context[:15000]}
+
+INSTRUCTIONS:
+- If the query mentions a section number (like "section 3.4" or just "3.4"), locate
+  ALL content under that section heading in the documents and extract the key points.
+  Section headings may appear as "3.4 Title", "§3.4", or just "3.4" followed by text.
+- Summarize what that section actually says — do not just report that the heading exists.
+- If the query asks for "key findings" or a "summary", provide the substantive content
+  even if the document doesn't explicitly label anything as "key findings."
+- If you genuinely cannot find the section content anywhere in the provided text,
+  only then state that it's not found.
+
+Return your answer in this JSON format:
+{{"found": true/false, "answer": "your detailed answer", "citation": "document name, page X"}}"""
+
+        try:
+            import asyncio, json
+            response = await asyncio.wait_for(
+                self.llm.generate(prompt, max_tokens=300, temperature=0.2),
+                timeout=25
+            )
+            start = response.find("{")
+            end = response.rfind("}") + 1
+            if start >= 0 and end > start:
+                result = json.loads(response[start:end])
+                return result
+            return {"found": False, "answer": response[:500], "citation": ""}
+        except Exception as e:
+            return {"found": False, "answer": f"(LLM reading unavailable: {e})", "citation": ""}

package/vendors/precis/backend/agents/stat_anomaly.py

@@ -0,0 +1,93 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import numpy as np
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+
+@dataclass
+class AnomalyFlag:
+    entity_id: str
+    flag_type: str
+    severity: float
+    evidence: Dict[str, Any]
+    timestamp: datetime
+    description: str
+    recommendations: List[str] = field(default_factory=list)
+
+
+class StatAnomalyDetector:
+    """Multi-method anomaly detector: entity mapping, temporal spikes, geospatial, value outliers."""
+
+    def __init__(self) -> None:
+        self.entity_hash: Dict[str, Dict[str, int]] = defaultdict(dict)
+        self.temporal_hash: Dict[str, List[Tuple[datetime, float]]] = defaultdict(list)
+        self.geo_hash: Dict[str, List[str]] = defaultdict(list)
+        self.value_tracker: Dict[str, List[float]] = defaultdict(list)
+        self.flags: List[AnomalyFlag] = []
+
+    def ingest(self, entity_id: str, counterparty_id: Optional[str] = None,
+               value: Optional[float] = None, timestamp: Optional[datetime] = None,
+               jurisdiction: Optional[str] = None, metadata: Optional[Dict] = None) -> None:
+        if counterparty_id:
+            self.entity_hash[entity_id][counterparty_id] = self.entity_hash[entity_id].get(counterparty_id, 0) + 1
+        if timestamp is not None and value is not None:
+            self.temporal_hash[entity_id].append((timestamp, value))
+        if jurisdiction:
+            self.geo_hash[entity_id].append(jurisdiction)
+        if value is not None:
+            self.value_tracker[entity_id].append(value)
+
+    def ingest_batch(self, events: List[Dict[str, Any]]) -> int:
+        for evt in events:
+            self.ingest(**{k: evt.get(k) for k in ("entity_id", "counterparty_id", "value", "timestamp", "jurisdiction")})
+        return len(events)
+
+    def detect_multi_entity_anomalies(self, max_relationships: int = 50) -> List[AnomalyFlag]:
+        flags = []
+        for eid, counterparties in self.entity_hash.items():
+            n = len(counterparties)
+            if n > max_relationships:
+                flags.append(AnomalyFlag(entity_id=eid, flag_type="multi_entity",
+                    severity=min(1.0, n / (2 * max_relationships)),
+                    evidence={"n_counterparties": n, "counterparties": dict(counterparties)},
+                    timestamp=datetime.now(),
+                    description=f"Entity has {n} counterparties (threshold: {max_relationships})"))
+        return sorted(flags, key=lambda f: f.severity, reverse=True)
+
+    def detect_temporal_spikes(self, spike_threshold_sigma: float = 3.0,
+                                window_hours: int = 1) -> List[AnomalyFlag]:
+        flags = []
+        for eid, events in self.temporal_hash.items():
+            if len(events) < 10:
+                continue
+            hourly = defaultdict(list)
+            for ts, amount in events:
+                hour_key = ts.replace(minute=0, second=0, microsecond=0)
+                hourly[hour_key].append(amount)
+            counts = [len(v) for v in hourly.values()]
+            mean_c, std_c = float(np.mean(counts)), float(np.std(counts))
+            if std_c == 0:
+                continue
+            for hour, amounts in hourly.items():
+                z = (len(amounts) - mean_c) / std_c
+                if z > spike_threshold_sigma:
+                    flags.append(AnomalyFlag(entity_id=eid, flag_type="temporal_spike",
+                        severity=min(1.0, z / (2 * spike_threshold_sigma)),
+                        evidence={"hour": str(hour), "count": len(amounts), "total_amount": sum(amounts), "z_score": float(z)},
+                        timestamp=hour,
+                        description=f"{len(amounts)} events at {hour} ({z:.1f} above mean)"))
+        return sorted(flags, key=lambda f: f.severity, reverse=True)
+
+    def detect_all(self, trace=None) -> List[AnomalyFlag]:
+        self.flags = []
+        self.flags.extend(self.detect_multi_entity_anomalies())
+        self.flags.extend(self.detect_temporal_spikes())
+        self.flags.sort(key=lambda f: f.severity, reverse=True)
+        if trace:
+            for flag in self.flags[:5]:
+                trace.event(type("TE", (), {"value": "anomaly.flagged"})(), agent_name="StatAnomaly",
+                            message=flag.description, data={"severity": flag.severity, "type": flag.flag_type})
+        return self.flags

package/vendors/precis/backend/agents/vector_index.py

@@ -0,0 +1,123 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT
+
+Adds embedding-based retrieval alongside the hash index for multi-source fusion.
+Uses all-MiniLM-L6-v2 (80MB model, 384-dim vectors) — runs locally, no API calls.
+"""
+
+import numpy as np
+from typing import Any, Dict, List, Optional, Tuple
+
+try:
+    import faiss
+    from sentence_transformers import SentenceTransformer
+    HAS_FAISS = True
+except ImportError:
+    HAS_FAISS = False
+
+
+class VectorIndex:
+    """Semantic document retrieval using FAISS in-memory vector index.
+
+    Complements the hash index by finding semantically similar content
+    that doesn't share exact words with the query.
+
+    Parameters
+    ----------
+    model_name : str
+        HuggingFace sentence-transformers model name.
+    encode_batch_size : int
+        Batch size for model.encode().  Larger values (64–128) encode
+        many chunks faster on CPU; reduce to 8–16 if GPU OOM occurs.
+    """
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2",
+                 encode_batch_size: int = 64) -> None:
+        if not HAS_FAISS:
+            raise ImportError("faiss-cpu and sentence-transformers required: pip install faiss-cpu sentence-transformers")
+        
+        self.model = SentenceTransformer(model_name)
+        self.dimension = self.model.get_sentence_embedding_dimension()  # 384
+        self.encode_batch_size = encode_batch_size
+        self.index = faiss.IndexFlatIP(self.dimension)  # Inner product (cosine with normalized vectors)
+        self.chunks: List[Dict[str, Any]] = []  # {"text": ..., "source": ..., "page": ...}
+        self._chunk_vectors: Optional[np.ndarray] = None
+
+    def index_text(self, text: str, source: str = "unknown",
+                   chunk_size: int = 300, chunk_overlap: int = 50) -> int:
+        """Split text into overlapping chunks and index as vectors. Returns chunk count."""
+        words = text.split()
+        chunks = []
+        start = 0
+        while start < len(words):
+            end = min(start + chunk_size, len(words))
+            chunk = " ".join(words[start:end])
+            chunks.append({"text": chunk, "source": source, "page": 1})
+            start += chunk_size - chunk_overlap
+        
+        if not chunks:
+            return 0
+        
+        # Encode in batches for throughput — sentence-transformers uses an
+        # internal batch loop; a larger batch_size reduces Python→C round-trips.
+        vectors = self.model.encode(
+            [c["text"] for c in chunks],
+            batch_size=self.encode_batch_size,
+            show_progress_bar=False,
+        )
+        # Normalize for cosine similarity (inner product on unit vectors = cosine)
+        faiss.normalize_L2(vectors)
+        self.index.add(vectors.astype(np.float32))
+        self.chunks.extend(chunks)
+        return len(chunks)
+
+    def search(self, query: str, top_k: int = 10,
+               source_filter: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """Search for semantically similar chunks. Returns scored results.
+
+        Parameters
+        ----------
+        source_filter : Optional[List[str]]
+            If provided, only return chunks from these source documents
+            (case-insensitive basename matching).
+        """
+        if self.index.ntotal == 0:
+            return []
+        
+        query_vec = self.model.encode([query], show_progress_bar=False)
+        faiss.normalize_L2(query_vec)
+        # Fetch more candidates than needed so filtering doesn't starve results
+        fetch_k = min(top_k * 3, self.index.ntotal) if source_filter else min(top_k, self.index.ntotal)
+        scores, indices = self.index.search(query_vec.astype(np.float32), fetch_k)
+        
+        results = []
+        for score, idx in zip(scores[0], indices[0]):
+            if idx >= 0 and idx < len(self.chunks):
+                chunk = self.chunks[idx]
+                # ── Document-scope filter ────────────────────────
+                if source_filter:
+                    import os
+                    filter_set = set()
+                    for f in source_filter:
+                        f = str(f).lower().strip()
+                        f = os.path.basename(f)
+                        if f:
+                            filter_set.add(f)
+                    if filter_set and os.path.basename(chunk.get("source", "").lower().strip()) not in filter_set:
+                        continue
+                results.append({
+                    "text": chunk["text"],
+                    "source": chunk["source"],
+                    "page": chunk.get("page", 1),
+                    "score": round(float(score), 3),  # Cosine similarity, 0-1
+                    "match_type": "semantic",
+                })
+                if len(results) >= top_k:
+                    break
+        return results
+
+    def get_stats(self) -> Dict[str, Any]:
+        return {
+            "total_vectors": int(self.index.ntotal),
+            "total_chunks": len(self.chunks),
+            "dimension": self.dimension,
+        }