mcp-agentic-pipelines 1.0.1

package/vendors/precis/backend/core/hashing.py

@@ -0,0 +1,22 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import hashlib
+from typing import Tuple
+
+
+def make_query_hash(query_tokens: Tuple[str, ...]) -> str:
+    """SHA-256 hash of pipe-joined query tokens. Deterministic cache key for MemoryAgent."""
+    joined = "|".join(query_tokens)
+    return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:16]
+
+
+def make_document_hash(source_doc: str, source_page: int, source_position: int) -> str:
+    """Unique hash for a document location (file + page + position). Used for citation deduplication."""
+    combined = f"{source_doc}|{source_page}|{source_position}"
+    return hashlib.sha256(combined.encode("utf-8")).hexdigest()[:16]
+
+
+def make_session_key(session_id: str) -> str:
+    """Normalized session key. Strips whitespace, lowercases, hashes."""
+    normalized = session_id.strip().lower()
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]

package/vendors/precis/backend/core/metrics.py

@@ -0,0 +1,77 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import math
+import numpy as np
+from typing import List, Dict, Any
+
+
+def compute_relevancy(query_tokens: List[str], chunk_tokens_list: List[List[str]],
+                      pmi_scorer=None) -> float:
+    """Average Jaccard similarity. Uses PMI scorer when available."""
+    if not chunk_tokens_list:
+        return 0.0
+    scores = []
+    query_set = set(query_tokens)
+    for chunk_tokens in chunk_tokens_list:
+        chunk_set = set(chunk_tokens)
+        intersection = query_set & chunk_set
+        union = query_set | chunk_set
+        jaccard = len(intersection) / len(union) if union else 0.0
+        scores.append(jaccard)
+    return sum(scores) / len(scores)
+
+
+def compute_trustworthiness(source_metadata: List[Dict[str, Any]]) -> float:
+    """Score source reliability from metadata: type, recency, cross-referencing."""
+    if not source_metadata:
+        return 0.5
+    scores = []
+    for meta in source_metadata:
+        s = 0.5
+        if meta.get("is_title"):
+            s += 0.2
+        if meta.get("font_size", 0) > 14:
+            s += 0.1
+        if meta.get("token_type") == "contextual":
+            s += 0.15
+        scores.append(min(s, 1.0))
+    return sum(scores) / len(scores)
+
+
+def compute_hallucination_rate(generated_claims: List[str],
+                                source_evidence: List[Dict[str, Any]]) -> float:
+    """Proportion of claims not supported by source evidence."""
+    if not generated_claims:
+        return 0.0
+    supported = 0
+    for claim in generated_claims:
+        claim_lower = claim.lower()
+        for evidence in source_evidence:
+            text = evidence.get("text", "").lower()
+            if any(word in text for word in claim_lower.split() if len(word) > 3):
+                supported += 1
+                break
+    return 1.0 - (supported / len(generated_claims))
+
+
+def compute_r_squared(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """Coefficient of determination. 1.0 = perfect prediction."""
+    ss_res = float(np.sum((y_true - y_pred) ** 2))
+    ss_tot = float(np.sum((y_true - np.mean(y_true)) ** 2))
+    if ss_tot == 0:
+        return 1.0 if ss_res == 0 else 0.0
+    return 1.0 - (ss_res / ss_tot)
+
+
+def compute_hellinger_distance(p_counts: Dict[str, int],
+                                q_counts: Dict[str, int]) -> float:
+    """Hellinger distance between two categorical distributions. 0 = identical."""
+    all_keys = set(p_counts) | set(q_counts)
+    p_total = max(1, sum(p_counts.values()))
+    q_total = max(1, sum(q_counts.values()))
+    sum_sq = 0.0
+    for k in all_keys:
+        p = p_counts.get(k, 0) / p_total
+        q = q_counts.get(k, 0) / q_total
+        sum_sq += (math.sqrt(p) - math.sqrt(q)) ** 2
+    return math.sqrt(0.5 * sum_sq)

package/vendors/precis/backend/core/multitoken.py

@@ -0,0 +1,166 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT
+
+Optimised: pre-stems all words once (O(N) stemming) then slides an n-gram window
+over the pre-stemmed tokens — a ~12× speedup vs. stemming each n-gram independently.
+Element-level extraction can run in parallel via index_document_async().
+"""
+
+import asyncio
+from typing import List, Tuple
+
+from backend.agents.exact_hash_retriever import MultiToken, NestedHashIndex
+from backend.core.stemming import PrecisStemmer
+
+
+class MultiTokenExtractor:
+    """Extracts variable-length multi-tokens (n-grams of stemmed words) from document text.
+
+    Parameters
+    ----------
+    max_token_length : int
+        Maximum n-gram length (inclusive).
+    min_token_length : int
+        Minimum n-gram length (inclusive).
+    """
+
+    def __init__(self, max_token_length: int = 7, min_token_length: int = 2) -> None:
+        self.max_token_length = max_token_length
+        self.min_token_length = min_token_length
+        self.stemmer = PrecisStemmer()
+
+    # ── Core extraction (pre-stemmed, single-element) ──────────────
+
+    def extract(self, text: str, source_doc: str, source_page: int,
+                font_size: float = 12.0, is_title: bool = False,
+                is_header: bool = False,
+                global_position: int = 0) -> List[MultiToken]:
+        """Extract all valid multi-tokens from a text segment.
+
+        Optimisation: words are stemmed ONCE, then n-grams are formed from
+        the pre-stemmed list.  This is O(N) stemming instead of O(N×M).
+
+        global_position is the LINE number in the full document — used by
+        get_context() to retrieve surrounding text from the correct location.
+        """
+        words = text.strip().split()
+        if len(words) < self.min_token_length:
+            return []
+
+        # ── Pre-stem all words once ────────────────────────────
+        stemmed_words = self.stemmer.stem_tokens(words)
+        if len(stemmed_words) < self.min_token_length:
+            return []
+
+        token_type = "contextual" if (is_title or is_header) else "standard"
+        multitokens: List[MultiToken] = []
+
+        n = len(stemmed_words)
+        for start in range(n):
+            max_len = min(self.max_token_length, n - start)
+            for length in range(self.min_token_length, max_len + 1):
+                stemmed = tuple(stemmed_words[start:start + length])
+                # All tokens already stemmed & filtered; just check length
+                if len(stemmed) >= self.min_token_length:
+                    mt = MultiToken(
+                        tokens=stemmed,
+                        token_type=token_type,
+                        source_doc=source_doc,
+                        source_page=source_page,
+                        source_position=global_position,  # ← LINE number, not word index
+                        font_size=font_size,
+                        is_title=is_title,
+                        is_header=is_header,
+                        metadata={"original_words": words[start:start + length]},
+                    )
+                    multitokens.append(mt)
+        return multitokens
+
+    # ── Document indexing (sequential) ────────────────────────────
+
+    def index_document(self, doc_path: str, parsed_content: List[dict],
+                       index: NestedHashIndex) -> int:
+        """Parse a full document and index all multi-tokens. Returns count indexed."""
+        total = 0
+        global_line = 0  # track actual line number in the document
+        for page in parsed_content:
+            page_num = page.get("page_number", 0)
+            for element in page.get("elements", []):
+                text = element.get("text", "")
+                mts = self.extract(
+                    text, doc_path, page_num,
+                    font_size=element.get("font_size", 12.0),
+                    is_title=element.get("is_title", False),
+                    is_header=element.get("is_header", False),
+                    global_position=global_line,  # ← pass line number, not word index
+                )
+                for mt in mts:
+                    index.insert(mt)
+                total += len(mts)
+                global_line += 1
+        return total
+
+    # ── Document indexing (async / parallel pages) ──────────────
+
+    async def index_document_async(self, doc_path: str,
+                                    parsed_content: List[dict],
+                                    index: NestedHashIndex,
+                                    max_tasks: int = 32) -> int:
+        """Like index_document(), but processes **pages** concurrently.
+
+        Each page's full set of elements is extracted in a single thread —
+        this avoids the thread-pool explosion that would result from
+        launching one task per element (thousands of tasks for a large doc).
+
+        Parameters
+        ----------
+        max_tasks : int
+            Upper bound on concurrent page-extraction tasks.  Prevents
+            thread-pool exhaustion on documents with many short pages.
+        """
+        # ── Build one task per PAGE (not per element!) ────────
+        # Each task extracts ALL elements on that page sequentially.
+        # Tracks global line numbers so get_context() retrieves from
+        # the correct position in the document.
+        _line_counter = [0]  # mutable counter shared across pages
+
+        def extract_page(page: dict) -> List[MultiToken]:
+            page_num = page.get("page_number", 0)
+            all_mts: List[MultiToken] = []
+            for element in page.get("elements", []):
+                text = element.get("text", "")
+                if not text.strip():
+                    _line_counter[0] += 1
+                    continue
+                mts = self.extract(
+                    text, doc_path, page_num,
+                    font_size=element.get("font_size", 12.0),
+                    is_title=element.get("is_title", False),
+                    is_header=element.get("is_header", False),
+                    global_position=_line_counter[0],
+                )
+                all_mts.extend(mts)
+                _line_counter[0] += 1
+            return all_mts
+
+        pages = [p for p in parsed_content if p.get("elements")]
+        if not pages:
+            return 0
+
+        # ── Throttle: at most max_tasks in flight at once ─────
+        sem = asyncio.Semaphore(max_tasks)
+
+        async def bounded_extract(page: dict) -> List[MultiToken]:
+            async with sem:
+                return await asyncio.to_thread(extract_page, page)
+
+        results: List[List[MultiToken]] = await asyncio.gather(
+            *(bounded_extract(p) for p in pages)
+        )
+
+        # ── Batch-insert all MultiTokens ──────────────────────
+        total = 0
+        for mts in results:
+            for mt in mts:
+                index.insert(mt)
+            total += len(mts)
+        return total

package/vendors/precis/backend/core/pmi.py

@@ -0,0 +1,54 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import math
+from typing import Dict, List, Tuple
+from collections import defaultdict
+
+
+class PMIScorer:
+    """Computes PMI-based relevance scores using incremental corpus token statistics."""
+
+    def __init__(self) -> None:
+        self.token_count: Dict[str, int] = defaultdict(int)
+        self.pair_count: Dict[Tuple[str, str], int] = defaultdict(int)
+        self.total_tokens: int = 0
+
+    def ingest_tokens(self, tokens: List[str]) -> None:
+        """Update corpus statistics with a sequence of stemmed tokens from one multi-token."""
+        for i, t in enumerate(tokens):
+            self.token_count[t] += 1
+            self.total_tokens += 1
+            for j in range(i + 1, len(tokens)):
+                pair = (t, tokens[j]) if t <= tokens[j] else (tokens[j], t)
+                self.pair_count[pair] += 1
+
+    def score(self, query_tokens: List[str], chunk_tokens: List[str]) -> float:
+        """Average PMI across all (query, chunk) token pairs. Higher = more relevant."""
+        if not query_tokens or not chunk_tokens:
+            return 0.0
+        pmi_values: List[float] = []
+        for q in query_tokens:
+            for c in chunk_tokens:
+                pair = (q, c) if q <= c else (c, q)
+                joint = self.pair_count.get(pair, 0)
+                if joint == 0:
+                    continue
+                p_q = self.token_count.get(q, 0) / max(1, self.total_tokens)
+                p_c = self.token_count.get(c, 0) / max(1, self.total_tokens)
+                p_joint = joint / max(1, self.total_tokens)
+                if p_q > 0 and p_c > 0:
+                    pmi = math.log2(p_joint / (p_q * p_c))
+                    pmi_values.append(pmi)
+        return sum(pmi_values) / len(pmi_values) if pmi_values else 0.0
+
+    def normalize_score(self, raw_score: float, max_observed: float = 10.0) -> float:
+        """Clip and normalize to [0, 1]."""
+        return min(max(raw_score, 0.0), max_observed) / max_observed
+
+
+# Global singleton — shared across index, search, and upload
+_pmi_scorer: PMIScorer = PMIScorer()
+
+
+def get_pmi_scorer() -> PMIScorer:
+    return _pmi_scorer

package/vendors/precis/backend/core/stemming.py

@@ -0,0 +1,74 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+from typing import List, Set
+from nltk.stem import PorterStemmer
+
+
+# ── Lazy-loaded NLTK stopwords, minus content-bearing words ────
+# Words like "other", "more", "same" are stopwords in general NLP
+# but ARE content in document section titles and technical text.
+_CONTENT_WORDS_TO_KEEP: Set[str] = {
+    "other", "more", "most", "some", "such", "only", "own", "same",
+    "very", "just", "both", "few", "each", "every", "any", "all",
+    "no", "not", "nor",  # negation is semantically important
+}
+
+def _load_nltk_stopwords() -> Set[str]:
+    """Return NLTK stopwords minus content-bearing words."""
+    try:
+        from nltk.corpus import stopwords
+        return set(stopwords.words("english")) - _CONTENT_WORDS_TO_KEEP
+    except (ImportError, LookupError, OSError):
+        pass
+    # Minimal fallback
+    return {"i", "me", "my", "we", "our", "you", "your", "he", "him",
+            "his", "she", "her", "it", "its", "they", "them", "their",
+            "this", "that", "these", "those", "am", "is", "are", "was",
+            "were", "be", "been", "being", "have", "has", "had", "do",
+            "does", "did", "a", "an", "the", "and", "but", "if", "or",
+            "because", "as", "of", "at", "by", "for", "with", "about",
+            "between", "into", "through", "during", "before", "after",
+            "to", "from", "in", "on", "off", "over", "under",
+            "can", "will", "should", "now", "don", "doesn", "didn",
+            "won", "wouldn", "couldn", "shouldn", "isn", "aren"}
+
+
+class PrecisStemmer:
+    """Combines Porter stemming with domain-specific rules. Acronyms (KYC, AML, ESG) are preserved as-is."""
+
+    _KNOWN_ACRONYMS: Set[str] = {"kyc", "aml", "esg", "gaap", "ifrs", "sec", "fdic", "finra",
+                                  "soc", "iso", "hipaa", "gdpr", "ccpa", "sox", "cfpb", "finra"}
+
+    # ── Precis-specific additions (query-structure words) ─────
+    _PRECIS_STOPWORDS: Set[str] = {
+        "summarize", "summary", "summarise", "explain", "describe",
+        "list", "identify", "compare", "contrast", "discuss", "analyze",
+        "key", "finding", "findings", "detail", "details", "overview",
+        "section", "chapter", "paragraph", "figure", "table", "page",
+        "get", "make", "made", "see", "show", "shown", "find", "found",
+    }
+
+    # ── Merged set: NLTK standard + Precis custom ─────────────
+    _STOPWORDS: Set[str] = _load_nltk_stopwords() | _PRECIS_STOPWORDS
+
+    def __init__(self) -> None:
+        self._stemmer = PorterStemmer()
+
+    def stem(self, word: str) -> str:
+        """Stem a single word. Preserves known acronyms. Filters stopwords to empty string."""
+        word_lower = word.strip().lower()
+        if not word_lower:
+            return ""
+        if word_lower in self._KNOWN_ACRONYMS:
+            return word_lower
+        if word_lower in self._STOPWORDS:
+            return ""
+        return self._stemmer.stem(word_lower)
+
+    def stem_tokens(self, tokens: List[str]) -> List[str]:
+        """Stem a list of tokens, filtering out stopwords and empty results."""
+        return [s for token in tokens if (s := self.stem(token))]
+
+    def add_acronym(self, acronym: str) -> None:
+        """Register a domain-specific acronym to preserve during stemming."""
+        self._KNOWN_ACRONYMS.add(acronym.strip().lower())

package/vendors/precis/backend/core/tracing.py

@@ -0,0 +1,150 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+import json
+import time
+import uuid
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Callable, Dict, List, Optional, Any
+
+
+class TraceEventType(str, Enum):
+    QUERY_STARTED = "query.started"
+    QUERY_COMPLETED = "query.completed"
+    QUERY_FAILED = "query.failed"
+    PLAN_CREATED = "plan.created"
+    AGENT_STARTED = "agent.started"
+    AGENT_COMPLETED = "agent.completed"
+    AGENT_FAILED = "agent.failed"
+    DECISION_SEARCH_TYPE = "decision.search_type"
+    DECISION_THRESHOLD = "decision.threshold"
+    DECISION_PREDICTION = "decision.prediction"
+    LLM_CALL_STARTED = "llm.call_started"
+    LLM_CALL_COMPLETED = "llm.call_completed"
+    LLM_TOKEN_USAGE = "llm.token_usage"
+    EVALUATION_COMPLETED = "evaluation.completed"
+    GUARDRAIL_ACTION = "guardrail.action"
+    RESULT_FOUND = "result.found"
+    ANOMALY_FLAGGED = "anomaly.flagged"
+    CITATION_ADDED = "citation.added"
+
+
+@dataclass
+class TraceEvent:
+    event_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
+    event_type: TraceEventType = TraceEventType.QUERY_STARTED
+    agent_name: str = ""
+    span_id: Optional[str] = None
+    message: str = ""
+    data: Dict[str, Any] = field(default_factory=dict)
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    duration_ms: float = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"event_id": self.event_id, "event_type": self.event_type.value,
+                "agent_name": self.agent_name, "span_id": self.span_id,
+                "message": self.message, "data": self.data,
+                "timestamp": self.timestamp, "duration_ms": self.duration_ms}
+
+
+@dataclass
+class TraceSpan:
+    span_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
+    parent_span_id: Optional[str] = None
+    agent_name: str = ""
+    operation: str = ""
+    start_time: float = 0.0
+    end_time: float = 0.0
+    events: List[TraceEvent] = field(default_factory=list)
+    child_spans: List["TraceSpan"] = field(default_factory=list)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def duration_ms(self) -> float:
+        return (self.end_time - self.start_time) * 1000 if self.end_time and self.start_time else 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"span_id": self.span_id, "parent_span_id": self.parent_span_id,
+                "agent_name": self.agent_name, "operation": self.operation,
+                "duration_ms": self.duration_ms,
+                "events": [e.to_dict() for e in self.events],
+                "child_spans": [c.to_dict() for c in self.child_spans],
+                "metadata": self.metadata}
+
+
+class TraceCollector:
+    """Collects trace events during query execution. Streams to WebSocket, persists to SQLite."""
+
+    def __init__(self, query_id: str, session_id: Optional[str] = None) -> None:
+        self.trace_id = str(uuid.uuid4())
+        self.query_id = query_id
+        self.session_id = session_id
+        self._span_stack: List[TraceSpan] = []
+        self._root_spans: List[TraceSpan] = []
+        self._events: List[TraceEvent] = []
+        self._query_start_time = time.time()
+        self._status = "running"
+        self._stream_callback: Optional[Callable[[Dict[str, Any]], None]] = None
+
+    def span_start(self, agent_name: str, operation: str, metadata: dict = None) -> str:
+        span = TraceSpan(agent_name=agent_name, operation=operation,
+                         start_time=time.time(), metadata=metadata or {})
+        if self._span_stack:
+            span.parent_span_id = self._span_stack[-1].span_id
+            self._span_stack[-1].child_spans.append(span)
+        else:
+            self._root_spans.append(span)
+        self._span_stack.append(span)
+        return span.span_id
+
+    def span_end(self, metadata: dict = None) -> str:
+        if not self._span_stack:
+            return ""
+        span = self._span_stack.pop()
+        span.end_time = time.time()
+        if metadata:
+            span.metadata.update(metadata)
+        return span.span_id
+
+    def event(self, event_type: TraceEventType, agent_name: str = "",
+              message: str = "", data: Dict[str, Any] = None,
+              duration_ms: float = 0.0) -> TraceEvent:
+        evt = TraceEvent(event_type=event_type, agent_name=agent_name,
+                         message=message, data=data or {}, duration_ms=duration_ms)
+        if self._span_stack:
+            evt.span_id = self._span_stack[-1].span_id
+            self._span_stack[-1].events.append(evt)
+        self._events.append(evt)
+        if self._stream_callback:
+            self._stream_callback(evt.to_dict())
+        return evt
+
+    def set_stream_callback(self, callback: Callable[[Dict[str, Any]], None]) -> None:
+        self._stream_callback = callback
+
+    def complete(self, status: str = "success") -> None:
+        while self._span_stack:
+            self.span_end()
+        self._status = status
+        self.event(TraceEventType.QUERY_COMPLETED if status == "success" else TraceEventType.QUERY_FAILED,
+                   message=f"Query {status}", data={"duration_ms": self.get_total_duration_ms()})
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"trace_id": self.trace_id, "query_id": self.query_id,
+                "session_id": self.session_id, "status": self._status,
+                "duration_ms": self.get_total_duration_ms(),
+                "agent_count": len(self._root_spans),
+                "event_count": len(self._events),
+                "root_spans": [s.to_dict() for s in self._root_spans],
+                "events": [e.to_dict() for e in self._events],
+                "created_at": datetime.now(timezone.utc).isoformat()}
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), default=str)
+
+    def get_event_count(self) -> int:
+        return len(self._events)
+
+    def get_total_duration_ms(self) -> float:
+        return (time.time() - self._query_start_time) * 1000

package/vendors/precis/backend/data/__init__.py

@@ -0,0 +1,3 @@
+# =============================================================================
+# © JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT
+# =============================================================================

package/vendors/precis/backend/data/chunker.py

@@ -0,0 +1,57 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+
+@dataclass
+class Chunk:
+    chunk_id: str
+    text: str
+    source_doc: str
+    source_page: int
+    start_position: int
+    end_position: int
+    level: str = "paragraph"
+    context_type: str = "body"
+    font_size: float = 12.0
+    is_title: bool = False
+    is_header: bool = False
+
+
+class HierarchicalChunker:
+    """Three-level overlapping chunking: section → paragraph → sentence."""
+
+    def __init__(self, overlap_ratio: float = 0.2) -> None:
+        self.overlap_ratio = overlap_ratio
+
+    def chunk_page(self, page_number: int, elements: List[Dict[str, Any]],
+                   source_doc: str) -> List[Chunk]:
+        chunks = []
+        full_text = " ".join(e.get("text", "") for e in elements)
+        if not full_text.strip():
+            return chunks
+        paragraphs = [p.strip() for p in full_text.split("\n\n") if p.strip()]
+        pos = 0
+        for i, para in enumerate(paragraphs):
+            chunks.append(Chunk(
+                chunk_id=f"{source_doc}_p{page_number}_para{i}",
+                text=para, source_doc=source_doc, source_page=page_number,
+                start_position=pos, end_position=pos + len(para), level="paragraph"))
+            pos += len(para) + 2
+            # Sentence-level chunks
+            sentences = [s.strip() + "." for s in para.replace("\n", " ").split(". ") if s.strip()]
+            for j, sent in enumerate(sentences):
+                if len(sent) > 10:
+                    chunks.append(Chunk(
+                        chunk_id=f"{source_doc}_p{page_number}_para{i}_sent{j}",
+                        text=sent, source_doc=source_doc, source_page=page_number,
+                        start_position=pos, end_position=pos + len(sent), level="sentence"))
+        return chunks
+
+    def chunk_document(self, parsed_pages: List[Dict[str, Any]], source_doc: str) -> List[Chunk]:
+        all_chunks = []
+        for page in parsed_pages:
+            all_chunks.extend(self.chunk_page(page.get("page_number", 0),
+                                              page.get("elements", []), source_doc))
+        return all_chunks

package/vendors/precis/backend/data/pdf_parser.py

@@ -0,0 +1,42 @@
+"""© JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT"""
+
+from typing import Any, Dict, List
+
+
+class PDFParser:
+    """Extracts text with font metadata for contextual multi-token scoring."""
+
+    def parse(self, file_path: str) -> List[Dict[str, Any]]:
+        try:
+            import fitz  # pymupdf
+            pages = []
+            doc = fitz.open(file_path)
+            for page_num, page in enumerate(doc):
+                blocks = page.get_text("dict")["blocks"]
+                elements = []
+                for block in blocks:
+                    if "lines" not in block:
+                        continue
+                    for line in block["lines"]:
+                        for span in line["spans"]:
+                            font_size = span.get("size", 12.0)
+                            text = span.get("text", "").strip()
+                            if text:
+                                elements.append({"text": text, "font_size": font_size,
+                                                 "is_title": font_size > 16,
+                                                 "is_header": 14 <= font_size <= 16})
+                pages.append({"page_number": page_num + 1, "elements": elements})
+            doc.close()
+            return pages
+        except ImportError:
+            text = open(file_path, "r", encoding="utf-8", errors="ignore").read()
+            return [{"page_number": 1, "elements": [{"text": text, "font_size": 12.0,
+                      "is_title": False, "is_header": False}]}]
+
+    def get_sections(self, parsed_content: List[Dict]) -> List[Dict]:
+        sections = []
+        for page in parsed_content:
+            for el in page.get("elements", []):
+                sections.append({"text": el["text"], "page": page["page_number"],
+                                 "is_title": el["is_title"], "is_header": el["is_header"]})
+        return sections

package/vendors/precis/backend/db/__init__.py

@@ -0,0 +1,3 @@
+# =============================================================================
+# © JINAN KORDAB — 2026 AI HYBRID AGENTIC RETRIEVAL-AUGMENTED GENERATION RAG PIPELINE - PERSONAL PROJECT
+# =============================================================================