igel-qe-core 1.0.0

package/knowledge_base/parsers/pdf_parser.py

@@ -0,0 +1,178 @@
+"""
+PDF Parser — three-tier strategy:
+  Tier 1: If a pre-converted markdown file exists in markdown_output/, use it.
+           (Higher quality — processed by marker-pdf + Azure OpenAI LLM)
+  Tier 2: Direct pdfplumber extraction.
+  Tier 3: PyMuPDF (fitz) fallback — handles PDFs pdfplumber can't (encrypted
+           XRefs, exotic fonts, malformed cross-reference tables).
+"""
+from __future__ import annotations
+import logging
+from pathlib import Path
+from knowledge_base.parsers.markdown_parser import Section, parse_markdown_file, find_markdown_for_pdf
+
+logger = logging.getLogger(__name__)
+
+
+def parse_pdf(pdf_path: Path, markdown_output_dir: Path) -> tuple[list[Section], str]:
+    """
+    Returns (sections, parse_method) where parse_method is one of:
+    'markdown', 'pdfplumber', 'pymupdf'.
+    """
+    # Tier 1: use pre-converted markdown
+    md_path = find_markdown_for_pdf(pdf_path.name, markdown_output_dir)
+    if md_path:
+        sections = parse_markdown_file(md_path)
+        if sections:
+            return sections, "markdown"
+
+    # Tier 2: pdfplumber
+    try:
+        sections = _parse_with_pdfplumber(pdf_path)
+        if sections and any(s.content.strip() for s in sections):
+            return sections, "pdfplumber"
+        logger.info("pdfplumber returned empty sections for %s — trying PyMuPDF", pdf_path.name)
+    except Exception as e:
+        logger.warning("pdfplumber failed on %s: %s — falling back to PyMuPDF", pdf_path.name, e)
+
+    # Tier 3: PyMuPDF
+    sections = _parse_with_pymupdf(pdf_path)
+    return sections, "pymupdf"
+
+
+def _parse_with_pymupdf(pdf_path: Path) -> list[Section]:
+    import fitz
+    sections: list[Section] = []
+    try:
+        doc = fitz.open(str(pdf_path))
+    except Exception as e:
+        logger.error("PyMuPDF could not open %s: %s", pdf_path.name, e)
+        return [Section(title=pdf_path.stem, level=0, content="", page_hint=1)]
+
+    try:
+        for page_idx in range(len(doc)):
+            page = doc[page_idx]
+            text = page.get_text("text") or ""
+            text = text.strip()
+            if not text:
+                continue
+            sections.append(Section(
+                title=pdf_path.stem,
+                level=0,
+                content=text,
+                page_hint=page_idx + 1,
+            ))
+    finally:
+        doc.close()
+
+    return sections if sections else [Section(title=pdf_path.stem, level=0, content="", page_hint=1)]
+
+
+def _parse_with_pdfplumber(pdf_path: Path) -> list[Section]:
+    import pdfplumber
+    import re
+
+    all_sections: list[Section] = []
+    current_title = pdf_path.stem
+    current_level = 0
+    current_lines: list[str] = []
+    current_page = 1
+    full_text_lines: list[str] = []
+
+    with pdfplumber.open(str(pdf_path)) as pdf:
+        for page_num, page in enumerate(pdf.pages, start=1):
+            # Extract tables first — preserve as markdown
+            tables = page.extract_tables()
+            table_md_blocks = [_table_to_markdown(t) for t in tables if t]
+
+            text = page.extract_text(x_tolerance=3, y_tolerance=3) or ""
+            lines = text.split("\n")
+
+            # Strip header/footer (first and last line are usually page number / product name)
+            if len(lines) > 3:
+                lines = lines[1:-1]
+
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    full_text_lines.append("")
+                    continue
+
+                # Heading detection by ALL_CAPS short lines or common IGEL heading patterns
+                if _is_heading(line):
+                    # Flush current section
+                    content = "\n".join(current_lines).strip()
+                    if content:
+                        all_sections.append(Section(
+                            title=current_title,
+                            level=current_level,
+                            content=content,
+                            page_hint=current_page,
+                        ))
+                    current_title = line.rstrip(":")
+                    current_level = _estimate_heading_level(line)
+                    current_lines = []
+                    current_page = page_num
+                else:
+                    current_lines.append(line)
+                    full_text_lines.append(line)
+
+            # Append tables as a block at end of page content
+            for table_md in table_md_blocks:
+                current_lines.append(table_md)
+
+    # Flush final section
+    content = "\n".join(current_lines).strip()
+    if content:
+        all_sections.append(Section(
+            title=current_title,
+            level=current_level,
+            content=content,
+            page_hint=current_page,
+        ))
+
+    # If no structure detected, treat whole doc as one section
+    if len(all_sections) <= 1:
+        all_text = "\n".join(full_text_lines).strip()
+        return [Section(title=pdf_path.stem, level=0, content=all_text, page_hint=1)]
+
+    return all_sections
+
+
+def _is_heading(line: str) -> bool:
+    import re
+    if len(line) > 120:
+        return False
+    # Short ALL CAPS lines
+    if line.isupper() and 3 < len(line) < 80:
+        return True
+    # Numbered heading: "1.2 Title" or "3. Title"
+    if re.match(r"^\d+(\.\d+)*\.?\s+[A-Z]", line):
+        return True
+    # Lines ending with colon that look like section titles
+    if line.endswith(":") and len(line) < 60 and line[0].isupper():
+        return True
+    return False
+
+
+def _estimate_heading_level(line: str) -> int:
+    import re
+    m = re.match(r"^(\d+)(\.\d+)*", line)
+    if m:
+        depth = line[:line.index(" ")].count(".")
+        return min(depth + 1, 3)
+    return 2
+
+
+def _table_to_markdown(table: list) -> str:
+    if not table or not table[0]:
+        return ""
+    rows = [[str(cell or "").strip() for cell in row] for row in table]
+    # Find max columns
+    cols = max(len(r) for r in rows)
+    rows = [r + [""] * (cols - len(r)) for r in rows]
+
+    header = "| " + " | ".join(rows[0]) + " |"
+    divider = "| " + " | ".join(["---"] * cols) + " |"
+    body = ["| " + " | ".join(r) + " |" for r in rows[1:]]
+    return "\n".join([header, divider] + body)

package/knowledge_base/retrieval/__init__.py

@@ -0,0 +1 @@
+"""Retrieval, reranking, and context fusion."""

package/knowledge_base/retrieval/context_fusion.py

@@ -0,0 +1,264 @@
+"""Fuse RAG hits, KG text, and image snippets under a dynamic token budget."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from knowledge_base.config import cfg
+from knowledge_base.core.precedence import (
+    apply_precedence_filter,
+    detect_precedence_conflicts,
+    infer_source_type,
+)
+
+if TYPE_CHECKING:
+    from knowledge_base.retrieval.retriever import SearchResult
+
+logger = logging.getLogger(__name__)
+
+_CHARS_PER_TOKEN = 3.8
+
+_EXCLUDE_SOURCE_MAP = {
+    "jira": "jira_tc",
+    "defects": "execution_history",
+    "defect": "execution_history",
+    "confluence": "confluence",
+    "screenshots": "screenshot",
+    "execution": "execution_history",
+}
+
+
+def _chunk_content(chunk) -> str:
+    """Extract searchable text from a chunk object."""
+    for attr in ("parent_content", "content", "child_content"):
+        val = getattr(chunk, attr, None)
+        if val:
+            return str(val)
+    return ""
+
+
+def _chunk_metadata(chunk) -> dict:
+    meta = getattr(chunk, "metadata", None)
+    if isinstance(meta, dict) and meta:
+        return meta
+    return {
+        "file_path": getattr(chunk, "file_name", "") or "",
+        "product": getattr(chunk, "product", "") or "",
+        "section_title": getattr(chunk, "section_title", "") or "",
+    }
+
+
+def _attach_chunk_helpers(chunk):
+    """Ensure precedence helpers can read content/metadata from SearchResult."""
+    if not getattr(chunk, "metadata", None):
+        chunk.metadata = _chunk_metadata(chunk)
+    if not getattr(chunk, "content", None):
+        chunk.content = _chunk_content(chunk)
+    return chunk
+
+
+def filter_excluded_sources(results: list) -> list:
+    """Remove chunks from excluded source types (RETRIEVAL_EXCLUDE_SOURCES)."""
+    excluded = cfg.RETRIEVAL_EXCLUDE_SOURCES
+    if not excluded or not results:
+        return results
+
+    excluded_types = set()
+    for label in excluded:
+        mapped = _EXCLUDE_SOURCE_MAP.get(label.lower(), label.lower())
+        excluded_types.add(mapped)
+        if "defect" in label.lower():
+            excluded_types.add("execution_history")
+
+    kept: list = []
+    removed = 0
+    for chunk in results:
+        chunk = _attach_chunk_helpers(chunk)
+        source = infer_source_type(_chunk_metadata(chunk))
+        file_path = (_chunk_metadata(chunk).get("file_path") or "").lower()
+        if source.name in excluded_types:
+            removed += 1
+            continue
+        if "defect" in excluded and "defect" in file_path:
+            removed += 1
+            continue
+        kept.append(chunk)
+
+    if removed:
+        logger.info("context_fusion: excluded %d chunks by source filter %s", removed, excluded)
+    return kept
+
+
+def apply_precedence_ranking(
+    results: list,
+    *,
+    max_chunks: int | None = None,
+) -> tuple[list, dict, list[dict]]:
+    """
+    Rank chunks by source precedence, detect conflicts.
+
+    Returns:
+        (filtered_chunks, attribution_metadata, conflicts)
+    """
+    if not results:
+        return [], {"total_chunks": 0, "source_breakdown": {}}, []
+
+    prepared = [_attach_chunk_helpers(r) for r in results]
+    top_k = max_chunks or len(prepared)
+    filtered, attribution = apply_precedence_filter(prepared, top_k=top_k)
+    conflicts = detect_precedence_conflicts(filtered)
+    if conflicts:
+        logger.info("context_fusion: detected %d precedence conflicts", len(conflicts))
+    return filtered, attribution, conflicts
+
+
+def fuse_retrieval_context(
+    results: list,
+    kg_text: str,
+    image_hits: list | None,
+    *,
+    token_budget: int,
+    reserve_prompt: int,
+    kg_ratio: float,
+    image_ratio: float,
+    min_score: float | None,
+    max_chunks: int,
+    web_block: str = "",
+    web_ratio: float | None = None,
+) -> tuple[list, str, str, str, int, dict, list[dict]]:
+    """
+    Full fusion pipeline: source filter → precedence → dynamic budget.
+
+    Returns:
+        (selected_results, kg_trimmed, image_block, web_trimmed, rag_tokens, attribution, conflicts)
+    """
+    filtered = filter_excluded_sources(results)
+    ranked, attribution, conflicts = apply_precedence_ranking(
+        filtered,
+        max_chunks=max(max_chunks, len(filtered)),
+    )
+    selected, kg_out, img_block, web_out, rag_used = apply_dynamic_budget(
+        ranked,
+        kg_text,
+        image_hits,
+        token_budget=token_budget,
+        reserve_prompt=reserve_prompt,
+        kg_ratio=kg_ratio,
+        image_ratio=image_ratio,
+        min_score=min_score,
+        max_chunks=max_chunks,
+        web_block=web_block,
+        web_ratio=web_ratio,
+    )
+    return selected, kg_out, img_block, web_out, rag_used, attribution, conflicts
+
+
+def _truncate_words(text: str, max_chars: int) -> str:
+    if len(text) <= max_chars:
+        return text
+    truncated = text[:max_chars]
+    last_space = truncated.rfind(" ")
+    return truncated[:last_space] if last_space > 0 else truncated
+
+
+def _est_tokens(text: str) -> int:
+    return max(1, int(len(text) / _CHARS_PER_TOKEN))
+
+
+def _format_image_snippets(image_hits: list, max_images: int = 4) -> str:
+    lines: list[str] = []
+    for h in image_hits[:max_images]:
+        lines.append(
+            f"[{h.image_type}] ({h.file_name} p.{h.page_number or '?'}) "
+            f"{getattr(h, 'semantic_description', '') or ''}\n"
+            f"Visible text: {(getattr(h, 'verbatim_text', '') or '')[:800]}\n"
+            f"Test hints: {(getattr(h, 'test_relevance', '') or '')[:800]}"
+        )
+    if not lines:
+        return ""
+    return "—— Related documentation images ——\n" + "\n\n".join(lines)
+
+
+def apply_dynamic_budget(
+    results: list,
+    kg_text: str,
+    image_hits: list | None,
+    *,
+    token_budget: int,
+    reserve_prompt: int,
+    kg_ratio: float,
+    image_ratio: float,
+    min_score: float | None,
+    max_chunks: int,
+    web_block: str = "",
+    web_ratio: float | None = None,
+) -> tuple[list, str, str, str, int]:
+    """
+    Select RAG parent sections + trim KG + images + web to fit token_budget.
+
+    Returns:
+        (selected_SearchResult_list, kg_trimmed, image_block, web_trimmed, rag_tokens_used_approx)
+    """
+    available = max(200, token_budget - reserve_prompt)
+    web_ratio_val = web_ratio if web_ratio is not None else float(getattr(cfg, "WEB_BUDGET_RATIO", 0.12))
+    web_cap = int(available * web_ratio_val) if web_block else 0
+    kg_cap = int(available * kg_ratio)
+    img_cap = int(available * image_ratio)
+    rag_cap = max(200, available - kg_cap - img_cap - web_cap)
+
+    filtered: list = []
+    for r in results:
+        score = getattr(r, "rerank_score", None)
+        if score is None:
+            score = float(getattr(r, "rrf_score", 0.0))
+        if min_score is not None and score < min_score:
+            continue
+        filtered.append(r)
+
+    filtered = filtered[: max(1, max_chunks)]
+
+    selected: list = []
+    rag_used = 0
+    for r in filtered:
+        body = getattr(r, "parent_content", "") or ""
+        need = _est_tokens(body)
+        if rag_used + need <= rag_cap:
+            selected.append(r)
+            rag_used += need
+        elif rag_cap - rag_used > 150:
+            remain = int((rag_cap - rag_used) * _CHARS_PER_TOKEN)
+            shortened = _truncate_words(body, max(0, remain))
+            import copy as _copy
+
+            cp = _copy.copy(r)
+            cp.parent_content = shortened + "\n[...truncated by context budget]"
+            selected.append(cp)
+            rag_used += _est_tokens(shortened)
+            break
+        else:
+            break
+
+    kg_out = (kg_text or "").strip()
+    if kg_out and _est_tokens(kg_out) > kg_cap:
+        kg_out = _truncate_words(kg_out, int(kg_cap * _CHARS_PER_TOKEN)) + "\n[...kg truncated]"
+
+    img_block = _format_image_snippets(image_hits or [], max_images=max(1, int(3 + img_cap // 400)))
+    if img_block and _est_tokens(img_block) > img_cap:
+        img_block = _truncate_words(img_block, int(img_cap * _CHARS_PER_TOKEN)) + "\n[...images truncated]"
+
+    web_out = (web_block or "").strip()
+    if web_out and _est_tokens(web_out) > web_cap:
+        web_out = _truncate_words(web_out, int(web_cap * _CHARS_PER_TOKEN)) + "\n[...web truncated]"
+
+    logger.debug(
+        "context_fusion: rag_chunks=%d rag_tok~%d kg_tok~%d img_tok~%d web_tok~%d budget=%d",
+        len(selected),
+        rag_used,
+        _est_tokens(kg_out),
+        _est_tokens(img_block),
+        _est_tokens(web_out),
+        token_budget,
+    )
+
+    return selected, kg_out, img_block, web_out, rag_used