igel-qe-core 1.0.0

package/knowledge_base/parsers/html_parser.py

@@ -0,0 +1,151 @@
+"""Parse HTML pages (kb.igel.com, community sites) into Section objects for chunking."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from html import unescape
+from typing import Optional
+from urllib.parse import urljoin, urlparse
+
+from knowledge_base.parsers.markdown_parser import Section
+
+
+@dataclass
+class ParsedWebPage:
+    url: str
+    title: str
+    sections: list[Section]
+    raw_text: str
+    source_type: str = "kb_web"
+
+
+def _strip_tags(html: str) -> str:
+    text = re.sub(r"(?is)<(script|style|nav|footer|header|aside)[^>]*>.*?</\1>", " ", html)
+    text = re.sub(r"(?is)<br\s*/?>", "\n", text)
+    text = re.sub(r"(?is)</?(p|div|li|h[1-6]|tr|td|th)[^>]*>", "\n", text)
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = unescape(text)
+    text = re.sub(r"[ \t]+\n", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def _extract_title(html: str, fallback: str) -> str:
+    m = re.search(r"(?is)<title[^>]*>(.*?)</title>", html)
+    if m:
+        title = _strip_tags(m.group(1)).strip()
+        if title:
+            return title[:200]
+    for tag in ("h1", "h2"):
+        m = re.search(rf"(?is)<{tag}[^>]*>(.*?)</{tag}>", html)
+        if m:
+            title = _strip_tags(m.group(1)).strip()
+            if title:
+                return title[:200]
+    return fallback
+
+
+def _extract_main_html(html: str) -> str:
+    """Best-effort main content extraction for Confluence / Scroll Sites pages."""
+    for pattern in (
+        r'(?is)<main[^>]*>(.*?)</main>',
+        r'(?is)<article[^>]*>(.*?)</article>',
+        r'(?is)<div[^>]*id="main-content"[^>]*>(.*?)</div>',
+        r'(?is)<div[^>]*class="[^"]*wiki-content[^"]*"[^>]*>(.*?)</div>',
+        r'(?is)<div[^>]*class="[^"]*page-content[^"]*"[^>]*>(.*?)</div>',
+        r'(?is)<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
+    ):
+        m = re.search(pattern, html)
+        if m and len(m.group(1).strip()) > 200:
+            return m.group(1)
+    return html
+
+
+def _split_headings(text: str, page_title: str, url: str) -> list[Section]:
+    lines = text.split("\n")
+    sections: list[Section] = []
+    current_title = page_title
+    current_lines: list[str] = []
+
+    for line in lines:
+        heading = re.match(r"^(#{1,4})\s+(.+)", line.strip())
+        if heading:
+            content = "\n".join(current_lines).strip()
+            if content:
+                sections.append(
+                    Section(
+                        title=current_title,
+                        level=len(heading.group(1)),
+                        content=content,
+                        page_hint=0,
+                        metadata={"canonical_url": url, "source_type": "kb_web"},
+                    )
+                )
+            current_title = heading.group(2).strip()
+            current_lines = []
+        else:
+            current_lines.append(line)
+
+    content = "\n".join(current_lines).strip()
+    if content:
+        sections.append(
+            Section(
+                title=current_title,
+                level=1,
+                content=content,
+                page_hint=0,
+                metadata={"canonical_url": url, "source_type": "kb_web"},
+            )
+        )
+
+    if not sections and text.strip():
+        sections.append(
+            Section(
+                title=page_title,
+                level=1,
+                content=text.strip(),
+                page_hint=0,
+                metadata={"canonical_url": url, "source_type": "kb_web"},
+            )
+        )
+    return sections
+
+
+def parse_html_page(html: str, url: str, *, source_type: str = "kb_web") -> ParsedWebPage:
+    """Convert an HTML page into structured sections."""
+    title = _extract_title(html, fallback=urlparse(url).path.split("/")[-1] or url)
+    main_html = _extract_main_html(html)
+    text = _strip_tags(main_html)
+
+    # Promote obvious headings (ALL CAPS lines or numbered sections)
+    promoted: list[str] = []
+    for line in text.split("\n"):
+        stripped = line.strip()
+        if re.match(r"^[A-Z][A-Z0-9\s\-/&]{8,}$", stripped):
+            promoted.append(f"## {stripped.title()}")
+        elif re.match(r"^\d+\.\s+[A-Z]", stripped):
+            promoted.append(stripped)
+        else:
+            promoted.append(line)
+    text = "\n".join(promoted)
+
+    sections = _split_headings(text, title, url)
+    for sec in sections:
+        sec.metadata["canonical_url"] = url
+        sec.metadata["source_type"] = source_type
+
+    return ParsedWebPage(url=url, title=title, sections=sections, raw_text=text, source_type=source_type)
+
+
+def resolve_links(html: str, base_url: str) -> list[str]:
+    """Extract same-domain links from HTML for crawl expansion."""
+    base_host = urlparse(base_url).netloc
+    links: list[str] = []
+    for href in re.findall(r'(?is)href=["\']([^"\']+)["\']', html):
+        if href.startswith("#") or href.startswith("mailto:"):
+            continue
+        full = urljoin(base_url, href)
+        if urlparse(full).netloc == base_host and full not in links:
+            links.append(full.split("#")[0])
+    return links[:50]

package/knowledge_base/parsers/image_extractor.py

@@ -0,0 +1,370 @@
+"""
+Image extraction from PDFs, DOCX, and DOC files.
+
+Each extracted image carries enough context to be linked back to its source
+document and the nearest text section:
+  - bytes (raw, byte-perfect)
+  - format / dimensions
+  - page_number (PDF) or paragraph_index (DOCX)
+  - bbox on page (PDF only)
+  - surrounding_text (paragraphs above + below)
+  - caption (if heuristically detectable)
+"""
+from __future__ import annotations
+
+import hashlib
+import io
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ExtractedImage:
+    sequence_index: int
+    image_bytes: bytes
+    image_format: str            # "png" | "jpeg" | "jpg" | ...
+    width: int | None
+    height: int | None
+    page_number: int | None      # 1-based for PDFs (first occurrence)
+    bbox: dict | None            # {x0,y0,x1,y1} for PDFs (first occurrence)
+    surrounding_text: str        # paragraph(s) above + below the FIRST occurrence
+    caption: str | None
+    occurrence_pages: list[int] = field(default_factory=list)  # all pages this image appears on
+    is_recurring: bool = False   # likely header/logo/footer if appears on >25% of pages
+    image_hash: str = field(init=False)
+
+    def __post_init__(self):
+        self.image_hash = hashlib.sha256(self.image_bytes).hexdigest()
+
+
+# ── PDFs (PyMuPDF) ────────────────────────────────────────────────────────────
+
+# Filter: skip tiny decorative images and giant page-wide background images
+MIN_IMAGE_BYTES = 2 * 1024     # 2 KB
+MIN_DIMENSION = 50             # pixels
+MAX_DIMENSION = 8000           # pixels (filter out scanned-page backgrounds)
+
+
+def extract_images_from_pdf(pdf_path: Path) -> list[ExtractedImage]:
+    """Extract unique images from a PDF.
+
+    Critical: same logo / header / footer often appears on every page.
+    We dedupe by sha256 across the WHOLE document, keep the first occurrence's
+    page_number/bbox/surrounding_text, and record all pages it appears on so the
+    describer can flag recurring headers.
+    """
+    import fitz  # PyMuPDF
+
+    try:
+        doc = fitz.open(str(pdf_path))
+    except Exception as e:
+        logger.error("PyMuPDF could not open %s: %s", pdf_path.name, e)
+        return []
+
+    page_count = len(doc)
+
+    # First pass: build (hash → list of occurrences) and (xref → bytes/meta)
+    # An "occurrence" = the page it appeared on, plus first-seen bbox + page text.
+    # We process pages in order so the FIRST occurrence becomes the canonical one.
+    occurrences: dict[str, dict] = {}   # hash -> {bytes, ext, w, h, first_page, first_bbox, first_page_text, pages: [int]}
+
+    try:
+        for page_idx in range(page_count):
+            page = doc[page_idx]
+            page_num = page_idx + 1
+            page_text = page.get_text("text") or ""
+
+            seen_in_page: set[str] = set()
+            for img_info in page.get_images(full=True):
+                xref = img_info[0]
+                try:
+                    info = doc.extract_image(xref)
+                except Exception as e:
+                    logger.warning("extract_image failed (xref=%s) in %s: %s", xref, pdf_path.name, e)
+                    continue
+
+                img_bytes = info.get("image")
+                ext = (info.get("ext") or "png").lower()
+                w = info.get("width")
+                h = info.get("height")
+
+                if not img_bytes or len(img_bytes) < MIN_IMAGE_BYTES:
+                    continue
+                if w and h:
+                    if w < MIN_DIMENSION or h < MIN_DIMENSION:
+                        continue
+                    if w > MAX_DIMENSION or h > MAX_DIMENSION:
+                        continue
+
+                hsh = hashlib.sha256(img_bytes).hexdigest()
+                if hsh in seen_in_page:
+                    continue
+                seen_in_page.add(hsh)
+
+                if hsh in occurrences:
+                    # Already recorded — just track the page
+                    occurrences[hsh]["pages"].append(page_num)
+                    continue
+
+                # First time seeing this image — record canonical position
+                bbox = None
+                try:
+                    rects = page.get_image_rects(xref)
+                    if rects:
+                        r = rects[0]
+                        bbox = {"x0": float(r.x0), "y0": float(r.y0),
+                                "x1": float(r.x1), "y1": float(r.y1)}
+                except Exception:
+                    pass
+
+                occurrences[hsh] = {
+                    "bytes": img_bytes,
+                    "ext": ext,
+                    "w": w, "h": h,
+                    "first_page": page_num,
+                    "first_page_idx": page_idx,
+                    "first_bbox": bbox,
+                    "first_page_text": page_text,
+                    "pages": [page_num],
+                }
+    finally:
+        # Defer doc.close until after we read previous/next page text below
+        pass
+
+    # Second pass: build ExtractedImage per unique hash, with surrounding_text
+    # composed from the FIRST page's neighbors. Mark recurring if appears on
+    # > 25% of pages.
+    images: list[ExtractedImage] = []
+    seq = 0
+    recurring_threshold = max(3, int(page_count * 0.25))
+
+    try:
+        # Sort by first occurrence so sequence_index reflects reading order
+        for hsh, occ in sorted(occurrences.items(), key=lambda kv: (kv[1]["first_page"], kv[1]["pages"][0])):
+            page_idx = occ["first_page_idx"]
+            page_num = occ["first_page"]
+            page_text = occ["first_page_text"]
+
+            surrounding_parts = []
+            if page_idx > 0:
+                try:
+                    prev = doc[page_idx - 1].get_text("text") or ""
+                    if prev:
+                        surrounding_parts.append(prev[-600:].strip())
+                except Exception:
+                    pass
+            if page_text.strip():
+                surrounding_parts.append(page_text.strip()[:1500])
+            if page_idx + 1 < page_count:
+                try:
+                    nxt = doc[page_idx + 1].get_text("text") or ""
+                    if nxt:
+                        surrounding_parts.append(nxt[:600].strip())
+                except Exception:
+                    pass
+            surrounding_text = "\n\n".join(surrounding_parts)
+
+            caption = _find_caption(page_text, occ["first_bbox"])
+            occurrence_pages = sorted(set(occ["pages"]))
+            is_recurring = len(occurrence_pages) >= recurring_threshold and page_count >= 4
+
+            seq += 1
+            images.append(ExtractedImage(
+                sequence_index=seq,
+                image_bytes=occ["bytes"],
+                image_format=occ["ext"],
+                width=occ["w"],
+                height=occ["h"],
+                page_number=page_num,
+                bbox=occ["first_bbox"],
+                surrounding_text=surrounding_text,
+                caption=caption,
+                occurrence_pages=occurrence_pages,
+                is_recurring=is_recurring,
+            ))
+    finally:
+        doc.close()
+
+    return images
+
+
+def _find_caption(page_text: str, bbox: dict | None) -> str | None:
+    import re
+    if not page_text:
+        return None
+    # Heuristic: find a "Figure N: ..." or "Fig. N ..." or "Table N: ..." line
+    m = re.search(r"^(?:Figure|Fig\.|Table)\s*\d+[:.]?\s*[^\n]{0,200}",
+                  page_text, flags=re.MULTILINE)
+    if m:
+        return m.group(0).strip()
+    return None
+
+
+# ── DOCX (python-docx + relationships) ───────────────────────────────────────
+
+# Inline shapes / images live in the document's relationships.
+# We iterate paragraphs in order; whenever a run contains a drawing element
+# referencing an image relationship, we record that image at that point.
+
+def extract_images_from_docx(docx_path: Path) -> list[ExtractedImage]:
+    from docx import Document
+    from docx.oxml.ns import qn
+
+    images: list[ExtractedImage] = []
+    seq = 0
+
+    try:
+        doc = Document(str(docx_path))
+    except Exception as e:
+        logger.error("python-docx could not open %s: %s", docx_path.name, e)
+        return []
+
+    # Build a flat ordered list of (paragraph_index, paragraph_text, image_rels[])
+    paragraphs = doc.paragraphs
+
+    # First pass: linearize into events
+    events: list[tuple[str, int, str | list]] = []
+    # event types: ("text", para_idx, text), ("image", para_idx, [rel_id, ...])
+    for p_idx, para in enumerate(paragraphs):
+        text = para.text.strip()
+        if text:
+            events.append(("text", p_idx, text))
+
+        # Find image relationship ids inside this paragraph
+        rel_ids: list[str] = []
+        for run in para.runs:
+            for blip in run._element.iter(qn("a:blip")):
+                rid = blip.get(qn("r:embed"))
+                if rid:
+                    rel_ids.append(rid)
+        if rel_ids:
+            events.append(("image", p_idx, rel_ids))
+
+    # Second pass: emit ExtractedImage at each "image" event with surrounding text
+    rels = doc.part.related_parts
+    for ev_idx, ev in enumerate(events):
+        if ev[0] != "image":
+            continue
+        _, p_idx, rel_ids = ev
+
+        # Surrounding text: 2 prior text events + 2 following text events
+        before = [e[2] for e in events[max(0, ev_idx - 6):ev_idx] if e[0] == "text"][-3:]
+        after = [e[2] for e in events[ev_idx + 1: ev_idx + 7] if e[0] == "text"][:3]
+        surrounding_text = "\n\n".join(before + after)
+
+        # Caption: heuristic — paragraph immediately after that starts with Figure/Table
+        caption = None
+        for e in events[ev_idx + 1: ev_idx + 4]:
+            if e[0] == "text":
+                t = e[2]
+                if t[:7].lower() in ("figure ", "fig. ", "table ") or \
+                   t.lower().startswith(("figure", "fig.", "table")):
+                    caption = t[:200]
+                break
+
+        for rel_id in rel_ids:
+            part = rels.get(rel_id)
+            if part is None or not hasattr(part, "blob"):
+                continue
+            img_bytes: bytes = part.blob
+            if not img_bytes or len(img_bytes) < MIN_IMAGE_BYTES:
+                continue
+
+            # Try to determine format + dimensions via Pillow
+            ext = "png"
+            w = h = None
+            try:
+                from PIL import Image
+                im = Image.open(io.BytesIO(img_bytes))
+                ext = (im.format or "png").lower()
+                if ext == "jpeg":
+                    ext = "jpg"
+                w, h = im.size
+                if w < MIN_DIMENSION or h < MIN_DIMENSION:
+                    continue
+                if w > MAX_DIMENSION or h > MAX_DIMENSION:
+                    continue
+            except Exception:
+                pass
+
+            seq += 1
+            images.append(ExtractedImage(
+                sequence_index=seq,
+                image_bytes=img_bytes,
+                image_format=ext,
+                width=w,
+                height=h,
+                page_number=None,            # DOCX has no page numbers without rendering
+                bbox=None,
+                surrounding_text=surrounding_text,
+                caption=caption,
+            ))
+
+    return images
+
+
+# ── DOC binary (Word legacy) — convert to DOCX first via Word COM if available ──
+
+def extract_images_from_doc(doc_path: Path) -> list[ExtractedImage]:
+    """Best-effort: convert .doc to .docx via Word COM, then reuse DOCX extractor.
+    Returns [] if Microsoft Word isn't available on this host."""
+    import tempfile
+
+    try:
+        import win32com.client  # type: ignore
+        import pythoncom  # type: ignore
+    except ImportError:
+        logger.info("pywin32 not available — cannot extract images from .doc: %s", doc_path.name)
+        return []
+
+    pythoncom.CoInitialize()
+    word = None
+    try:
+        word = win32com.client.Dispatch("Word.Application")
+        word.Visible = False
+    except Exception as e:
+        logger.info("Microsoft Word COM unavailable — skipping images for %s (%s)", doc_path.name, e)
+        return []
+
+    tmp = Path(tempfile.gettempdir()) / f"{doc_path.stem}_converted.docx"
+    try:
+        d = word.Documents.Open(str(doc_path.resolve()))
+        # WdSaveFormat.wdFormatXMLDocument = 12 (.docx)
+        d.SaveAs(str(tmp), FileFormat=12)
+        d.Close(False)
+        word.Quit()
+    except Exception as e:
+        logger.warning("Could not convert %s -> docx: %s", doc_path.name, e)
+        try:
+            if word is not None:
+                word.Quit()
+        except Exception:
+            pass
+        return []
+
+    if not tmp.exists():
+        return []
+
+    images = extract_images_from_docx(tmp)
+    try:
+        tmp.unlink()
+    except Exception:
+        pass
+    return images
+
+
+# ── Dispatcher ────────────────────────────────────────────────────────────────
+
+def extract_images(file_path: Path) -> list[ExtractedImage]:
+    suffix = file_path.suffix.lower()
+    if suffix == ".pdf":
+        return extract_images_from_pdf(file_path)
+    if suffix == ".docx":
+        return extract_images_from_docx(file_path)
+    if suffix == ".doc":
+        return extract_images_from_doc(file_path)
+    # CSV, MD, MHT, etc. — no embedded images
+    return []

package/knowledge_base/parsers/markdown_parser.py

@@ -0,0 +1,116 @@
+"""
+Parses pre-converted Markdown files from marker-pdf output.
+Priority: if a PDF already has a markdown_output folder, use it — it's higher quality
+than raw pdfplumber extraction because it's been processed with Azure OpenAI LLM.
+"""
+from __future__ import annotations
+import re
+from pathlib import Path
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Section:
+    title: str
+    level: int          # 1=H1, 2=H2, 3=H3, 0=no heading
+    content: str
+    page_hint: int = 0  # approximate page (from image filenames or context)
+    metadata: dict = field(default_factory=dict)
+
+
+def parse_markdown_file(md_path: Path) -> list[Section]:
+    text = md_path.read_text(encoding="utf-8", errors="replace")
+    return _split_into_sections(text, str(md_path.name))
+
+
+def _split_into_sections(text: str, source_name: str) -> list[Section]:
+    # Strip HTML comments (marker-pdf source headers)
+    text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL).strip()
+
+    lines = text.split("\n")
+    sections: list[Section] = []
+    current_title = source_name
+    current_level = 0
+    current_lines: list[str] = []
+    page_hint = 1
+
+    for line in lines:
+        heading_match = re.match(r"^(#{1,4})\s+(.+)", line)
+        if heading_match:
+            # Flush previous section
+            content = "\n".join(current_lines).strip()
+            if content:
+                sections.append(Section(
+                    title=current_title,
+                    level=current_level,
+                    content=content,
+                    page_hint=page_hint,
+                ))
+            current_title = heading_match.group(2).strip()
+            current_level = len(heading_match.group(1))
+            current_lines = []
+        else:
+            # Track page hints from image references like images/page_5_img_1.png
+            page_ref = re.search(r"page_(\d+)", line)
+            if page_ref:
+                page_hint = int(page_ref.group(1))
+            current_lines.append(line)
+
+    # Flush last section
+    content = "\n".join(current_lines).strip()
+    if content:
+        sections.append(Section(
+            title=current_title,
+            level=current_level,
+            content=content,
+            page_hint=page_hint,
+        ))
+
+    # Merge tiny orphan sections (heading-only, < 50 chars) into next sibling
+    return _merge_orphan_sections(sections)
+
+
+def _merge_orphan_sections(sections: list[Section]) -> list[Section]:
+    merged: list[Section] = []
+    i = 0
+    while i < len(sections):
+        sec = sections[i]
+        # If this section has almost no content and there's a next section at same/lower level
+        if len(sec.content) < 60 and i + 1 < len(sections):
+            next_sec = sections[i + 1]
+            # Prepend this section's content into next section with heading as context
+            next_sec.content = f"**{sec.title}**\n{sec.content}\n\n{next_sec.content}".strip()
+            i += 1
+            continue
+        merged.append(sec)
+        i += 1
+    return merged
+
+
+def find_markdown_for_pdf(pdf_filename: str, markdown_output_dir: Path) -> Path | None:
+    """
+    Given a PDF filename, find its pre-converted markdown file in markdown_output/.
+    Returns the .md file path if found, else None.
+    """
+    stem = Path(pdf_filename).stem
+    # Normalize: spaces → underscores, remove parens/commas (matches convert_pdfs.py logic)
+    safe_dir = stem.replace(" ", "_").replace("(", "").replace(")", "").replace(",", "")
+    md_path = markdown_output_dir / safe_dir / f"{stem}.md"
+
+    if md_path.exists():
+        return md_path
+
+    # Also try with cleaned filename
+    for folder in markdown_output_dir.iterdir():
+        if folder.is_dir():
+            candidate = folder / f"{folder.name}.md"
+            if candidate.exists() and _fuzzy_match(stem, folder.name):
+                return candidate
+
+    return None
+
+
+def _fuzzy_match(a: str, b: str) -> bool:
+    """Simple match ignoring spaces, underscores, case."""
+    norm = lambda s: s.lower().replace("_", "").replace(" ", "").replace("-", "")
+    return norm(a) in norm(b) or norm(b) in norm(a)