foxclaw 0.1.0 → 0.1.3

package/python/vendor/hkc/hkc_kde/loaders/pdf_loader.py

@@ -0,0 +1,162 @@
+"""
+hkc-kde / loaders / pdf_loader.py
+PDF 解析器，使用 PyMuPDF（fitz）。
+
+策略：
+- 按页提取文字
+- 检测标题（字体大小 > 阈值）作为章节边界
+- 表格识别（简单规则：连续短行 + 数字占比高）
+"""
+import uuid
+from pathlib import Path
+
+try:
+    import fitz  # PyMuPDF
+    HAS_FITZ = True
+except ImportError:
+    HAS_FITZ = False
+
+from ..models import ParsedDocument, Chapter, Section
+
+
+# 标题字体大小阈值（相对于正文字体）
+_TITLE_SIZE_RATIO = 1.2
+
+
+class PDFLoader:
+
+    def load(self, path: str) -> ParsedDocument:
+        if not HAS_FITZ:
+            raise ImportError("PyMuPDF 未安装，请运行: pip install pymupdf")
+
+        path = Path(path)
+        if not path.exists():
+            raise FileNotFoundError(f"文件不存在: {path}")
+
+        doc = fitz.open(str(path))
+        # try/finally:解析(坏 PDF)抛异常时也要 close,否则泄漏 PyMuPDF 原生句柄。
+        try:
+            doc_id  = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+            title   = self._extract_title(doc, path.stem)
+            chapters = self._parse_chapters(doc)
+            page_count = doc.page_count if hasattr(doc, 'page_count') else 0  # 必须在 close 前取
+        finally:
+            doc.close()
+
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = title,
+            source   = str(path),
+            doc_type = "PDF",
+            chapters = chapters,
+            metadata = {
+                "page_count": page_count,
+                "file_size":  path.stat().st_size,
+            },
+        )
+
+    def _extract_title(self, doc: "fitz.Document", fallback: str) -> str:
+        """尝试从第一页提取标题，失败则用文件名。"""
+        try:
+            meta = doc.metadata
+            if meta.get("title"):
+                return meta["title"].strip()
+            # 取第一页最大字体的文字
+            page = doc[0]
+            blocks = page.get_text("dict")["blocks"]
+            max_size = 0
+            title_text = fallback
+            for block in blocks:
+                for line in block.get("lines", []):
+                    for span in line.get("spans", []):
+                        if span["size"] > max_size and len(span["text"].strip()) > 3:
+                            max_size = span["size"]
+                            title_text = span["text"].strip()
+            return title_text
+        except Exception:
+            return fallback
+
+    def _parse_chapters(self, doc: "fitz.Document") -> list[Chapter]:
+        """
+        按页遍历，检测标题行作为章节边界。
+        同一页内的段落归入当前章节。
+        """
+        chapters: list[Chapter] = []
+        current_chapter: Chapter | None = None
+        current_sections: list[Section] = []
+        body_font_size = self._detect_body_font(doc)
+
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            blocks = page.get_text("dict")["blocks"]
+
+            for block in blocks:
+                block_text = ""
+                is_title   = False
+                lines = block.get("lines", [])
+
+                for line in lines:
+                    line_text = ""
+                    for span in line.get("spans", []):
+                        text = span["text"].strip()
+                        if not text:
+                            continue
+                        # 字体大于正文 1.2 倍 → 判定为标题
+                        if span["size"] >= body_font_size * _TITLE_SIZE_RATIO:
+                            is_title = True
+                        line_text += text + " "
+                    block_text += line_text.strip() + "\n"
+
+                block_text = block_text.strip()
+                if not block_text:
+                    continue
+
+                if is_title and len(block_text) < 120:
+                    # 保存上一章节
+                    if current_chapter is not None:
+                        current_chapter.sections = current_sections
+                        chapters.append(current_chapter)
+                    ch_id = f"CH_{len(chapters)+1:04d}"
+                    current_chapter  = Chapter(chapter_id=ch_id, title=block_text)
+                    current_sections = []
+                else:
+                    # 普通段落
+                    if current_chapter is None:
+                        current_chapter  = Chapter(chapter_id="CH_0000", title="")
+                        current_sections = []
+                    sec_id = f"SEC_{page_num:04d}_{len(current_sections):04d}"
+                    current_sections.append(Section(
+                        section_id = sec_id,
+                        content    = block_text,
+                    ))
+
+        # 收尾
+        if current_chapter is not None:
+            current_chapter.sections = current_sections
+            chapters.append(current_chapter)
+
+        return chapters if chapters else [
+            Chapter(
+                chapter_id = "CH_0001",
+                title      = "",
+                sections   = [Section("SEC_0000", doc[0].get_text() if len(doc) else "")]
+            )
+        ]
+
+    def _detect_body_font(self, doc: "fitz.Document") -> float:
+        """
+        统计全文字体大小分布，取众数作为正文字体。
+        取前3页采样，速度快且足够准确。
+        """
+        from collections import Counter
+        sizes: list[float] = []
+        sample_pages = min(3, len(doc))
+        for i in range(sample_pages):
+            for block in doc[i].get_text("dict")["blocks"]:
+                for line in block.get("lines", []):
+                    for span in line.get("spans", []):
+                        if span["text"].strip():
+                            sizes.append(round(span["size"], 1))
+        if not sizes:
+            return 12.0
+        return Counter(sizes).most_common(1)[0][0]

package/python/vendor/hkc/hkc_kde/loaders/text_loader.py

@@ -0,0 +1,157 @@
+"""
+hkc-kde / loaders / text_loader.py
+Markdown 和纯文本 Loader。
+
+Markdown：
+- ## 标题 → 章节边界
+- 段落（空行分隔）→ Section
+
+纯文本：
+- 空行分段，每 N 段合并为一个 Chapter
+"""
+import re
+import uuid
+from pathlib import Path
+
+from ..models import ParsedDocument, Chapter, Section
+
+
+class MarkdownLoader:
+
+    def load(self, path: str) -> ParsedDocument:
+        path = Path(path)
+        content = path.read_text(encoding="utf-8", errors="replace")
+        doc_id  = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+        chapters = self._parse_markdown(content)
+
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = self._extract_title(content, path.stem),
+            source   = str(path),
+            doc_type = "MARKDOWN",
+            chapters = chapters,
+        )
+
+    def load_string(self, content: str, source: str = "inline") -> ParsedDocument:
+        """直接从字符串加载，用于测试和管道传入。"""
+        doc_id   = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+        chapters = self._parse_markdown(content)
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = self._extract_title(content, source),
+            source   = source,
+            doc_type = "MARKDOWN",
+            chapters = chapters,
+        )
+
+    def _extract_title(self, content: str, fallback: str) -> str:
+        """取第一个 # 标题，或文件名。"""
+        m = re.search(r'^#\s+(.+)$', content, re.MULTILINE)
+        return m.group(1).strip() if m else fallback
+
+    def _parse_markdown(self, content: str) -> list[Chapter]:
+        """
+        按 ## 及以上标题切割章节，## 以下的段落归入当前章节。
+        """
+        chapters: list[Chapter] = []
+        current_title    = ""
+        current_paras:   list[str] = []
+        ch_counter       = 0
+
+        def _flush():
+            nonlocal ch_counter
+            if not current_paras:
+                return
+            ch_counter += 1
+            secs = [
+                Section(f"SEC_{ch_counter:04d}_{i:04d}", p.strip())
+                for i, p in enumerate(current_paras)
+                if p.strip()
+            ]
+            chapters.append(Chapter(
+                chapter_id = f"CH_{ch_counter:04d}",
+                title      = current_title,
+                sections   = secs,
+            ))
+
+        # 按空行分段
+        raw_paras = re.split(r'\n{2,}', content)
+
+        for para in raw_paras:
+            para = para.strip()
+            if not para:
+                continue
+            # 检测是否是标题行（## 或更高级）
+            m = re.match(r'^#{1,4}\s+(.+)$', para)
+            if m:
+                _flush()
+                current_title = m.group(1).strip()
+                current_paras = []
+            else:
+                # 去掉 Markdown 语法标记，保留纯文字
+                clean = self._strip_md(para)
+                if clean:
+                    current_paras.append(clean)
+
+        _flush()
+
+        if not chapters:
+            chapters = [Chapter("CH_0001", "", [Section("SEC_0001", content)])]
+
+        return chapters
+
+    def _strip_md(self, text: str) -> str:
+        """去掉常见 Markdown 标记，保留纯文字内容。"""
+        # 代码块
+        text = re.sub(r'```[\s\S]*?```', '', text)
+        text = re.sub(r'`[^`]+`', lambda m: m.group(0)[1:-1], text)
+        # 链接 [text](url) → text
+        text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text)
+        # 图片
+        text = re.sub(r'!\[[^\]]*\]\([^\)]+\)', '', text)
+        # 加粗 / 斜体
+        text = re.sub(r'\*{1,3}([^*]+)\*{1,3}', r'\1', text)
+        text = re.sub(r'_{1,3}([^_]+)_{1,3}',   r'\1', text)
+        # 引用块
+        text = re.sub(r'^>\s*', '', text, flags=re.MULTILINE)
+        # 列表符号
+        text = re.sub(r'^[-*+]\s+', '', text, flags=re.MULTILINE)
+        text = re.sub(r'^\d+\.\s+', '', text, flags=re.MULTILINE)
+        return text.strip()
+
+
+class TxtLoader:
+
+    PARAS_PER_CHAPTER = 10   # 每 10 个段落合并为一个虚拟章节
+
+    def load(self, path: str) -> ParsedDocument:
+        path    = Path(path)
+        content = path.read_text(encoding="utf-8", errors="replace")
+        doc_id  = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = path.stem,
+            source   = str(path),
+            doc_type = "TXT",
+            chapters = self._parse_txt(content),
+        )
+
+    def _parse_txt(self, content: str) -> list[Chapter]:
+        paras = [p.strip() for p in re.split(r'\n{2,}', content) if p.strip()]
+        chapters: list[Chapter] = []
+
+        for batch_start in range(0, len(paras), self.PARAS_PER_CHAPTER):
+            batch = paras[batch_start: batch_start + self.PARAS_PER_CHAPTER]
+            ch_num = len(chapters) + 1
+            secs = [
+                Section(f"SEC_{ch_num:04d}_{i:04d}", p)
+                for i, p in enumerate(batch)
+            ]
+            chapters.append(Chapter(
+                chapter_id = f"CH_{ch_num:04d}",
+                title      = f"段落组 {ch_num}",
+                sections   = secs,
+            ))
+
+        return chapters or [Chapter("CH_0001", "", [Section("SEC_0001", content)])]

package/python/vendor/hkc/hkc_kde/loaders/web_loader.py

@@ -0,0 +1,91 @@
+"""
+hkc-kde / loaders / web_loader.py
+网页 Loader，使用 trafilatura 提取正文。
+
+策略：
+- trafilatura 提取正文（自动去广告、导航等噪音）
+- 提取到的文本交给 MarkdownLoader 的段落解析逻辑
+- 保留原始 URL 作为 source
+"""
+import uuid
+from urllib.parse import urlparse
+
+try:
+    import trafilatura
+    HAS_TRAFILATURA = True
+except ImportError:
+    HAS_TRAFILATURA = False
+
+try:
+    import requests as _requests
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+from ..models import ParsedDocument
+from .text_loader import MarkdownLoader
+
+
+class WebLoader:
+
+    TIMEOUT = 15  # 秒
+
+    def load(self, url: str) -> ParsedDocument:
+        if not HAS_TRAFILATURA:
+            raise ImportError("trafilatura 未安装，请运行: pip install trafilatura")
+        if not HAS_REQUESTS:
+            raise ImportError("requests 未安装，请运行: pip install requests")
+
+        html    = self._fetch(url)
+        text    = self._extract(html, url)
+        doc_id  = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+        title   = self._extract_title(html, url)
+        md_loader = MarkdownLoader()
+        parsed    = md_loader.load_string(text, source=url)
+
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = title,
+            source   = url,
+            doc_type = "WEB",
+            chapters = parsed.chapters,
+            metadata = {"url": url, "domain": urlparse(url).netloc},
+        )
+
+    def load_html(self, html: str, url: str = "local") -> ParsedDocument:
+        """直接从 HTML 字符串加载，用于测试。"""
+        text    = self._extract(html, url)
+        doc_id  = f"DOC_{uuid.uuid4().hex[:8].upper()}"
+        title   = self._extract_title(html, url)
+        parsed  = MarkdownLoader().load_string(text, source=url)
+        return ParsedDocument(
+            doc_id   = doc_id,
+            title    = title,
+            source   = url,
+            doc_type = "WEB",
+            chapters = parsed.chapters,
+        )
+
+    def _fetch(self, url: str) -> str:
+        resp = _requests.get(url, timeout=self.TIMEOUT, headers={
+            "User-Agent": "Mozilla/5.0 (compatible; HKC-KDE/1.0)"
+        })
+        resp.raise_for_status()
+        return resp.text
+
+    def _extract(self, html: str, url: str) -> str:
+        text = trafilatura.extract(
+            html,
+            url              = url,
+            include_comments = False,
+            include_tables   = True,
+            no_fallback      = False,
+        )
+        return text or ""
+
+    def _extract_title(self, html: str, fallback: str) -> str:
+        import re
+        m = re.search(r'<title[^>]*>([^<]+)</title>', html, re.IGNORECASE)
+        if m:
+            return m.group(1).strip()
+        return urlparse(fallback).path.split("/")[-1] or fallback

package/python/vendor/hkc/hkc_kde/models.py

@@ -0,0 +1,74 @@
+"""
+hkc-kde / models.py
+KDE 内部数据结构。
+ParsedDocument → ExtractionResult → KU（交给 Packager）
+"""
+from __future__ import annotations
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Section:
+    """文档的最小文本单元，对应一个段落或小节。"""
+    section_id: str
+    content:    str
+    tables:     list[dict] = field(default_factory=list)  # 表格内容
+    images:     list[str]  = field(default_factory=list)  # OCR 文字
+
+
+@dataclass
+class Chapter:
+    chapter_id: str
+    title:      str
+    sections:   list[Section] = field(default_factory=list)
+
+
+@dataclass
+class ParsedDocument:
+    doc_id:   str
+    title:    str
+    source:   str           # 文件路径或 URL
+    doc_type: str           # PDF | MARKDOWN | WEB | TXT
+    chapters: list[Chapter] = field(default_factory=list)
+    metadata: dict          = field(default_factory=dict)
+
+    def all_sections(self) -> list[Section]:
+        """展平所有章节，返回全部 Section 列表。"""
+        result = []
+        for ch in self.chapters:
+            result.extend(ch.sections)
+        return result
+
+    def full_text(self) -> str:
+        """返回全文拼接，用于整体摘要等场景。"""
+        parts = []
+        for ch in self.chapters:
+            if ch.title:
+                parts.append(ch.title)
+            for sec in ch.sections:
+                if sec.content.strip():
+                    parts.append(sec.content)
+        return "\n\n".join(parts)
+
+
+@dataclass
+class RawExtraction:
+    """LLM 从单个 Chunk 提取的原始结果（未分配 ID）。"""
+    facts:    list[dict] = field(default_factory=list)
+    claims:   list[dict] = field(default_factory=list)
+    entities: list[dict] = field(default_factory=list)
+    concepts: list[dict] = field(default_factory=list)
+    source_hint: str = ""   # 所在章节标题，用于 Evidence 溯源
+
+
+@dataclass
+class ExtractionResult:
+    """整个文档的提取结果汇总。"""
+    doc_id:   str
+    source:   str
+    items:    list[RawExtraction] = field(default_factory=list)
+
+    def all_facts(self)    -> list[dict]: return [f for r in self.items for f in r.facts]
+    def all_claims(self)   -> list[dict]: return [c for r in self.items for c in r.claims]
+    def all_entities(self) -> list[dict]: return [e for r in self.items for e in r.entities]
+    def all_concepts(self) -> list[dict]: return [c for r in self.items for c in r.concepts]