ltcai 9.9.3 → 9.9.4

package/lattice_brain/graph/_kg_common.py

@@ -92,6 +92,341 @@ def _chunks(text: str, size: int = 1200, overlap: int = 160) -> List[str]:
     return chunks
 
 
+# ── Typed chunking (review 2026-07-25 §5.2 S2 — Wave 2.1 + 2.4) ──────────────
+# ``_chunks`` above is a compatibility contract (chunk ids hash over the chunk
+# text) and stays byte-for-byte untouched. ``typed_chunks`` layers strategy-
+# aware boundaries plus per-chunk provenance (start_char / heading_path) on
+# top; ``strategy="plain"`` reproduces the exact ``_chunks`` boundaries so
+# unchanged plain content keeps identical chunk ids.
+
+_MARKDOWN_CHUNK_EXTENSIONS = {".md", ".markdown"}
+_CODE_CHUNK_EXTENSIONS = {
+    ".py", ".js", ".jsx", ".ts", ".tsx", ".go", ".rs", ".java", ".rb",
+    ".c", ".h", ".cpp", ".css", ".sh", ".sql", ".vue", ".svelte",
+    ".json", ".yaml", ".yml", ".toml",
+}
+_CHUNK_STRATEGIES = {"plain", "markdown", "code"}
+# Markdown sections smaller than this merge forward into the next section so
+# heading-dense documents don't shatter into confetti chunks.
+_MARKDOWN_MIN_SECTION_CHARS = 200
+_MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6}) (.*)$", re.MULTILINE)
+_CODE_BOUNDARY_LINE_RE = re.compile(
+    r"^(?:def |class |function |export |const |public |private )", re.MULTILINE
+)
+_CODE_BLANK_RUN_RE = re.compile(r"\n\s*\n")
+
+
+def chunk_strategy_for(filename: Any, *, content_type: str = "") -> str:
+    """Route a filename / path / URI (plus optional MIME hint) to a strategy.
+
+    Returns ``"markdown"`` for .md/.markdown, ``"code"`` for known source-code
+    extensions, ``"plain"`` otherwise. Case-insensitive, tolerant of URLs
+    (query/fragment stripped) and ``Path`` objects; never raises — any
+    malformed input falls back to ``"plain"``.
+    """
+    try:
+        name = str(filename or "").strip().lower()
+        for sep in ("?", "#"):
+            name = name.split(sep, 1)[0]
+        name = name.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
+        dot = name.rfind(".")
+        ext = name[dot:] if dot > 0 else ""
+        if ext in _MARKDOWN_CHUNK_EXTENSIONS:
+            return "markdown"
+        if ext in _CODE_CHUNK_EXTENSIONS:
+            return "code"
+        if "markdown" in str(content_type or "").strip().lower():
+            return "markdown"
+    except Exception:
+        pass
+    return "plain"
+
+
+def _plain_windows(
+    cleaned: str,
+    size: int,
+    overlap: int,
+    *,
+    base_offset: int = 0,
+    strategy: str = "plain",
+    heading_path: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """The exact ``_chunks`` walk with ``start_char`` tracked.
+
+    Boundaries and chunk texts are byte-identical to ``_chunks`` over the same
+    string — this is the plain-strategy compatibility guarantee.
+    """
+    out: List[Dict[str, Any]] = []
+    start = 0
+    total = len(cleaned)
+    while start < total:
+        end = min(total, start + size)
+        out.append(
+            {
+                "text": cleaned[start:end],
+                "meta": {
+                    "strategy": strategy,
+                    "start_char": base_offset + start,
+                    "heading_path": heading_path,
+                },
+            }
+        )
+        if end >= total:
+            break
+        start = max(0, end - overlap)
+    return out
+
+
+def _markdown_section_spans(cleaned: str) -> List[Tuple[int, int, Optional[str]]]:
+    """``(start, end, heading_path)`` spans split at ``^#{1,6} `` heading lines.
+
+    ``heading_path`` is the " > "-joined path of the enclosing headings
+    including the section's own heading (e.g. ``"Guide > Setup"``); the
+    preamble before the first heading carries ``None``. Spans are contiguous
+    raw slices of ``cleaned`` so every chunk text round-trips via start_char.
+    """
+    spans: List[Tuple[int, int, Optional[str]]] = []
+    stack: List[Tuple[int, str]] = []
+    prev_start = 0
+    prev_path: Optional[str] = None
+    for match in _MARKDOWN_HEADING_RE.finditer(cleaned):
+        offset = match.start()
+        if offset > prev_start:
+            spans.append((prev_start, offset, prev_path))
+        level = len(match.group(1))
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+        stack.append((level, match.group(2).strip()))
+        prev_start = offset
+        prev_path = " > ".join(title for _, title in stack) or None
+    if len(cleaned) > prev_start:
+        spans.append((prev_start, len(cleaned), prev_path))
+    return spans
+
+
+def _merge_small_sections(
+    spans: List[Tuple[int, int, Optional[str]]], min_chars: int
+) -> List[Tuple[int, int, Optional[str]]]:
+    """Merge sections under ``min_chars`` forward into the next section.
+
+    A merged section keeps the heading_path of its first constituent (the
+    path in effect at the chunk start). A trailing undersized section merges
+    backward into the previous emitted section when one exists.
+    """
+    merged: List[Tuple[int, int, Optional[str]]] = []
+    pending: Optional[Tuple[int, int, Optional[str]]] = None
+    for start, end, path in spans:
+        if pending is None:
+            pending = (start, end, path)
+        else:
+            pending = (pending[0], end, pending[2])
+        if pending[1] - pending[0] >= min_chars:
+            merged.append(pending)
+            pending = None
+    if pending is not None:
+        if merged and pending[1] - pending[0] < min_chars:
+            last = merged.pop()
+            merged.append((last[0], pending[1], last[2]))
+        else:
+            merged.append(pending)
+    return merged
+
+
+def _markdown_chunks(cleaned: str, size: int, overlap: int) -> List[Dict[str, Any]]:
+    sections = _merge_small_sections(
+        _markdown_section_spans(cleaned), _MARKDOWN_MIN_SECTION_CHARS
+    )
+    out: List[Dict[str, Any]] = []
+    for start, end, path in sections:
+        body = cleaned[start:end]
+        if len(body) <= size:
+            out.append(
+                {
+                    "text": body,
+                    "meta": {
+                        "strategy": "markdown",
+                        "start_char": start,
+                        "heading_path": path,
+                    },
+                }
+            )
+        else:
+            out.extend(
+                _plain_windows(
+                    body,
+                    size,
+                    overlap,
+                    base_offset=start,
+                    strategy="markdown",
+                    heading_path=path,
+                )
+            )
+    return out
+
+
+def _code_segment_spans(cleaned: str) -> List[Tuple[int, int]]:
+    """Contiguous top-level segments split at blank-line runs and decl lines."""
+    boundaries = {0, len(cleaned)}
+    for match in _CODE_BLANK_RUN_RE.finditer(cleaned):
+        boundaries.add(match.end())
+    for match in _CODE_BOUNDARY_LINE_RE.finditer(cleaned):
+        boundaries.add(match.start())
+    ordered = sorted(boundaries)
+    return [
+        (ordered[i], ordered[i + 1])
+        for i in range(len(ordered) - 1)
+        if ordered[i + 1] > ordered[i]
+    ]
+
+
+def _code_chunks(cleaned: str, size: int, overlap: int) -> List[Dict[str, Any]]:
+    hard_limit = int(size * 1.5)
+    out: List[Dict[str, Any]] = []
+    pack: Optional[Tuple[int, int]] = None
+
+    def _emit(span: Tuple[int, int]) -> None:
+        out.append(
+            {
+                "text": cleaned[span[0] : span[1]],
+                "meta": {
+                    "strategy": "code",
+                    "start_char": span[0],
+                    "heading_path": None,
+                },
+            }
+        )
+
+    for start, end in _code_segment_spans(cleaned):
+        if end - start > hard_limit:
+            # Monster segment: flush the pack, then window it like plain text.
+            if pack is not None:
+                _emit(pack)
+                pack = None
+            out.extend(
+                _plain_windows(
+                    cleaned[start:end],
+                    size,
+                    overlap,
+                    base_offset=start,
+                    strategy="code",
+                )
+            )
+            continue
+        if pack is None:
+            pack = (start, end)
+        elif end - pack[0] <= size:
+            pack = (pack[0], end)
+        else:
+            _emit(pack)
+            pack = (start, end)
+    if pack is not None:
+        _emit(pack)
+    return out
+
+
+def typed_chunks(
+    text: str,
+    *,
+    strategy: str = "plain",
+    size: int = 1200,
+    overlap: int = 160,
+) -> List[Dict[str, Any]]:
+    """Strategy-aware chunking with per-chunk provenance metadata.
+
+    Returns ``[{"text": str, "meta": {"strategy", "start_char", "heading_path"}}]``
+    where ``start_char`` is the offset in ``str(text or "").strip()`` (every
+    chunk text is an exact substring at that offset).
+
+    Contract: ``[c["text"] for c in typed_chunks(t)] == _chunks(t)`` for the
+    default plain strategy — unknown strategies also fall back to plain.
+    """
+    cleaned = str(text or "").strip()
+    if not cleaned:
+        return []
+    try:
+        size = max(1, int(size))
+    except Exception:
+        size = 1200
+    try:
+        overlap = min(max(0, int(overlap)), size - 1)
+    except Exception:
+        overlap = min(160, size - 1)
+    label = strategy if strategy in _CHUNK_STRATEGIES else "plain"
+    if label == "markdown":
+        return _markdown_chunks(cleaned, size, overlap)
+    if label == "code":
+        return _code_chunks(cleaned, size, overlap)
+    return _plain_windows(cleaned, size, overlap)
+
+
+def typed_chunk_meta_fields(piece: Dict[str, Any]) -> Dict[str, Any]:
+    """Additive chunk-metadata fields for one ``typed_chunks`` piece.
+
+    Ingest call sites merge this into the existing ``{"index", "source_node"}``
+    chunk metadata; ``heading_path`` is only present when known — honest
+    absence over empty labels.
+    """
+    meta = piece.get("meta") or {}
+    fields: Dict[str, Any] = {
+        "strategy": str(meta.get("strategy") or "plain"),
+        "start_char": int(meta.get("start_char") or 0),
+    }
+    heading_path = meta.get("heading_path")
+    if heading_path:
+        fields["heading_path"] = str(heading_path)
+    return fields
+
+
+def pdf_page_offsets(structure: Any) -> List[int]:
+    """Start offset of each PDF page in the "\\n\\n"-joined page text.
+
+    ``structure`` is the ``metadata["structure"]`` dict produced by
+    ``_pdf_structure`` (``pages`` = ``[{"chars": int, ...}, ...]``); pages were
+    joined with ``"\\n\\n"`` (see ``read_document``), so page k starts at
+    ``sum(chars[j] + 2 for j < k)``. Empty or malformed input returns ``[]``.
+    """
+    if not isinstance(structure, dict):
+        return []
+    pages = structure.get("pages")
+    if not isinstance(pages, list) or not pages:
+        return []
+    offsets: List[int] = []
+    cursor = 0
+    for page in pages:
+        if not isinstance(page, dict):
+            return []
+        chars = page.get("chars")
+        if isinstance(chars, bool) or not isinstance(chars, (int, float)) or chars < 0:
+            return []
+        offsets.append(cursor)
+        cursor += int(chars) + 2  # +2 for the "\n\n" page joiner
+    return offsets
+
+
+def page_for_offset(page_offsets: List[int], offset: int) -> Optional[int]:
+    """1-based page number containing ``offset`` given page start offsets.
+
+    Returns ``None`` when ``page_offsets`` is empty or the offset precedes the
+    first page start (honest absence over a wrong label).
+    """
+    if not page_offsets:
+        return None
+    try:
+        target = int(offset)
+    except Exception:
+        return None
+    page = 0
+    for index, start in enumerate(page_offsets):
+        try:
+            if target >= int(start):
+                page = index + 1
+            else:
+                break
+        except Exception:
+            return None
+    return page if page >= 1 else None
+
+
 _LLM_EXTRACT_CONCEPT_PROMPT = """Extract the key concepts from the following text.
 Return ONLY a JSON array of objects, each with "concept" (string) and "importance" (float 0-1).
 Extract up to {limit} concepts. Focus on named entities, technical terms, and domain-specific nouns.

package/lattice_brain/graph/discovery_index.py

@@ -474,7 +474,12 @@ class KnowledgeGraphLocalIndexMixin:
                 metadata={"source": "ocr", "workspace_id": workspace_id},
             )
 
-        for index, chunk in enumerate(_chunks(text)):
+        # Typed chunking by file extension (markdown/code/plain); plain files
+        # keep legacy _chunks boundaries and therefore identical chunk ids.
+        chunk_strategy = chunk_strategy_for(file_path.name)
+        for index, piece in enumerate(typed_chunks(text, strategy=chunk_strategy)):
+            chunk = piece["text"]
+            chunk_fields = typed_chunk_meta_fields(piece)
             chunk_id = f"chunk:{_sha256_text(f'{file_node_id}:{index}:{chunk}')[:24]}"
             self._upsert_node(
                 conn,
@@ -487,6 +492,7 @@ class KnowledgeGraphLocalIndexMixin:
                     "source_node": file_node_id,
                     "source_id": source_id,
                     "workspace_id": workspace_id,
+                    **chunk_fields,
                 },
                 owner=user_email,
                 workspace_id=workspace_id,
@@ -501,6 +507,7 @@ class KnowledgeGraphLocalIndexMixin:
                     "source_node": file_node_id,
                     "source_id": source_id,
                     "workspace_id": workspace_id,
+                    **chunk_fields,
                 },
             )
             self._upsert_edge(

package/lattice_brain/graph/ingest.py

@@ -108,7 +108,11 @@ class KnowledgeGraphIngestMixin:
             )
 
             # ── 4. RAG chunks  (검색용, 그래프에서 숨김) ──────────────────────
-            for index, chunk in enumerate(_chunks(content)):
+            # Chat text is always the plain strategy: boundaries (and thus
+            # chunk ids) stay byte-identical to the legacy _chunks path.
+            for index, piece in enumerate(typed_chunks(content, strategy="plain")):
+                chunk = piece["text"]
+                chunk_fields = typed_chunk_meta_fields(piece)
                 chunk_id = f"chunk:{_sha256_text(f'{node_id}:{index}:{chunk}')[:24]}"
                 self._upsert_node(
                     conn,
@@ -116,7 +120,7 @@ class KnowledgeGraphIngestMixin:
                     "Chunk",
                     f"chunk {index + 1}",
                     summary=chunk[:500],
-                    metadata={"index": index, "source_node": node_id},
+                    metadata={"index": index, "source_node": node_id, **chunk_fields},
                     owner=user_email,
                     workspace_id=workspace_id,
                 )
@@ -125,7 +129,7 @@ class KnowledgeGraphIngestMixin:
                     chunk_id=chunk_id,
                     source_node=node_id,
                     text=chunk,
-                    metadata={"index": index, "source_node": node_id},
+                    metadata={"index": index, "source_node": node_id, **chunk_fields},
                 )
                 self._upsert_edge(conn, node_id, chunk_id, "포함함")
 
@@ -322,7 +326,25 @@ class KnowledgeGraphIngestMixin:
                 self._upsert_edge(conn, conv_id, file_id, "언급함", weight=0.8)
 
             # ── RAG chunks (검색용, 그래프 비표시) ────────────────────────────
-            for index, chunk in enumerate(_chunks(text)):
+            # Strategy from the filename (plain files keep legacy boundaries →
+            # identical chunk ids). For PDFs, per-page start offsets in the
+            # "\n\n"-joined page text let each chunk carry a 1-based "page" —
+            # only when the claimed extent matches the text we actually chunk
+            # (honest absence over wrong labels).
+            chunk_strategy = chunk_strategy_for(filename, content_type=mime_type or "")
+            page_offsets: List[int] = []
+            if ext == ".pdf":
+                page_offsets = pdf_page_offsets(doc_meta)
+                cleaned_len = len(str(text or "").strip())
+                if not page_offsets or page_offsets[-1] >= cleaned_len * 1.2:
+                    page_offsets = []
+            for index, piece in enumerate(typed_chunks(text, strategy=chunk_strategy)):
+                chunk = piece["text"]
+                chunk_fields = typed_chunk_meta_fields(piece)
+                if page_offsets:
+                    page = page_for_offset(page_offsets, chunk_fields["start_char"])
+                    if page is not None:
+                        chunk_fields["page"] = page
                 chunk_id = f"chunk:{_sha256_text(f'{file_id}:{index}:{chunk}')[:24]}"
                 chunk_ids.append(chunk_id)
                 self._upsert_node(
@@ -331,7 +353,7 @@ class KnowledgeGraphIngestMixin:
                     "Chunk",
                     f"{filename} chunk {index + 1}",
                     summary=chunk[:500],
-                    metadata={"index": index, "source_node": file_id, "workspace_id": workspace_id},
+                    metadata={"index": index, "source_node": file_id, "workspace_id": workspace_id, **chunk_fields},
                     owner=owner or uploader,
                     workspace_id=workspace_id,
                 )
@@ -340,7 +362,7 @@ class KnowledgeGraphIngestMixin:
                     chunk_id=chunk_id,
                     source_node=file_id,
                     text=chunk,
-                    metadata={"index": index, "source_node": file_id},
+                    metadata={"index": index, "source_node": file_id, **chunk_fields},
                 )
                 self._upsert_edge(conn, file_id, chunk_id, "포함함")
 
@@ -651,7 +673,19 @@ class KnowledgeGraphIngestMixin:
                 )
                 self._upsert_edge(conn, conv_id, content_id, "언급함", weight=0.8)
             # ── RAG 청크 ────────────────────────────────────────────────────
-            for index, chunk in enumerate(_chunks(text)):
+            # Best available filename hint: source URI, then path/filename from
+            # caller metadata. Plain content keeps legacy chunk boundaries and
+            # therefore identical chunk ids.
+            chunk_strategy = chunk_strategy_for(
+                source_uri
+                or (metadata or {}).get("path")
+                or (metadata or {}).get("filename")
+                or "",
+                content_type=str((metadata or {}).get("mime_type") or ""),
+            )
+            for index, piece in enumerate(typed_chunks(text, strategy=chunk_strategy)):
+                chunk = piece["text"]
+                chunk_fields = typed_chunk_meta_fields(piece)
                 chunk_id = f"chunk:{_sha256_text(f'{content_id}:{index}:{chunk}')[:24]}"
                 chunk_ids.append(chunk_id)
                 self._upsert_node(
@@ -660,7 +694,7 @@ class KnowledgeGraphIngestMixin:
                     "Chunk",
                     f"{title} chunk {index + 1}",
                     summary=chunk[:500],
-                    metadata={"index": index, "source_node": content_id, "workspace_id": workspace_id},
+                    metadata={"index": index, "source_node": content_id, "workspace_id": workspace_id, **chunk_fields},
                     owner=owner,
                     workspace_id=workspace_id,
                 )
@@ -669,7 +703,7 @@ class KnowledgeGraphIngestMixin:
                     chunk_id=chunk_id,
                     source_node=content_id,
                     text=chunk,
-                    metadata={"index": index, "source_node": content_id},
+                    metadata={"index": index, "source_node": content_id, **chunk_fields},
                 )
                 self._upsert_edge(conn, content_id, chunk_id, "포함함")
             # ── Concept / Feature / Error / Code 노드 + 엣지 ────────────────