ltcai 9.9.2 → 9.9.4

package/lattice_brain/graph/_kg_common.py

@@ -92,6 +92,341 @@ def _chunks(text: str, size: int = 1200, overlap: int = 160) -> List[str]:
     return chunks
 
 
+# ── Typed chunking (review 2026-07-25 §5.2 S2 — Wave 2.1 + 2.4) ──────────────
+# ``_chunks`` above is a compatibility contract (chunk ids hash over the chunk
+# text) and stays byte-for-byte untouched. ``typed_chunks`` layers strategy-
+# aware boundaries plus per-chunk provenance (start_char / heading_path) on
+# top; ``strategy="plain"`` reproduces the exact ``_chunks`` boundaries so
+# unchanged plain content keeps identical chunk ids.
+
+_MARKDOWN_CHUNK_EXTENSIONS = {".md", ".markdown"}
+_CODE_CHUNK_EXTENSIONS = {
+    ".py", ".js", ".jsx", ".ts", ".tsx", ".go", ".rs", ".java", ".rb",
+    ".c", ".h", ".cpp", ".css", ".sh", ".sql", ".vue", ".svelte",
+    ".json", ".yaml", ".yml", ".toml",
+}
+_CHUNK_STRATEGIES = {"plain", "markdown", "code"}
+# Markdown sections smaller than this merge forward into the next section so
+# heading-dense documents don't shatter into confetti chunks.
+_MARKDOWN_MIN_SECTION_CHARS = 200
+_MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6}) (.*)$", re.MULTILINE)
+_CODE_BOUNDARY_LINE_RE = re.compile(
+    r"^(?:def |class |function |export |const |public |private )", re.MULTILINE
+)
+_CODE_BLANK_RUN_RE = re.compile(r"\n\s*\n")
+
+
+def chunk_strategy_for(filename: Any, *, content_type: str = "") -> str:
+    """Route a filename / path / URI (plus optional MIME hint) to a strategy.
+
+    Returns ``"markdown"`` for .md/.markdown, ``"code"`` for known source-code
+    extensions, ``"plain"`` otherwise. Case-insensitive, tolerant of URLs
+    (query/fragment stripped) and ``Path`` objects; never raises — any
+    malformed input falls back to ``"plain"``.
+    """
+    try:
+        name = str(filename or "").strip().lower()
+        for sep in ("?", "#"):
+            name = name.split(sep, 1)[0]
+        name = name.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
+        dot = name.rfind(".")
+        ext = name[dot:] if dot > 0 else ""
+        if ext in _MARKDOWN_CHUNK_EXTENSIONS:
+            return "markdown"
+        if ext in _CODE_CHUNK_EXTENSIONS:
+            return "code"
+        if "markdown" in str(content_type or "").strip().lower():
+            return "markdown"
+    except Exception:
+        pass
+    return "plain"
+
+
+def _plain_windows(
+    cleaned: str,
+    size: int,
+    overlap: int,
+    *,
+    base_offset: int = 0,
+    strategy: str = "plain",
+    heading_path: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """The exact ``_chunks`` walk with ``start_char`` tracked.
+
+    Boundaries and chunk texts are byte-identical to ``_chunks`` over the same
+    string — this is the plain-strategy compatibility guarantee.
+    """
+    out: List[Dict[str, Any]] = []
+    start = 0
+    total = len(cleaned)
+    while start < total:
+        end = min(total, start + size)
+        out.append(
+            {
+                "text": cleaned[start:end],
+                "meta": {
+                    "strategy": strategy,
+                    "start_char": base_offset + start,
+                    "heading_path": heading_path,
+                },
+            }
+        )
+        if end >= total:
+            break
+        start = max(0, end - overlap)
+    return out
+
+
+def _markdown_section_spans(cleaned: str) -> List[Tuple[int, int, Optional[str]]]:
+    """``(start, end, heading_path)`` spans split at ``^#{1,6} `` heading lines.
+
+    ``heading_path`` is the " > "-joined path of the enclosing headings
+    including the section's own heading (e.g. ``"Guide > Setup"``); the
+    preamble before the first heading carries ``None``. Spans are contiguous
+    raw slices of ``cleaned`` so every chunk text round-trips via start_char.
+    """
+    spans: List[Tuple[int, int, Optional[str]]] = []
+    stack: List[Tuple[int, str]] = []
+    prev_start = 0
+    prev_path: Optional[str] = None
+    for match in _MARKDOWN_HEADING_RE.finditer(cleaned):
+        offset = match.start()
+        if offset > prev_start:
+            spans.append((prev_start, offset, prev_path))
+        level = len(match.group(1))
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+        stack.append((level, match.group(2).strip()))
+        prev_start = offset
+        prev_path = " > ".join(title for _, title in stack) or None
+    if len(cleaned) > prev_start:
+        spans.append((prev_start, len(cleaned), prev_path))
+    return spans
+
+
+def _merge_small_sections(
+    spans: List[Tuple[int, int, Optional[str]]], min_chars: int
+) -> List[Tuple[int, int, Optional[str]]]:
+    """Merge sections under ``min_chars`` forward into the next section.
+
+    A merged section keeps the heading_path of its first constituent (the
+    path in effect at the chunk start). A trailing undersized section merges
+    backward into the previous emitted section when one exists.
+    """
+    merged: List[Tuple[int, int, Optional[str]]] = []
+    pending: Optional[Tuple[int, int, Optional[str]]] = None
+    for start, end, path in spans:
+        if pending is None:
+            pending = (start, end, path)
+        else:
+            pending = (pending[0], end, pending[2])
+        if pending[1] - pending[0] >= min_chars:
+            merged.append(pending)
+            pending = None
+    if pending is not None:
+        if merged and pending[1] - pending[0] < min_chars:
+            last = merged.pop()
+            merged.append((last[0], pending[1], last[2]))
+        else:
+            merged.append(pending)
+    return merged
+
+
+def _markdown_chunks(cleaned: str, size: int, overlap: int) -> List[Dict[str, Any]]:
+    sections = _merge_small_sections(
+        _markdown_section_spans(cleaned), _MARKDOWN_MIN_SECTION_CHARS
+    )
+    out: List[Dict[str, Any]] = []
+    for start, end, path in sections:
+        body = cleaned[start:end]
+        if len(body) <= size:
+            out.append(
+                {
+                    "text": body,
+                    "meta": {
+                        "strategy": "markdown",
+                        "start_char": start,
+                        "heading_path": path,
+                    },
+                }
+            )
+        else:
+            out.extend(
+                _plain_windows(
+                    body,
+                    size,
+                    overlap,
+                    base_offset=start,
+                    strategy="markdown",
+                    heading_path=path,
+                )
+            )
+    return out
+
+
+def _code_segment_spans(cleaned: str) -> List[Tuple[int, int]]:
+    """Contiguous top-level segments split at blank-line runs and decl lines."""
+    boundaries = {0, len(cleaned)}
+    for match in _CODE_BLANK_RUN_RE.finditer(cleaned):
+        boundaries.add(match.end())
+    for match in _CODE_BOUNDARY_LINE_RE.finditer(cleaned):
+        boundaries.add(match.start())
+    ordered = sorted(boundaries)
+    return [
+        (ordered[i], ordered[i + 1])
+        for i in range(len(ordered) - 1)
+        if ordered[i + 1] > ordered[i]
+    ]
+
+
+def _code_chunks(cleaned: str, size: int, overlap: int) -> List[Dict[str, Any]]:
+    hard_limit = int(size * 1.5)
+    out: List[Dict[str, Any]] = []
+    pack: Optional[Tuple[int, int]] = None
+
+    def _emit(span: Tuple[int, int]) -> None:
+        out.append(
+            {
+                "text": cleaned[span[0] : span[1]],
+                "meta": {
+                    "strategy": "code",
+                    "start_char": span[0],
+                    "heading_path": None,
+                },
+            }
+        )
+
+    for start, end in _code_segment_spans(cleaned):
+        if end - start > hard_limit:
+            # Monster segment: flush the pack, then window it like plain text.
+            if pack is not None:
+                _emit(pack)
+                pack = None
+            out.extend(
+                _plain_windows(
+                    cleaned[start:end],
+                    size,
+                    overlap,
+                    base_offset=start,
+                    strategy="code",
+                )
+            )
+            continue
+        if pack is None:
+            pack = (start, end)
+        elif end - pack[0] <= size:
+            pack = (pack[0], end)
+        else:
+            _emit(pack)
+            pack = (start, end)
+    if pack is not None:
+        _emit(pack)
+    return out
+
+
+def typed_chunks(
+    text: str,
+    *,
+    strategy: str = "plain",
+    size: int = 1200,
+    overlap: int = 160,
+) -> List[Dict[str, Any]]:
+    """Strategy-aware chunking with per-chunk provenance metadata.
+
+    Returns ``[{"text": str, "meta": {"strategy", "start_char", "heading_path"}}]``
+    where ``start_char`` is the offset in ``str(text or "").strip()`` (every
+    chunk text is an exact substring at that offset).
+
+    Contract: ``[c["text"] for c in typed_chunks(t)] == _chunks(t)`` for the
+    default plain strategy — unknown strategies also fall back to plain.
+    """
+    cleaned = str(text or "").strip()
+    if not cleaned:
+        return []
+    try:
+        size = max(1, int(size))
+    except Exception:
+        size = 1200
+    try:
+        overlap = min(max(0, int(overlap)), size - 1)
+    except Exception:
+        overlap = min(160, size - 1)
+    label = strategy if strategy in _CHUNK_STRATEGIES else "plain"
+    if label == "markdown":
+        return _markdown_chunks(cleaned, size, overlap)
+    if label == "code":
+        return _code_chunks(cleaned, size, overlap)
+    return _plain_windows(cleaned, size, overlap)
+
+
+def typed_chunk_meta_fields(piece: Dict[str, Any]) -> Dict[str, Any]:
+    """Additive chunk-metadata fields for one ``typed_chunks`` piece.
+
+    Ingest call sites merge this into the existing ``{"index", "source_node"}``
+    chunk metadata; ``heading_path`` is only present when known — honest
+    absence over empty labels.
+    """
+    meta = piece.get("meta") or {}
+    fields: Dict[str, Any] = {
+        "strategy": str(meta.get("strategy") or "plain"),
+        "start_char": int(meta.get("start_char") or 0),
+    }
+    heading_path = meta.get("heading_path")
+    if heading_path:
+        fields["heading_path"] = str(heading_path)
+    return fields
+
+
+def pdf_page_offsets(structure: Any) -> List[int]:
+    """Start offset of each PDF page in the "\\n\\n"-joined page text.
+
+    ``structure`` is the ``metadata["structure"]`` dict produced by
+    ``_pdf_structure`` (``pages`` = ``[{"chars": int, ...}, ...]``); pages were
+    joined with ``"\\n\\n"`` (see ``read_document``), so page k starts at
+    ``sum(chars[j] + 2 for j < k)``. Empty or malformed input returns ``[]``.
+    """
+    if not isinstance(structure, dict):
+        return []
+    pages = structure.get("pages")
+    if not isinstance(pages, list) or not pages:
+        return []
+    offsets: List[int] = []
+    cursor = 0
+    for page in pages:
+        if not isinstance(page, dict):
+            return []
+        chars = page.get("chars")
+        if isinstance(chars, bool) or not isinstance(chars, (int, float)) or chars < 0:
+            return []
+        offsets.append(cursor)
+        cursor += int(chars) + 2  # +2 for the "\n\n" page joiner
+    return offsets
+
+
+def page_for_offset(page_offsets: List[int], offset: int) -> Optional[int]:
+    """1-based page number containing ``offset`` given page start offsets.
+
+    Returns ``None`` when ``page_offsets`` is empty or the offset precedes the
+    first page start (honest absence over a wrong label).
+    """
+    if not page_offsets:
+        return None
+    try:
+        target = int(offset)
+    except Exception:
+        return None
+    page = 0
+    for index, start in enumerate(page_offsets):
+        try:
+            if target >= int(start):
+                page = index + 1
+            else:
+                break
+        except Exception:
+            return None
+    return page if page >= 1 else None
+
+
 _LLM_EXTRACT_CONCEPT_PROMPT = """Extract the key concepts from the following text.
 Return ONLY a JSON array of objects, each with "concept" (string) and "importance" (float 0-1).
 Extract up to {limit} concepts. Focus on named entities, technical terms, and domain-specific nouns.

package/lattice_brain/graph/curator.py

@@ -412,6 +412,142 @@ def curate_nodes(
     return enriched
 
 
+# ── Noise reduction (backlog #10, review §7.2 D) ─────────────────────────────
+# Relation-verb normalization dictionary (ko/en). Keys are canonical verbs;
+# values are the free-string labels observed in the legacy edge table. The
+# canonical labels stay aligned with the v2 EdgeType legacy mapping so this
+# never fights the schema normalization.
+
+RELATION_VERB_GROUPS: Dict[str, List[str]] = {
+    "created": [
+        "creates", "create", "만들다", "만든", "만들었다", "만듦", "만들어냄",
+        "생성함", "생성", "생성했다", "작성함", "작성", "작성했다",
+    ],
+    "mentions": ["mention", "언급함", "언급", "언급했다", "언급됨"],
+    "contains": ["contain", "포함함", "포함", "포함했다", "포함됨"],
+    "uses": ["use", "used", "사용함", "사용", "사용했다", "이용함", "이용"],
+    "related_to": ["related", "relates_to", "관련", "관련됨", "관련있음", "연관됨", "연관"],
+    "fixed": ["fixes", "fix", "수정함", "수정", "수정했다", "고침", "고쳤다"],
+    "decided": ["decides", "decide", "결정함", "결정", "결정했다"],
+    "uploaded": ["uploads", "upload", "업로드함", "업로드", "올림", "올렸다"],
+}
+
+
+def build_relation_verb_index(
+    groups: Optional[Dict[str, List[str]]] = None,
+) -> Dict[str, str]:
+    """{alias(lower) → canonical} — canonical labels map to themselves."""
+    groups = groups or RELATION_VERB_GROUPS
+    index: Dict[str, str] = {}
+    for canonical, aliases in groups.items():
+        canon = str(canonical).strip().lower()
+        index[canon] = canon
+        for alias in aliases:
+            index[str(alias).strip().lower()] = canon
+    return index
+
+
+def normalize_relation_verb(
+    verb: str,
+    *,
+    index: Optional[Dict[str, str]] = None,
+) -> str:
+    """Map a free-string edge verb to its canonical form (identity if unknown).
+
+    Korean verbs are additionally tried with the trailing josa stripped so
+    "생성함을" style variants still normalize; unknown labels pass through
+    unchanged (lossless — this function is a rename map, not a filter).
+    """
+    raw = str(verb or "").strip()
+    if not raw:
+        return raw
+    index = index if index is not None else build_relation_verb_index()
+    low = raw.lower()
+    if low in index:
+        return index[low]
+    stripped = _strip_josa(low)
+    if stripped in index:
+        return index[stripped]
+    return raw
+
+
+# v4 write-door enum labels are SCREAMING_SNAKE_CASE ASCII; those are already
+# canonical schema taxonomy and must never be rewritten by the verb dictionary.
+_V4_ENUM_LABEL_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
+
+
+def plan_relation_normalization(
+    edge_types: Iterable[str],
+    *,
+    index: Optional[Dict[str, str]] = None,
+) -> Dict[str, str]:
+    """{observed_type → canonical} for every type the dictionary changes.
+
+    Skips v4-canonical enum labels (``MENTIONS``, ``INDEXED_FROM``, …): this
+    plan targets the free-string verbs of pre-v4 rows, not the schema enum.
+    """
+    index = index if index is not None else build_relation_verb_index()
+    plan: Dict[str, str] = {}
+    for edge_type in edge_types:
+        original = str(edge_type or "")
+        if _V4_ENUM_LABEL_RE.match(original):
+            continue
+        canonical = normalize_relation_verb(original, index=index)
+        if canonical and canonical != original:
+            plan[original] = canonical
+    return plan
+
+
+def plan_concept_noise_reduction(
+    concepts: Iterable[Dict[str, Any]],
+    total_docs: int,
+    *,
+    max_df_ratio: float = 0.8,
+    min_doc_frequency: int = 1,
+    min_corpus_docs: int = 5,
+) -> Dict[str, List[Dict[str, Any]]]:
+    """Decide which heuristic concept nodes are graph noise.
+
+    ``concepts``: ``[{"id", "label", "df", "heuristic"}]`` where ``df`` is the
+    number of distinct content documents linking to the concept.
+
+    Rules (dry-run friendly — pure decision, no side effects):
+
+    * **never** flag ``heuristic=False`` nodes (explicit user-created nodes
+      are untouchable, whatever their stats);
+    * low IDF: with a corpus of at least ``min_corpus_docs`` documents, a
+      concept appearing in ``> max_df_ratio`` of them separates nothing —
+      remove;
+    * frequency floor: ``df < min_doc_frequency`` (orphaned/near-orphaned
+      auto concepts) — remove.
+    """
+    total = max(0, int(total_docs))
+    remove: List[Dict[str, Any]] = []
+    keep: List[Dict[str, Any]] = []
+    for concept in concepts:
+        entry = {
+            "id": str(concept.get("id") or ""),
+            "label": concept.get("label"),
+            "df": int(concept.get("df") or 0),
+            "heuristic": bool(concept.get("heuristic")),
+        }
+        if not entry["heuristic"]:
+            keep.append({**entry, "reason": "user_created_protected"})
+            continue
+        df = entry["df"]
+        if df < int(min_doc_frequency):
+            df_ratio = (df / total) if total else 0.0
+            remove.append({**entry, "df_ratio": round(df_ratio, 4), "reason": "below_frequency_floor"})
+            continue
+        if total >= int(min_corpus_docs):
+            df_ratio = df / total
+            if df_ratio > float(max_df_ratio):
+                remove.append({**entry, "df_ratio": round(df_ratio, 4), "reason": "low_idf_ubiquitous"})
+                continue
+        keep.append({**entry, "reason": "signal"})
+    return {"remove": remove, "keep": keep}
+
+
 # ── End-to-end helper ─────────────────────────────────────────────────────────
 
 
@@ -457,6 +593,11 @@ def auto_build_graph_overlay(
 
 
 __all__ = [
+    "RELATION_VERB_GROUPS",
+    "build_relation_verb_index",
+    "normalize_relation_verb",
+    "plan_relation_normalization",
+    "plan_concept_noise_reduction",
     "TopicCandidate",
     "PromotionDecision",
     "ThreadEdge",

package/lattice_brain/graph/discovery_index.py

@@ -474,7 +474,12 @@ class KnowledgeGraphLocalIndexMixin:
                 metadata={"source": "ocr", "workspace_id": workspace_id},
             )
 
-        for index, chunk in enumerate(_chunks(text)):
+        # Typed chunking by file extension (markdown/code/plain); plain files
+        # keep legacy _chunks boundaries and therefore identical chunk ids.
+        chunk_strategy = chunk_strategy_for(file_path.name)
+        for index, piece in enumerate(typed_chunks(text, strategy=chunk_strategy)):
+            chunk = piece["text"]
+            chunk_fields = typed_chunk_meta_fields(piece)
             chunk_id = f"chunk:{_sha256_text(f'{file_node_id}:{index}:{chunk}')[:24]}"
             self._upsert_node(
                 conn,
@@ -487,6 +492,7 @@ class KnowledgeGraphLocalIndexMixin:
                     "source_node": file_node_id,
                     "source_id": source_id,
                     "workspace_id": workspace_id,
+                    **chunk_fields,
                 },
                 owner=user_email,
                 workspace_id=workspace_id,
@@ -501,6 +507,7 @@ class KnowledgeGraphLocalIndexMixin:
                     "source_node": file_node_id,
                     "source_id": source_id,
                     "workspace_id": workspace_id,
+                    **chunk_fields,
                 },
             )
             self._upsert_edge(

package/lattice_brain/graph/documents.py

@@ -6,6 +6,117 @@ from ._kg_common import *  # noqa: F403,F401
 
 
 class KnowledgeGraphDocumentsMixin:
+    def find_documents_by_uri_prefix(self, prefix: str) -> List[Dict[str, Any]]:
+        """Content nodes whose ``metadata.source_uri`` starts with ``prefix``.
+
+        Powers removable seeded corpora (e.g. the ``demo://`` First Value Loop
+        documents): callers stamp a URI scheme at ingest time and can later
+        enumerate exactly those nodes without guessing ids.
+        """
+        prefix = str(prefix or "").strip()
+        if not prefix:
+            return []
+        nt, _ = self._read_tables()
+        documents: List[Dict[str, Any]] = []
+        with self._connect() as conn:
+            rows = conn.execute(
+                f"""
+                SELECT id, type, title, metadata_json, created_at, updated_at
+                FROM {nt}
+                WHERE json_extract(metadata_json, '$.source_uri') LIKE ? || '%'
+                  AND type NOT IN ('Source', 'Chunk')
+                ORDER BY created_at ASC, id ASC
+                """,
+                (prefix,),
+            ).fetchall()
+            for row in rows:
+                meta = _safe_loads(row["metadata_json"]) or {}
+                documents.append(
+                    {
+                        "id": row["id"],
+                        "type": row["type"],
+                        "title": row["title"],
+                        "source_uri": meta.get("source_uri"),
+                        "workspace_id": meta.get("workspace_id"),
+                        "created_at": row["created_at"],
+                        "updated_at": row["updated_at"],
+                    }
+                )
+        return documents
+
+    def delete_document_tree(self, node_id: str) -> Dict[str, Any]:
+        """Delete an ingested content node plus everything it owns.
+
+        Removes the node, its retrieval chunks (``chunks`` table + ``Chunk``
+        nodes), auto-extracted Task/Decision nodes whose ``source_node`` is
+        this document, every touching edge, the vector rows, and — when it
+        becomes orphaned — the linked ``Source`` node. Shared Concept nodes
+        are intentionally left alone (they may be cited by other content).
+        Mirrored into the v2 projection via ``_v2_delete_nodes``.
+        """
+        node_id = str(node_id or "").strip()
+        if not node_id:
+            return {"status": "skipped", "removed_nodes": 0}
+        with self._connect() as conn:
+            if not conn.execute(
+                "SELECT 1 FROM nodes WHERE id=?", (node_id,)
+            ).fetchone():
+                return {"status": "not_found", "node_id": node_id, "removed_nodes": 0}
+
+            remove_ids = {node_id}
+            # Owned children: chunks + auto-extracted semantic nodes that
+            # explicitly point back at this document via metadata.source_node.
+            for row in conn.execute(
+                """
+                SELECT id FROM nodes
+                WHERE json_extract(metadata_json, '$.source_node') = ?
+                """,
+                (node_id,),
+            ):
+                remove_ids.add(row["id"])
+            # The Source node this content was indexed from (if it ends up
+            # referenced by nothing else after the delete, drop it too).
+            source_ids = [
+                row["to_node"]
+                for row in conn.execute(
+                    "SELECT to_node FROM edges WHERE from_node=? AND type IN ('indexed_from', 'INDEXED_FROM')",
+                    (node_id,),
+                )
+            ]
+
+            ids = list(remove_ids)
+            ph = ",".join("?" * len(ids))
+            conn.execute(f"DELETE FROM chunks WHERE source_node IN ({ph})", ids)
+            conn.execute(
+                f"DELETE FROM edges WHERE from_node IN ({ph}) OR to_node IN ({ph})",
+                ids * 2,
+            )
+            conn.execute(
+                f"DELETE FROM vector_embeddings WHERE item_id IN ({ph}) OR source_node IN ({ph})",
+                ids * 2,
+            )
+            conn.execute(f"DELETE FROM nodes WHERE id IN ({ph})", ids)
+            self._v2_delete_nodes(conn, ids)
+
+            removed_sources = 0
+            for source_id in source_ids:
+                still_linked = conn.execute(
+                    "SELECT 1 FROM edges WHERE from_node=? OR to_node=? LIMIT 1",
+                    (source_id, source_id),
+                ).fetchone()
+                if still_linked:
+                    continue
+                conn.execute("DELETE FROM vector_embeddings WHERE item_id=?", (source_id,))
+                conn.execute("DELETE FROM nodes WHERE id=?", (source_id,))
+                self._v2_delete_nodes(conn, [source_id])
+                removed_sources += 1
+
+        return {
+            "status": "ok",
+            "node_id": node_id,
+            "removed_nodes": len(remove_ids) + removed_sources,
+        }
+
     def _ingest_structure_nodes(
         self,
         conn: sqlite3.Connection,