ltcai 11.5.2 → 11.7.0

package/lattice_brain/graph/_kg_common/__init__.py

@@ -1,9 +1,15 @@
-"""
-SQLite knowledge graph for Lattice AI workspace memory.
+"""The graph's *compute* vocabulary, in one importable place.
+
+This package was the shared surface of the SQLite knowledge graph: constants,
+path/hash helpers, chunking, concept and triple extraction, and the schema
+handles the write mixins composed into ``KnowledgeGraphStore``. v11.6.0 moved
+every write into ``lattice-core``, and the store went with them.
 
-The graph keeps raw event JSON, normalized node metadata, and edges in one
-portable database so it can later migrate to Neo4j/Postgres without changing
-the ingestion contract.
+What is left is the half that produces *structures*: chunk a document, pull the
+concepts and triples out of a passage, classify a node type, infer an edge verb,
+hash a file. ``POST /worker/parse``, ``POST /worker/embed`` and
+``POST /worker/extract`` are the routes that answer with them; Rust decides what
+to write.
 """
 
 # ruff: noqa: F401,F841
@@ -26,21 +32,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
 
-try:
-    from ..schema import EdgeType, KGStoreV2, NodeType, _exec_script
-except Exception:  # pragma: no cover - v2 schema is optional at import time
-    KGStoreV2 = None  # type: ignore[assignment,misc]
-    NodeType = None  # type: ignore[assignment,misc]
-    EdgeType = None  # type: ignore[assignment,misc]
-    _exec_script = None  # type: ignore[assignment]
-
 from ...embeddings import LocalEmbeddingModel
-from ..json_utils import _json, _safe_loads
-from ..runtime import get_llm_router, set_llm_router
-
-# Default read source for the graph queries: v2 reconstruction views.
-# Override with LATTICEAI_KG_READ_V2=0 to fall back to the legacy tables.
-_READ_FROM_V2_DEFAULT = os.getenv("LATTICEAI_KG_READ_V2", "1") != "0"
 
 # Static constants (projection/format versions, local-ingestion classification
 # tables, OS exclusion lists) live in ._kg_constants; re-exported here so every
@@ -96,6 +88,8 @@ from .._kg_fsutil import (  # noqa: E402,F401
     _size_limit_for_category,
     _slug,
 )
+from ..json_utils import _json, _safe_loads
+from ..runtime import get_llm_router, set_llm_router
 
 # The logic this module used to hold inline lives in three cohesive submodules
 # (v11.3.0 decomposition). Every name is re-exported below, so the twelve
@@ -172,11 +166,9 @@ __all__ = [
     "Dict",
     "EDGE_VERB",
     "ENABLE_LLM_EXTRACTION",
-    "EdgeType",
     "GRAPH_SCHEMA_VERSION",
     "Iterable",
     "Iterator",
-    "KGStoreV2",
     "LINUX_EXCLUDED_PREFIXES",
     "LOCAL_CODE_EXTENSIONS",
     "LOCAL_DOCUMENT_EXTENSIONS",
@@ -189,7 +181,6 @@ __all__ = [
     "List",
     "LocalEmbeddingModel",
     "MACOS_EXCLUDED_PREFIXES",
-    "NodeType",
     "Optional",
     "Path",
     "SENSITIVE_PATH_KEYWORDS",
@@ -214,7 +205,6 @@ __all__ = [
     "_PROSE_MIN_SPAN_RATIO",
     "_PROSE_STRONG_BOUNDARY_RE",
     "_PROSE_WEAK_BOUNDARY_RE",
-    "_READ_FROM_V2_DEFAULT",
     "_V2_WRITE_MASTER_KEY",
     "_chunks",
     "_classify_node_type",
@@ -224,7 +214,6 @@ __all__ = [
     "_current_os_type",
     "_drive_id_for_path",
     "_excluded_directory_reason",
-    "_exec_script",
     "_extract_concepts",
     "_extract_concepts_rules",
     "_extract_triples",

package/lattice_brain/ingestion/__init__.py

@@ -1,63 +1,34 @@
-"""Unified ingestion pipeline — the single write-side seam into the Knowledge Graph.
+"""Ingestion: what a source *is*, how it hashes, and how good it looks.
 
-v3.6.0 Knowledge Graph First principle: *no data source bypasses the Knowledge
-Graph and no source creates an isolated silo*. Every source — local files,
-connected folders, PDFs/Markdown/text/code, web URLs, browser tabs — is
-normalized into one :class:`IngestionItem` and pushed through one
-:meth:`IngestionPipeline.ingest` entrypoint:
+v3.6.0's Knowledge Graph First principle — *no data source bypasses the
+Knowledge Graph and no source creates an isolated silo* — is still the rule.
+The door it goes through is no longer here: v11.6.0 §Wave 2.5 made Rust the
+single writer of the Brain, so ``IngestionPipeline.ingest``, the per-source
+routing doors, the folder walk, the ``.latticeignore`` reader and the durable
+background queue moved to ``lattice-core`` / ``lattice-ingest`` /
+``lattice-jobs``.
 
-    Source → normalize → content hash → (file | text) ingest → provenance
+What stays is the vocabulary both sides share, and the compute the write path
+asks this worker for:
 
-The pipeline is deliberately thin. It owns normalization, idempotency reporting,
-provenance capture, and — crucially — routing every ingest through the shared
-``dispatch_tool`` lifecycle so ``pre_tool``/``post_tool`` hooks fire on data
-ingestion exactly as they do on tool calls. The heavy graph construction lives in
-:class:`knowledge_graph.KnowledgeGraphStore` (``ingest_document`` for files,
-``ingest_source`` for text/web), which this module composes rather than
-re-implements.
-
-Web ingestion seam
-------------------
-The graph layer never fetches or parses the web. Fetching, rendering,
-readability extraction, and parse quality are the responsibility of the
-*upstream* capture surfaces (browser extension, tools layer, MCP servers):
-they hand this module already-extracted text. :meth:`IngestionPipeline.
-ingest_web_page` is the convenience wrapper for that hand-off — it normalizes
-``(url, extracted_text)`` into an ``IngestionItem(source_type="web_url")`` and
-routes it through the exact same :meth:`IngestionPipeline.ingest` door as every
-other source. If the extracted text is bad, fix the extractor upstream; the
-pipeline will not attempt network access or HTML parsing.
-
-Folder ingestion (:meth:`IngestionPipeline.ingest_folder`) walks a local
-directory, honors a gitignore-like ``.latticeignore`` file at the root
-(blank lines, ``#`` comments, ``fnmatch`` glob patterns, ``dir/`` suffix for
-directories), always skips common noise (``.git``, ``node_modules``,
-``__pycache__``, virtualenvs, ``dist``, hidden entries by default), applies
-size/extension filters, and either ingests inline or schedules through the
-existing :class:`BackgroundIngestionQueue`.
-
-Split into cohesive submodules in v11.3.0 (no behaviour change): ``constants``
-(routing tables + gates), ``quality`` (advisory scoring), ``models`` (item and
-result), ``hashing``, ``folder_scan`` (``.latticeignore``), and the three mixins
-``routing`` / ``folders`` / ``jobs_api`` that ``pipeline`` composes into
-``IngestionPipeline``. This module re-exports every name the single file
-exposed, so ``lattice_brain.ingestion.X`` keeps working.
+* ``constants`` — the routing tables and the multi-modal gates;
+* ``models`` — :class:`IngestionItem` / :class:`IngestionResult`, now the DTOs
+  of a parse request rather than of a write;
+* ``hashing`` — ``content_hash_text`` and the file digest, which decide
+  idempotency and must produce the same bytes on both sides;
+* ``quality`` — the advisory extraction score behind ``POST /worker/parse``;
+* ``pipeline`` — the multi-modal capability probe.
 """
 
 from __future__ import annotations
 
-# The single file had no ``__all__``, so its public surface was "every module
-# global" — including the names it imported for its own use. Every re-export
-# below therefore uses the redundant-alias form: it reproduces exactly that
-# surface, and it marks each name as deliberate rather than a leftover import.
+# Re-exports use the redundant-alias form: it says "this is a re-export", not a
+# leftover import.
 #
 # Stubbing note: rebinding one of these *here* changes only this module's name.
 # The submodule that calls it holds its own reference, so a test standing in for
 # a helper patches the submodule that uses it.
 from ..gates import FeatureGate as FeatureGate
-from ..ingestion_jobs import JOB_ERRORS_CAP as JOB_ERRORS_CAP
-from ..ingestion_jobs import BackgroundIngestionJob as BackgroundIngestionJob
-from ..ingestion_jobs import BackgroundIngestionQueue as BackgroundIngestionQueue
 from ..multimodal import AUDIO_EXTENSIONS as AUDIO_EXTENSIONS
 from ..multimodal import DEFAULT_KEYFRAMES as DEFAULT_KEYFRAMES
 from ..multimodal import IMAGE_EXTENSIONS as IMAGE_EXTENSIONS
@@ -75,12 +46,8 @@ from ..multimodal import ffmpeg_available as ffmpeg_available
 from ..multimodal import image_quality_score as image_quality_score
 from ..multimodal import read_video_facts as read_video_facts
 from ..multimodal import transcribe_audio as transcribe_audio
-from ..multimodal import video_frame_dir as video_frame_dir
 from ..multimodal import video_quality_score as video_quality_score
-from ..multimodal import write_image_memory as write_image_memory
-from ..multimodal import write_video_memory as write_video_memory
 from ..quiet import quiet as quiet
-from ..runtime.hooks import dispatch_tool as dispatch_tool
 from ..utils import utc_now_iso as utc_now_iso
 from .constants import _MEMORY_NODE_TYPES as _MEMORY_NODE_TYPES
 from .constants import ALLOW_MULTIMODAL_ENV as ALLOW_MULTIMODAL_ENV
@@ -107,15 +74,10 @@ from .constants import MULTIMODAL_GATE as MULTIMODAL_GATE
 from .constants import TEXT_SOURCE_TYPES as TEXT_SOURCE_TYPES
 from .constants import VIDEO_GATE as VIDEO_GATE
 from .constants import VIDEO_SOURCE_TYPES as VIDEO_SOURCE_TYPES
-from .folder_scan import _load_latticeignore as _load_latticeignore
-from .folder_scan import _matches_ignore as _matches_ignore
-from .folders import IngestionFolderMixin as IngestionFolderMixin
 from .hashing import _file_digest as _file_digest
 from .hashing import content_hash_text as content_hash_text
-from .jobs_api import IngestionJobsMixin as IngestionJobsMixin
 from .models import IngestionItem as IngestionItem
 from .models import IngestionResult as IngestionResult
-from .pipeline import VECTOR_TICK_LIMIT as VECTOR_TICK_LIMIT
 from .pipeline import IngestionPipeline as IngestionPipeline
 from .quality import _BOILERPLATE_LINE_MARKERS as _BOILERPLATE_LINE_MARKERS
 from .quality import _CAPTURE_REASON_LABELS as _CAPTURE_REASON_LABELS
@@ -127,4 +89,3 @@ from .quality import QUALITY_LOW_WARNING as QUALITY_LOW_WARNING
 from .quality import _quality_level as _quality_level
 from .quality import assess_extraction_quality as assess_extraction_quality
 from .quality import capture_quality_verdict as capture_quality_verdict
-from .routing import IngestionRoutingMixin as IngestionRoutingMixin