@pentatonic-ai/ai-agent-sdk 0.9.6 → 0.10.0

package/packages/memory-engine/engine/services/l2/l2-hybridrag-proxy.py

@@ -1,2721 +0,0 @@
-#!/usr/bin/env python3
-"""
-Sequential HybridRAG Proxy — Graph-First Architecture
-
-Based on BlackRock-NVIDIA paper methodology:
-1. HybridRAG trigger (early detection)
-2. Neo4j graph search (entities/relationships first)
-3. QMD vector search (informed by graph context)
-4. Result fusion with graph-priority scoring
-
-Port: 8031 (replaces neo4j-qmd-proxy.py)
-"""
-
-import argparse
-import asyncio
-import hashlib
-import json
-import logging
-import os
-import sqlite3
-import struct
-import sys
-import time
-from contextlib import asynccontextmanager
-from datetime import datetime
-from pathlib import Path
-from typing import Any, AsyncIterator, Dict, List, Optional, Set
-
-import re
-import httpx
-import requests
-from fastapi import FastAPI, HTTPException, Request
-from neo4j import AsyncGraphDatabase
-from neo4j.time import DateTime as Neo4jDateTime, Date as Neo4jDate
-from pydantic import BaseModel
-import uvicorn
-
-try:
-    import sqlite_vec  # 0.1.9 — native KNN MATCH over packed-f32 vec0 tables
-except ImportError:
-    sqlite_vec = None  # Caller logs loudly if helpers can't load the extension
-
-# Shared embed client lives at engine/services/_shared/.
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-from _shared.embed_provider import EmbedClient  # noqa: E402
-
-
-def _serialize_neo4j_value(v: Any) -> Any:
-    """Convert neo4j-specific types to JSON-serialisable equivalents.
-
-    Without this, FastAPI's pydantic serialiser raises
-    ``PydanticSerializationError: Unable to serialize unknown type:
-    <class 'neo4j.time.DateTime'>`` when L3 graph results include nodes
-    with datetime properties (e.g. created_at on entity nodes).
-    """
-    if isinstance(v, (Neo4jDateTime, Neo4jDate)):
-        return v.iso_format()
-    if isinstance(v, dict):
-        return {k: _serialize_neo4j_value(x) for k, x in v.items()}
-    if isinstance(v, (list, tuple)):
-        return [_serialize_neo4j_value(x) for x in v]
-    return v
-
-# ---------------------------------------------------------------------------
-# Config
-# ---------------------------------------------------------------------------
-
-WORKSPACE = Path(os.environ.get("PME_WORKSPACE", Path.home() / "pentatonic"))
-
-NEO4J_URI = os.environ.get("PME_NEO4J_URI", "bolt://localhost:7687")
-
-def _load_neo4j_password() -> str:
-    """Resolve Neo4j password from env or secrets file."""
-    pw = os.environ.get("PME_NEO4J_PASSWORD", os.environ.get("NEO4J_PASSWORD"))
-    if pw:
-        return pw
-    for path in [
-        WORKSPACE / ".secrets.json",
-        Path.home() / ".pentatonic" / "workspace" / ".secrets.json",
-        Path.home() / ".pentatonic" / "secrets.json",
-    ]:
-        if path.exists():
-            try:
-                data = json.loads(path.read_text())
-                for key in ("neo4j_password", "NEO4J_PASSWORD", "neo4jPassword"):
-                    if key in data:
-                        return data[key]
-            except (json.JSONDecodeError, OSError):
-                continue
-    return "password"  # fallback default
-
-NEO4J_AUTH = ("neo4j", _load_neo4j_password())
-def _resolve_qmd_db() -> str:
-    """Resolve QMD DB path, checking common locations."""
-    env = os.environ.get("PME_QMD_DB")
-    if env:
-        return env
-    for candidate in [
-        Path.home() / ".pentatonic" / "memory" / "main.sqlite",
-        Path.home() / ".openclaw" / "memory" / "main.sqlite",
-    ]:
-        if candidate.exists():
-            return str(candidate)
-    return str(Path.home() / ".pentatonic" / "memory" / "main.sqlite")
-
-QMD_DB_PATH = _resolve_qmd_db()
-OLLAMA_URL = os.environ.get("PME_OLLAMA_URL", "http://localhost:11434/api/embeddings")
-EMBEDDING_MODEL = os.environ.get("PME_EMBED_MODEL", "nomic-embed-text")
-
-# Embedding dimension for the vec0 virtual table. Production gateway
-# (lambda-gateway.pentatonic.com/v1/embed via pentatonic-gateway provider)
-# returns NV-Embed-v2 4096-dim vectors. The vec0 schema requires the dim
-# at DDL time and writers must match — keep this in lockstep with the
-# gateway / EmbedClient config.
-EMBED_DIM = int(os.environ.get("PME_EMBED_DIM", "4096"))
-
-
-def _open_qmd_conn() -> sqlite3.Connection:
-    """Open qmd.sqlite with sqlite-vec loaded.
-
-    Falls back to a plain sqlite3 connection if the extension can't load —
-    MATCH-form queries will then fail loudly at execute time, which is the
-    right signal (loud error > silent degradation back to Python cosine).
-    Callers that only need scalar columns (chunks.path, chunks.text) work
-    fine without the extension.
-
-    ``check_same_thread=False`` is intentional: the async backfill yields
-    via ``asyncio.to_thread`` to keep /search responsive, which means the
-    connection is handed off between event-loop / thread-pool workers.
-    sqlite's default thread-safety check would otherwise reject the
-    cross-thread reuse even though only one worker touches it at a time.
-    """
-    conn = sqlite3.connect(QMD_DB_PATH, timeout=10, check_same_thread=False)
-    if sqlite_vec is None:
-        log.error("sqlite_vec module not importable — qmd vec_index unavailable")
-        return conn
-    try:
-        conn.enable_load_extension(True)
-        sqlite_vec.load(conn)
-        conn.enable_load_extension(False)
-    except Exception as e:
-        log.error(f"sqlite-vec load failed: {e} — qmd search will be degraded")
-    return conn
-
-
-def _ensure_vec_index(conn: sqlite3.Connection) -> None:
-    """Create the vec0 KNN index if not already present. Idempotent.
-
-    `distance_metric=cosine` is non-default — sqlite-vec defaults to L2
-    (Euclidean). Probe confirmed cosine returns `1 - cos_sim` as the
-    distance. The id column is a regular INTEGER PRIMARY KEY so we can
-    JOIN back to `chunks` on the row's autoinc id.
-    """
-    conn.execute(
-        f"""
-        CREATE VIRTUAL TABLE IF NOT EXISTS vec_index USING vec0(
-            id INTEGER PRIMARY KEY,
-            embedding float[{EMBED_DIM}] distance_metric=cosine
-        )
-        """
-    )
-
-# NV-Embed-v2 service (primary, 4096-dim). URL/auth/path/body/response are
-# managed by the shared EmbedClient; PME_EMBED_PROVIDER (default openai)
-# selects auth scheme (Bearer vs X-API-Key) and request shape.
-NV_EMBED_ENABLED = os.environ.get("PME_NV_EMBED_ENABLED", "true").lower() == "true"
-
-_embed: EmbedClient | None = None
-
-
-def _embed_client() -> EmbedClient:
-    """Lazily build the shared EmbedClient for L2."""
-    global _embed
-    if _embed is None:
-        _embed = EmbedClient.from_env(
-            prefix="PME_",
-            url_var="PME_NV_EMBED_URL",
-            key_var="PME_EMBED_API_KEY",
-            model_var="PME_NV_EMBED_MODEL",
-            default_url="http://localhost:8041/v1/embeddings",
-        )
-    return _embed
-
-# Sequential processing weights - OPTIMIZED FOR QUALITY
-GRAPH_PRIORITY_BOOST = 0.5  # Extra score for graph-derived results (↑ for better entity/relationship context)
-VECTOR_BASE_WEIGHT = 0.5     # Base weight for vector results (↓ balanced for accuracy over speed)
-
-# Memory tracking
-TRACKER_FILE = WORKSPACE / "memory" / "memory-tracker.jsonl"
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-log = logging.getLogger("sequential-hybridrag")
-
-
-# ---------------------------------------------------------------------------
-# Async driver + HTTP client singletons
-# ---------------------------------------------------------------------------
-#
-# v0.8.4: migrated from sync neo4j (`GraphDatabase`) to async
-# (`AsyncGraphDatabase`) and from per-call drivers to one process-wide
-# pool. Previously every handler created a fresh sync driver inline,
-# which (a) blocked the event loop on connection establishment and
-# every query, and (b) held one thread per in-flight request. Under
-# sustained ingest the threadpool exhausted and /health itself timed
-# out (the `l0/l1/l2: unreachable: ReadTimeout` we saw in prod).
-#
-# The async driver multiplexes many queries through bolt without
-# blocking the event loop; one shared pool means connection
-# establishment is amortised. fastapi lifespan handles open/close.
-
-_neo4j_driver: "AsyncGraphDatabase | None" = None
-_http_client: "httpx.AsyncClient | None" = None
-
-
-def get_neo4j_driver():
-    """Lazy module-level singleton — created at first call (which the
-    lifespan handler does at startup). Sharing across requests is the
-    documented neo4j Python-driver pattern; the driver itself is
-    thread- and task-safe."""
-    global _neo4j_driver
-    if _neo4j_driver is None:
-        _neo4j_driver = AsyncGraphDatabase.driver(NEO4J_URI, auth=NEO4J_AUTH)
-    return _neo4j_driver
-
-
-def get_http_client() -> httpx.AsyncClient:
-    """Shared async HTTP client for L4/L5/L6 fan-out and embedding-proxy
-    pass-through. Reuses TCP connections via httpx's built-in pool."""
-    global _http_client
-    if _http_client is None:
-        _http_client = httpx.AsyncClient(timeout=30.0)
-    return _http_client
-
-
-@asynccontextmanager
-async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
-    """Open the neo4j driver + HTTP client at process startup, close on
-    shutdown. Without this, the first request pays driver-open latency
-    and the driver is never properly closed on SIGTERM (leaking conns).
-
-    Also schedules the vec_index backfill as a background task so the
-    proxy can start serving immediately while older chunks copy across
-    into the KNN index — first-time migration of ~450k rows takes
-    minutes and would otherwise block /health.
-    """
-    global _neo4j_driver, _http_client
-    _neo4j_driver = AsyncGraphDatabase.driver(NEO4J_URI, auth=NEO4J_AUTH)
-    _http_client = httpx.AsyncClient(timeout=30.0)
-    backfill_task = asyncio.create_task(_backfill_vec_index())
-    try:
-        yield
-    finally:
-        backfill_task.cancel()
-        try:
-            await backfill_task
-        except (asyncio.CancelledError, Exception):
-            pass
-        if _neo4j_driver is not None:
-            await _neo4j_driver.close()
-            _neo4j_driver = None
-        if _http_client is not None:
-            await _http_client.aclose()
-            _http_client = None
-
-
-async def _backfill_vec_index() -> None:
-    """One-time migration: copy existing chunks.embedding (JSON) into
-    vec_index (f32 bytes).
-
-    Idempotent. Runs at proxy startup if vec_index has fewer rows than
-    chunks. Async so it doesn't block /health — the proxy serves
-    requests in parallel and search degrades to partial-corpus results
-    until the backfill finishes (any chunk already mirrored into
-    vec_index is findable; the rest are invisible to search but still
-    in L0/L3/L5/L6).
-
-    At 450k rows + ~460 rows/s insert rate this takes ~16 min on a
-    cold prod instance. Subsequent restarts no-op cleanly.
-    """
-    if sqlite_vec is None:
-        log.error("sqlite_vec module missing — backfill skipped, search will be degraded")
-        return
-    if not os.path.exists(QMD_DB_PATH):
-        log.info("vec_index backfill skipped — qmd.sqlite does not exist yet")
-        return
-    try:
-        conn = await asyncio.to_thread(_open_qmd_conn)
-        await asyncio.to_thread(_ensure_vec_index, conn)
-        chunks_n = conn.execute(
-            "SELECT count(*) FROM chunks WHERE embedding IS NOT NULL"
-        ).fetchone()[0]
-        vec_n = conn.execute("SELECT count(*) FROM vec_index").fetchone()[0]
-        if vec_n >= chunks_n:
-            log.info(f"vec_index backfill skipped — already in sync ({vec_n}/{chunks_n})")
-            conn.close()
-            return
-        missing = chunks_n - vec_n
-        log.info(f"vec_index backfill starting — {missing} rows to copy")
-        cursor = conn.execute(
-            """
-            SELECT c.id, c.embedding
-            FROM chunks c
-            LEFT JOIN vec_index v ON v.id = c.id
-            WHERE v.id IS NULL AND c.embedding IS NOT NULL
-            """
-        )
-        BATCH = 500
-        copied = 0
-        while True:
-            batch = await asyncio.to_thread(cursor.fetchmany, BATCH)
-            if not batch:
-                break
-            def _insert_batch() -> int:
-                inserted = 0
-                with conn:
-                    for cid, emb_json in batch:
-                        try:
-                            vec = json.loads(emb_json)
-                        except Exception:
-                            continue
-                        if len(vec) != EMBED_DIM:
-                            continue
-                        conn.execute(
-                            "INSERT INTO vec_index(id, embedding) VALUES (?, ?)",
-                            (cid, struct.pack(f"{len(vec)}f", *vec)),
-                        )
-                        inserted += 1
-                return inserted
-            copied += await asyncio.to_thread(_insert_batch)
-            log.info(f"vec_index backfill progress: {copied}/{missing}")
-            # Yield generously so /search + writers aren't starved.
-            await asyncio.sleep(0)
-        log.info(f"vec_index backfill done — {copied} rows copied")
-        conn.close()
-    except asyncio.CancelledError:
-        log.info("vec_index backfill cancelled during shutdown")
-        raise
-    except Exception as e:
-        log.error(f"vec_index backfill failed: {e}")
-
-
-app = FastAPI(title="Sequential HybridRAG Proxy", version="1.0.0", lifespan=lifespan)
-
-# ---------------------------------------------------------------------------
-# Memory Usage Tracking
-# ---------------------------------------------------------------------------
-
-def log_memory_usage(query: str, layers_hit: List[str], l1_hits: int = 0,
-                    l3_hits: int = 0, l4_hits: int = 0, search_time_ms: float = 0.0,
-                    entities_extracted: List[str] = None) -> None:
-    """Log memory layer usage for evolution tracking."""
-    try:
-        TRACKER_FILE.parent.mkdir(parents=True, exist_ok=True)
-
-        # L2 HybridRAG is active whenever L3 or L4 are used
-        l2_active = 1 if (l3_hits > 0 or l4_hits > 0) else 0
-
-        event = {
-            "timestamp": datetime.utcnow().isoformat() + "Z",
-            "query": query[:100],  # Truncate for privacy
-            "source": "sequential-hybridrag",
-            "layers_hit": layers_hit,
-            "l1_system_hits": l1_hits,
-            "l2_hybridrag_active": l2_active,
-            "l3_graph_hits": l3_hits,
-            "l4_vector_hits": l4_hits,
-            "total_hits": l1_hits + l3_hits + l4_hits,
-            "search_time_ms": round(search_time_ms, 1),
-            "entities_extracted": entities_extracted or [],
-            "entity_count": len(entities_extracted or [])
-        }
-
-        with open(TRACKER_FILE, "a", encoding="utf-8") as f:
-            f.write(json.dumps(event) + "\n")
-
-    except Exception as e:
-        log.warning(f"Memory tracking failed: {e}")
-
-def get_layer_stats() -> Dict:
-    """Get usage statistics by layer."""
-    if not TRACKER_FILE.exists():
-        return {"error": "No tracking data found"}
-
-    try:
-        layer_counts = {"L1": 0, "L2": 0, "L3": 0, "L4": 0}
-        total_queries = 0
-        recent_queries = 0
-
-        with open(TRACKER_FILE, "r", encoding="utf-8") as f:
-            for line in f:
-                try:
-                    event = json.loads(line.strip())
-                    if event.get("source") == "sequential-hybridrag":
-                        total_queries += 1
-
-                        # Count layer hits
-                        layers = event.get("layers_hit", [])
-                        if "system" in layers or event.get("l1_system_hits", 0) > 0:
-                            layer_counts["L1"] += 1
-
-                        # L2 HybridRAG orchestration
-                        if event.get("l2_hybridrag_active", 0) > 0:
-                            layer_counts["L2"] += 1
-
-                        if "graph" in layers or event.get("l3_graph_hits", 0) > 0:
-                            layer_counts["L3"] += 1
-                        if "vector" in layers or event.get("l4_vector_hits", 0) > 0:
-                            layer_counts["L4"] += 1
-
-                        # Count recent (last 24h)
-                        event_time = datetime.fromisoformat(event["timestamp"].replace("Z", "+00:00"))
-                        if (datetime.now(event_time.tzinfo) - event_time).days < 1:
-                            recent_queries += 1
-                except Exception as e:
-                    logging.debug(f"Suppressed: {e}")
-
-        return {
-            "total_queries": total_queries,
-            "recent_24h": recent_queries,
-            "layer_usage": layer_counts,
-            "layer_percentages": {
-                f"L{i}": round(count / max(total_queries, 1) * 100, 1)
-                for i, count in enumerate(layer_counts.values(), 1)
-            } if total_queries > 0 else {}
-        }
-
-    except Exception as e:
-        return {"error": str(e)}
-
-# ---------------------------------------------------------------------------
-# Models
-# ---------------------------------------------------------------------------
-
-class ChatMessage(BaseModel):
-    role: str
-    content: str
-
-class ChatCompletionRequest(BaseModel):
-    messages: List[ChatMessage]
-    model: str = "gpt-3.5-turbo"
-    max_tokens: int = 1000
-    temperature: float = 0.1
-    # Optional tenant scope. When absent, the L3 graph layer returns no
-    # results (rather than walking the global graph) — other layers
-    # still respond, so the call succeeds but with reduced L3 context.
-    # Existing single-tenant callers (benchmarks, dev) keep working.
-    arena: Optional[str] = None
-    arenas: Optional[List[str]] = None
-
-class EmbeddingRequest(BaseModel):
-    input: Any
-    model: str = EMBEDDING_MODEL
-
-# ---------------------------------------------------------------------------
-# HybridRAG Processing Functions
-# ---------------------------------------------------------------------------
-
-def extract_query_entities(query: str) -> List[str]:
-    """Extract potential entities from query (early detection)."""
-    import re
-    # Strip punctuation from words
-    words = [re.sub(r'[^\w\s-]', '', w).strip() for w in query.split()]
-    words = [w for w in words if w]
-    potential_entities = []
-    stop_words = {'what', 'who', 'where', 'when', 'how', 'does', 'did', 'the', 'and', 'for', 'with', 'from', 'about', 'this', 'that'}
-
-    # Look for capitalized words (proper nouns)
-    for word in words:
-        if word.istitle() and len(word) > 2 and word.lower() not in stop_words:
-            potential_entities.append(word)
-
-    # Look for multi-word entities (title case phrases)
-    for i in range(len(words) - 1):
-        if words[i].istitle() and words[i+1].istitle() and words[i].lower() not in stop_words:
-            potential_entities.append(f"{words[i]} {words[i+1]}")
-
-    log.info(f"Extracted entities: {potential_entities}")
-    return potential_entities
-
-async def _hebbian_strengthen(session, arenas: List[str], node_names: List[str], increment: float = 0.05) -> None:
-    """Hebbian: strengthen edges between co-accessed nodes during query.
-
-    Scoped by arena so a search inside tenant A can't reinforce edges
-    inside tenant B's graph (which would happen via shared entity-name
-    nodes pre-arena). When `arenas` is empty (single-tenant local dev,
-    benchmarks) we no-op rather than risk a cross-tenant write.
-    """
-    if len(node_names) < 2 or not arenas:
-        return
-    now = datetime.utcnow().isoformat() + "Z"
-    for i, n1 in enumerate(node_names):
-        for n2 in node_names[i+1:]:
-            try:
-                await session.run(
-                    """MATCH (a {name: $n1})-[r]-(b {name: $n2})
-                       WHERE a.arena IN $arenas AND b.arena IN $arenas
-                       SET r.weight = coalesce(r.weight, 1.0) + $inc,
-                           r.last_accessed = $now""",
-                    n1=n1, n2=n2, arenas=arenas, inc=increment, now=now
-                )
-            except Exception:
-                pass  # non-critical
-
-
-async def search_neo4j_sequential(query: str, entities: List[str], arenas: List[str], limit: int = 12) -> Dict:
-    """Phase 1: Neo4j graph search with spreading activation + Hebbian.
-
-    `arenas` is the tenant-scope set the caller is authorised for —
-    typically [clientId] or [clientId, clientId:userId]. Every Cypher
-    clause filters on `n.arena IN $arenas`, so a search from tenant A
-    can never traverse into entity nodes belonging to tenant B even
-    when their names collide. Empty `arenas` short-circuits to no
-    results — that's safer than walking the entire graph in dev/test.
-    """
-    if not arenas:
-        log.warning("search_neo4j_sequential called without arenas — returning empty results")
-        return {"results": [], "graph_entities": [], "entity_count": 0}
-    try:
-        driver = get_neo4j_driver()
-        results = []
-        graph_entities = set()
-
-        async with driver.session() as session:
-            # Search for specific entities — use weighted spreading activation
-            for entity in entities:
-                # Direct match first — arena-scoped on every node we touch.
-                cypher = """
-                MATCH (n)
-                WHERE n.name CONTAINS $entity AND n.arena IN $arenas
-                OPTIONAL MATCH (n)-[r]-(connected)
-                WHERE connected.arena IN $arenas
-                  AND coalesce(r.weight, 1.0) >= 0.2
-                RETURN n, r, connected, $entity as search_entity,
-                       coalesce(r.weight, 1.0) AS edge_weight
-                ORDER BY edge_weight DESC
-                LIMIT $limit
-                """
-
-                result = await session.run(cypher, entity=entity, arenas=arenas, limit=8)
-
-                async for record in result:
-                    node = _serialize_neo4j_value(dict(record["n"]))
-                    rel = record["r"]
-                    connected = record["connected"]
-                    search_entity = record["search_entity"]
-                    edge_weight = record["edge_weight"]
-
-                    context = f"Entity: {node.get('name', 'Unknown')} (type: {node.get('type', 'Unknown')})"
-                    if rel and connected:
-                        rel_type = type(rel).__name__ if rel else "CONNECTED_TO"
-                        connected_dict = _serialize_neo4j_value(dict(connected)) if connected else {}
-                        context += f" → {rel_type} → {connected_dict.get('name', 'Unknown')}"
-                        if edge_weight != 1.0:
-                            context += f" [weight: {edge_weight:.2f}]"
-
-                    if 'source_file' in node:
-                        context += f"\nSource: {node['source_file']}"
-
-                    name = node.get('name', '')
-                    graph_entities.add(name)
-
-                    # Score boosted by edge weight (spreading activation)
-                    score = min(0.95, 0.7 + (edge_weight * 0.1))
-
-                    results.append({
-                        "path": f"neo4j://entity/{search_entity}",
-                        "text": context,
-                        "score": score,
-                        "source": "graph",
-                        "entity": search_entity,
-                        "node_data": node
-                    })
-
-                # 2-hop spreading activation for high-weight paths.
-                # Every node along the walk must be in-arena. Without
-                # the filter, an activation could walk into another
-                # tenant's graph via a name-collision on the start node.
-                if entity:
-                    activation_results = await session.run("""
-                        MATCH (start)-[r1]-(mid)-[r2]-(end)
-                        WHERE start.name CONTAINS $entity
-                          AND start.arena IN $arenas
-                          AND mid.arena IN $arenas
-                          AND end.arena IN $arenas
-                          AND coalesce(r1.weight, 1.0) >= 0.5
-                          AND coalesce(r2.weight, 1.0) >= 0.5
-                          AND start <> end
-                        RETURN end,
-                             coalesce(r1.weight, 1.0) * coalesce(r2.weight, 1.0) AS activation,
-                             mid.name AS via
-                        ORDER BY activation DESC
-                        LIMIT 5
-                    """, entity=entity, arenas=arenas)
-
-                    async for rec in activation_results:
-                        end_node = _serialize_neo4j_value(dict(rec["end"])) if rec["end"] else {}
-                        name = end_node.get("name", "")
-                        if name and name not in graph_entities:
-                            graph_entities.add(name)
-                            results.append({
-                                "path": f"neo4j://activation/{entity}/{name}",
-                                "text": f"Activated: {name} (via {rec['via']}, activation: {rec['activation']:.3f})",
-                                "score": min(0.85, 0.5 + rec["activation"] * 0.05),
-                                "source": "graph",
-                                "entity": entity,
-                                "node_data": end_node
-                            })
-
-            # General query search if no specific entities found —
-            # arena-gated so the fallback can't walk other tenants'
-            # nodes when the heuristic entity extractor returned nothing.
-            if not results:
-                general_words = [w for w in query.split() if len(w) > 3 and w.lower() not in ['what', 'who', 'where', 'when', 'how']]
-
-                for word in general_words[:2]:
-                    cypher = """
-                    MATCH (n)
-                    WHERE n.arena IN $arenas
-                      AND ANY(prop IN keys(n) WHERE n[prop] IS :: STRING AND n[prop] CONTAINS $term)
-                    OPTIONAL MATCH (n)-[r]-(connected)
-                    WHERE connected.arena IN $arenas
-                    RETURN n, r, connected
-                    LIMIT $limit
-                    """
-
-                    result = await session.run(cypher, term=word, arenas=arenas, limit=4)
-
-                    async for record in result:
-                        node = _serialize_neo4j_value(dict(record["n"]))
-                        context = f"Related: {node}"
-                        graph_entities.add(node.get('name', ''))
-
-                        results.append({
-                            "path": f"neo4j://search/{word}",
-                            "text": context,
-                            "score": 0.7,
-                            "source": "graph",
-                            "entity": word,
-                            "node_data": node
-                        })
-
-            # Hebbian: strengthen edges between all accessed entities
-            await _hebbian_strengthen(session, arenas, list(graph_entities))
-
-        # Note: driver is a module-level singleton, do NOT close here.
-
-        return {
-            "results": results[:limit],
-            "graph_entities": list(graph_entities),
-            "entity_count": len(graph_entities)
-        }
-
-    except Exception as e:
-        log.error(f"Neo4j search failed: {e}")
-        return {"results": [], "graph_entities": [], "entity_count": 0}
-
-def get_embedding(text: str) -> List[float]:
-    """Get embedding — tries NV-Embed-v2 (4096-dim) first, falls back to Ollama."""
-    # Try NV-Embed-v2 service first via the shared EmbedClient (handles
-    # provider selection, auth scheme, path, and 401 auto-detect).
-    if NV_EMBED_ENABLED:
-        try:
-            return _embed_client().embed_one(text)
-        except Exception as e:
-            log.warning(f"NV-Embed-v2 failed, falling back to Ollama: {e}")
-
-    # Fallback to Ollama
-    try:
-        r = requests.post(OLLAMA_URL, json={"model": EMBEDDING_MODEL, "prompt": text}, timeout=30)
-        r.raise_for_status()
-        return r.json()["embedding"]
-    except Exception as e:
-        log.error(f"Embedding failed (both NV-Embed-v2 and Ollama): {e}")
-        return []
-
-
-# ---------------------------------------------------------------------------
-# HyDE — Hypothetical Document Embeddings
-# ---------------------------------------------------------------------------
-
-HYDE_MODEL = os.environ.get("PME_HYDE_MODEL", "qwen2.5:7b")
-HYDE_ENABLED = os.environ.get("PME_HYDE_ENABLED", "true").lower() == "true"
-
-def hyde_expand(query: str) -> str:
-    """Generate a hypothetical answer to the query, then concatenate with the
-    original query for richer vector search embeddings.
-    Uses a small local LLM via Ollama /api/generate."""
-    if not HYDE_ENABLED:
-        return query
-    try:
-        r = requests.post(
-            "http://localhost:11434/api/generate",
-            json={
-                "model": HYDE_MODEL,
-                "prompt": f"Answer this question in one concise sentence:\n{query}",
-                "stream": False,
-                "options": {"temperature": 0.3, "num_predict": 60},
-            },
-            timeout=15,
-        )
-        r.raise_for_status()
-        hypo = r.json().get("response", "").strip()
-        if hypo:
-            log.info(f"HyDE expansion: '{query}' → +'{hypo[:80]}…'")
-            return f"{query} {hypo}"
-    except Exception as e:
-        log.warning(f"HyDE expansion failed (falling back to raw query): {e}")
-    return query
-
-
-# ---------------------------------------------------------------------------
-# Cross-Encoder Reranking
-# ---------------------------------------------------------------------------
-
-RERANK_ENABLED = os.environ.get("PME_RERANK_ENABLED", "true").lower() == "true"
-RERANK_WINDOW = int(os.environ.get("PME_RERANK_WINDOW", "5"))
-
-def _cosine_sim(a: List[float], b: List[float]) -> float:
-    """Cosine similarity between two vectors (handles mismatched dims)."""
-    if len(a) != len(b):
-        # Truncate to shorter length — still meaningful for cosine
-        min_len = min(len(a), len(b))
-        a, b = a[:min_len], b[:min_len]
-    dot = sum(x * y for x, y in zip(a, b))
-    na = sum(x * x for x in a) ** 0.5
-    nb = sum(x * x for x in b) ** 0.5
-    return dot / (na * nb) if na and nb else 0.0
-
-def cross_encoder_rerank(query: str, results: List[Dict], top_k: int = 16) -> List[Dict]:
-    """Re-embed the query and top-N result texts, then sort by cosine
-    similarity. Acts as a lightweight cross-encoder reranker without needing
-    a dedicated reranking model."""
-    if not RERANK_ENABLED or len(results) <= top_k:
-        return results
-
-    query_emb = get_embedding(query)
-    if not query_emb:
-        return results[:top_k]
-
-    window = results[:RERANK_WINDOW]
-    scored = []
-    for r in window:
-        # Get text from 'text' or 'content' field, fallback to path
-        text_content = r.get("text") or r.get("content") or r.get("path", "")
-        if not text_content:
-            scored.append(r)
-            continue
-        r_emb = get_embedding(text_content[:512])
-        if r_emb:
-            sim = _cosine_sim(query_emb, r_emb)
-            # Blend original score (layer priority) with cosine similarity
-            blended = 0.6 * r["score"] + 0.4 * sim
-            scored.append({**r, "score": round(blended, 4), "_rerank_sim": round(sim, 4)})
-        else:
-            scored.append(r)
-
-    scored.sort(key=lambda x: x["score"], reverse=True)
-    # Append remaining results (outside rerank window) unchanged
-    remaining = results[RERANK_WINDOW:]
-    return scored[:top_k] + remaining
-
-def search_qmd_informed(query: str, graph_context: Dict, limit: int = 12) -> List[Dict]:
-    """Phase 2: QMD vector search via sqlite-vec MATCH.
-
-    Replaces the legacy Python cosine loop over JSON-serialised embeddings
-    (which also had an `ORDER BY id LIMIT 2000` bug — only the OLDEST
-    2000 rows were ever considered, so 99%+ of the corpus was invisible to
-    search at production scale). Now: native KNN over the vec0 index,
-    full-corpus top-k. Wall time at 450k rows: ~50ms native MATCH vs
-    ~15s timeout previously.
-    """
-    if not os.path.exists(QMD_DB_PATH):
-        return []
-
-    query_embedding = get_embedding(query)
-    if not query_embedding:
-        return []
-
-    enhanced_query = query
-    if graph_context["graph_entities"]:
-        enhanced_query += " " + " ".join(graph_context["graph_entities"][:3])
-    enhanced_embedding = get_embedding(enhanced_query) or query_embedding
-
-    if len(enhanced_embedding) != EMBED_DIM:
-        # Dim mismatch vs vec0 DDL — the MATCH would error inside sqlite-vec.
-        # Bail with a loud log; an embedding-model mismatch in prod is the
-        # likely root cause and silent degradation would hide it.
-        log.error(
-            f"QMD search: query dim {len(enhanced_embedding)} != vec_index dim "
-            f"{EMBED_DIM} — embedding model mismatch?"
-        )
-        return []
-    qbytes = struct.pack(f"{len(enhanced_embedding)}f", *enhanced_embedding)
-
-    try:
-        conn = _open_qmd_conn()
-        # Pull a candidate pool larger than `limit` so entity-boost
-        # re-ranking has material to work with — 4× limit, floor 50.
-        k_pool = max(limit * 4, 50)
-        rows = conn.execute(
-            """
-            SELECT c.id, c.path, c.text, v.distance
-            FROM vec_index v
-            JOIN chunks c ON c.id = v.id
-            WHERE v.embedding MATCH ? AND k = ?
-            ORDER BY v.distance
-            """,
-            (qbytes, k_pool),
-        ).fetchall()
-        conn.close()
-
-        results = []
-        for row_id, path, text, distance in rows:
-            # vec0 distance_metric=cosine returns `1 - cos_sim` —
-            # invert to align with the rest of the codebase's `similarity`
-            # convention (1.0 = identical, 0.0 = orthogonal).
-            similarity = 1.0 - distance
-            if similarity <= 0.2:
-                continue
-            entity_boost = 0
-            path_lower = (path or "").lower()
-            text_lower = (text or "").lower()
-            for entity in graph_context["graph_entities"]:
-                el = entity.lower()
-                if el in path_lower or el in text_lower:
-                    entity_boost = GRAPH_PRIORITY_BOOST
-                    break
-            final_score = (similarity * VECTOR_BASE_WEIGHT) + entity_boost
-            results.append({
-                "path": path,
-                "text": (text or "")[:600],
-                "score": final_score,
-                "source": "vector",
-                "base_similarity": similarity,
-                "entity_boost": entity_boost,
-            })
-        results.sort(key=lambda x: x["score"], reverse=True)
-        return results[:limit]
-
-    except Exception as e:
-        log.error(f"QMD search failed: {e}")
-        return []
-
-def search_core_memory_files(query: str, limit: int = 8) -> List[Dict]:
-    """L1 System Files - Section-aware search with synonym expansion (v2)."""
-    core_files = [
-        "MEMORY.md",
-        "plans.md",
-        "SESSION-STATE.md",
-        "SOUL.md",
-        "USER.md",
-        f"memory/daily/{time.strftime('%Y-%m-%d')}.md",  # Today's notes
-        f"memory/daily/{time.strftime('%Y-%m-%d', time.localtime(time.time() - 86400))}.md"  # Yesterday
-    ]
-
-    # Synonym expansion for common query terms
-    SYNONYMS = {
-        'birthday': ['birthday', 'born', 'birth', 'date'],
-        'car': ['car', 'vehicle', 'inster', 'hyundai'],
-        'wife': ['wife', 'spouse', 'partner', 'family'],
-        'budget': ['budget', 'cap', 'cost', 'monthly', 'spending'],
-        'port': ['port', 'listen', 'service', 'endpoint'],
-        'neo4j': ['neo4j', 'graph', 'knowledge'],
-        'tes': ['tes', 'thing', 'event', 'system'],  # example: add domain-specific terms
-        'phone': ['phone', 'mobile', 'number', 'contact'],
-        'password': ['password', 'secret', 'credential'],
-        'home': ['home', 'address', 'live', 'residence'],
-        'hobbies': ['hobbies', 'hobby', 'interests', 'leisure'],
-        'hobby': ['hobbies', 'hobby', 'interests', 'leisure'],
-        'interests': ['hobbies', 'hobby', 'interests', 'leisure'],
-        'patent': ['patent', 'deadline', 'filing'],  # example: legal terms
-        'deadline': ['deadline', 'patent', 'due', 'expiry'],
-        'dimensions': ['dimensions', 'embed', 'vector', 'dim', '4096'],
-        'embed': ['embed', 'embedding', 'dimensions', 'vector', 'nv'],
-    }
-
-    stop_words = {'what', 'where', 'when', 'how', 'who', 'which', 'does', 'the',
-                  'is', 'are', 'was', 'were', 'for', 'and', 'with', 'has', 'have'}
-
-    query_lower = query.lower()
-    raw_keywords = [w for w in re.findall(r'\w+', query_lower) if len(w) > 2 and w not in stop_words]
-
-    # Expand with synonyms
-    expanded = set(raw_keywords)
-    for k in raw_keywords:
-        if k in SYNONYMS:
-            expanded.update(SYNONYMS[k])
-    keywords = list(expanded)
-
-    results = []
-
-    for file_path in core_files:
-        try:
-            full_path = WORKSPACE / file_path
-            if not full_path.exists():
-                continue
-            content = full_path.read_text(encoding='utf-8', errors='ignore')
-
-            # Split into sections by ## headers for granular matching
-            sections = re.split(r'\n(?=##\s)', content)
-
-            for section in sections:
-                lines = section.strip().split('\n')
-                header = lines[0] if lines and lines[0].startswith('#') else ""
-                section_lower = section.lower()
-                header_lower = header.lower()
-
-                # Score 1: Header keyword match (3x weight — header match is very precise)
-                header_score = sum(k in header_lower for k in keywords) * 3
-
-                # Score 2: Content keyword hits
-                content_score = sum(section_lower.count(k) for k in keywords)
-
-                # Score 3: Exact answer patterns (IPs, ports, dates, reg plates)
-                exact_patterns = re.findall(
-                    r'\b(?:\d{1,3}\.){3}\d{1,3}\b'   # IP addresses
-                    r'|\b\d{4,5}\b'                    # port numbers
-                    r'|[A-Z]{2}\d{2}[A-Z]{3}'         # UK reg plates
-                    r'|\b\d{1,2}\s+(?:January|February|March|April|May|June|July|'
-                    r'August|September|October|November|December)\b',  # dates
-                    section
-                )
-                exact_score = len(exact_patterns)
-
-                total_score = header_score + content_score + exact_score
-
-                if total_score > 0:
-                    results.append({
-                        "path": file_path,
-                        "text": section[:600],
-                        "score": 1.0,  # HIGHEST PRIORITY
-                        "source": "core_memory",
-                        "keyword_hits": total_score,
-                        "file_type": "core",
-                        "layer": "L1_system",
-                    })
-
-        except Exception as e:
-            log.warning(f"Failed to read core file {file_path}: {e}")
-            continue
-
-    # Sort by total score, deduplicate by path+header prefix
-    results.sort(key=lambda x: x["keyword_hits"], reverse=True)
-    seen = set()
-    deduped = []
-    for r in results:
-        key = r["text"][:50]
-        if key not in seen:
-            seen.add(key)
-            deduped.append(r)
-
-    return deduped[:limit]
-
-def extract_relevant_snippet(content: str, keywords: List[str], max_chars: int = 400) -> str:
-    """Extract most relevant snippet from content around keywords."""
-    content_lower = content.lower()
-    best_pos = 0
-    best_score = 0
-
-    # Find position with highest keyword density
-    for i in range(0, len(content), 100):
-        window = content_lower[i:i + max_chars]
-        score = sum(window.count(keyword) for keyword in keywords)
-        if score > best_score:
-            best_score = score
-            best_pos = i
-
-    # Extract snippet around best position
-    start = max(0, best_pos - 50)
-    end = min(len(content), best_pos + max_chars + 50)
-    snippet = content[start:end].strip()
-
-    # Clean up to sentence boundaries if possible
-    if '. ' in snippet:
-        sentences = snippet.split('. ')
-        # Keep middle sentences (most likely to be complete)
-        if len(sentences) > 2:
-            snippet = '. '.join(sentences[1:-1]) + '.'
-
-    return snippet
-
-# ---------------------------------------------------------------------------
-# L0: Native BM25 Workspace Memory Search
-# ---------------------------------------------------------------------------
-
-L0_MEMORY_DB = Path(os.environ.get(
-    "PME_MEMORY_DB",
-    str(Path.home() / ".pentatonic" / "memory" / "main.sqlite"),
-))
-
-def search_l0_bm25(query: str, limit: int = 6, arena: str = None,
-                   arenas: List[str] = None) -> List[Dict]:
-    """Search native BM25 index over workspace memory files.
-
-    Covers chunks from daily notes, memory files, people profiles,
-    infrastructure docs, project files — corpus that L3-L6 don't index.
-    Sub-millisecond local SQLite reads, zero network overhead.
-
-    arena / arenas: when set, filter to paths under bench/<arena>/.
-    Multi-arena queries (e.g. tenant-wide + user-scoped in one search)
-    use OR'd path-prefix LIKE clauses. `arenas` wins when both are
-    supplied; `arena` is treated as a one-element list for back-compat.
-    """
-    if not L0_MEMORY_DB.exists():
-        return []
-    try:
-        # Tokenize query for FTS5 match
-        tokens = query.lower().split()
-        meaningful = [t for t in tokens if len(t) > 2 and t not in {
-            "the", "and", "for", "with", "that", "this", "from", "what",
-            "how", "does", "have", "has", "are", "was", "were", "been",
-        }]
-        if not meaningful:
-            return []
-        fts_query = " OR ".join(f'"{t}"' for t in meaningful)
-
-        # Normalize single+multi arena inputs into one list.
-        arena_list = list(arenas) if arenas else ([arena] if arena else [])
-
-        conn = sqlite3.connect(str(L0_MEMORY_DB), timeout=2)
-        conn.execute("PRAGMA journal_mode=WAL")
-        sql = """
-            SELECT path, text, bm25(chunks_fts) as rank
-            FROM chunks_fts
-            WHERE chunks_fts MATCH ?
-              AND path NOT LIKE '%/snapshots/%'
-              AND path NOT LIKE '%/archive/%'
-              AND path NOT LIKE '%-backup-%'
-        """
-        params: list = [fts_query]
-        if arena_list:
-            clauses = " OR ".join(["path LIKE ?"] * len(arena_list))
-            sql += f" AND ({clauses})"
-            params.extend([f"bench/{a}/%" for a in arena_list])
-        sql += " ORDER BY rank ASC LIMIT ?"
-        params.append(limit * 2)
-        rows = conn.execute(sql, params).fetchall()
-        conn.close()
-
-        results = []
-        seen_paths = set()
-        for path, text, rank in rows:
-            if path in seen_paths:
-                continue
-            seen_paths.add(path)
-            relevance = -rank if rank < 0 else 0.001
-            score = min(relevance / (1 + relevance) * 0.85, 0.75)
-            # Parse arena from path (bench/<arena>/...) so downstream
-            # consumers can read it directly without parsing again.
-            row_arena = ""
-            if path.startswith("bench/"):
-                parts = path.split("/", 2)
-                if len(parts) >= 3:
-                    row_arena = parts[1]
-            results.append({
-                "path": f"L0/{path}",
-                "snippet": text[:500],
-                "score": round(score, 4),
-                "layer": "L0_workspace_bm25",
-                "source": path,
-                "arena": row_arena,
-            })
-            if len(results) >= limit:
-                break
-        return results
-    except Exception as e:
-        log.debug(f"L0 BM25 search error: {e}")
-        return []
-
-
-# ---------------------------------------------------------------------------
-# L5: Communications Context Search
-# ---------------------------------------------------------------------------
-
-L5_API_URL = os.environ.get("PME_L5_URL", "http://127.0.0.1:8034")
-
-async def search_l5_communications(query: str, limit: int = 6, arena: str = None,
-                                   arenas: List[str] = None) -> List[Dict]:
-    """Search L5 Communications Context via L5 API (emails, chats, calendar).
-
-    arena / arenas (optional): forwarded to L5; filters Milvus by the
-    arena dynamic field. Multi-arena calls become a Milvus
-    `arena IN ["X","Y"]` filter expression on the L5 side.
-    """
-    try:
-        # Build a list of (key, value) tuples so multi-valued query
-        # params (?arenas=A&arenas=B) wire-shape correctly.
-        arena_list = list(arenas) if arenas else ([arena] if arena else [])
-        params: list = [("q", query), ("limit", str(limit))]
-        for a in arena_list:
-            params.append(("arenas", a))
-        resp = await get_http_client().get(
-            f"{L5_API_URL}/search",
-            params=params,
-            timeout=10,
-        )
-        if resp.status_code != 200:
-            return []
-        data = resp.json()
-        results = []
-        for hit in data.get("results", []):
-            source = hit.get("source", "")
-            score = hit.get("score", 0)
-            # Scale Milvus cosine similarity to HybridRAG range
-            # Milvus returns 0.4-0.7 for relevant results; boost to compete with other layers
-            scaled_score = round(min((score - 0.3) * 2.0 + 0.5, 0.82), 4)
-            if scaled_score < 0.4:
-                continue  # skip low relevance
-            contact = hit.get("contact", "")
-            channel = hit.get("channel", "")
-            hit_id = hit.get("id", "")
-            # Use record id as path label so the shim can attach
-            # metadata via _META_CACHE; falls back to source label
-            # for legacy records that have no id.
-            path_label = hit_id or f"L5/{source}"
-            if not hit_id and contact:
-                path_label = f"L5/{channel}/{contact}"
-            results.append({
-                "id": hit_id,
-                "path": path_label,
-                "snippet": hit.get("text", "")[:500],
-                "score": scaled_score,
-                "layer": "L5_communications",
-                "source": source,
-                "collection": hit.get("collection", ""),
-                "timestamp": hit.get("timestamp", ""),
-                "arena": hit.get("arena", ""),
-            })
-        return results
-    except Exception as e:
-        log.debug(f"L5 search error: {e}")
-        return []
-
-
-# L6: Document Store Search
-L6_URL = os.environ.get("PME_L6_URL", "http://localhost:8037")
-
-async def search_l6_documents(query: str, limit: int = 6, arena: str = None,
-                              arenas: List[str] = None) -> List[Dict]:
-    """Search L6 Document Store (research, legal, financial, project docs).
-
-    arena / arenas (optional): forwarded to L6 — L6 supports multi-arena
-    natively (see l6-document-store.py search_vector / search_fts).
-    """
-    try:
-        arena_list = list(arenas) if arenas else ([arena] if arena else [])
-        params: list = [
-            ("q", query),
-            ("method", "hybrid"),
-            ("limit", str(limit)),
-            ("rerank", "true"),
-        ]
-        for a in arena_list:
-            params.append(("arenas", a))
-        resp = await get_http_client().get(
-            f"{L6_URL}/search",
-            params=params,
-            timeout=10,
-        )
-        if resp.status_code != 200:
-            return []
-        data = resp.json()
-        results = []
-        for hit in data.get("results", []):
-            source = hit.get("source_file", "")
-            heading = hit.get("heading", "")
-            arena = hit.get("arena", "")
-            rrf = hit.get("rrf_score", 0)
-            reranker = hit.get("reranker_score", None)
-
-            # Scale score: RRF max is ~0.033, normalize to 0.0-0.82 range
-            # (below L1's 1.0, comparable to L3/L4/L5)
-            score = min(rrf * 25, 0.82)
-            if reranker is not None and reranker > 0:
-                score = min(score + 0.05, 0.85)
-
-            path_label = f"L6/{arena}/{Path(source).name}" if arena else f"L6/{Path(source).name}"
-            if heading:
-                path_label += f": {heading[:50]}"
-
-            snippet = hit.get("text", "")[:500]
-            entities = hit.get("entities", [])
-            if entities:
-                snippet += f"\n[Entities: {', '.join(entities[:5])}]"
-
-            results.append({
-                "path": f"l6:{Path(source).stem}:{hit.get('chunk_index', 0)}",
-                "snippet": snippet,
-                "score": round(score, 4),
-                "layer": "L6_documents",
-                "source": source,
-                "arena": arena,
-                "doc_type": hit.get("doc_type", ""),
-                "engines": hit.get("engines", []),
-            })
-        return results
-    except Exception as e:
-        log.debug(f"L6 search error: {e}")
-        return []
-
-
-async def sequential_hybridrag_search(query: str, limit: int = 16,
-                                      arena: str = None,
-                                      arenas: List[str] = None) -> List[Dict]:
-    """Main HybridRAG processing: L0 BM25 → L1 System Files → L2 HybridRAG (L3 Graph + L4 Vector + L5 Comms + L6 Docs).
-
-    arena / arenas (optional): tenant + user scope. Multi-arena lets a
-    user's search span tenant-wide rows + their own user-scoped rows in
-    a single hybrid pass. Forwarded to L0, L5, L6 native filters; L4
-    and L3 still rely on the compat shim post-filter.
-
-    Async since v0.8.4: independent layer fan-out runs concurrently via
-    asyncio.gather; sync workers (sqlite, file I/O, PyTorch reranker)
-    are dispatched to threads via asyncio.to_thread to keep the event
-    loop responsive under sustained ingest.
-    """
-    arena_list = list(arenas) if arenas else ([arena] if arena else [])
-    start_time = time.time()
-    log.info(f"Starting sequential HybridRAG search for: '{query}' arenas={arena_list!r}")
-
-    # L0 (sqlite) + L1 (file I/O) + entity extraction run in parallel.
-    # Each is sync — offloaded to the threadpool so the event loop stays
-    # free for the in-flight L3/L5/L6 calls.
-    l0_results, system_results, entities = await asyncio.gather(
-        asyncio.to_thread(search_l0_bm25, query, 6, None, arena_list),
-        asyncio.to_thread(search_core_memory_files, query, 4),
-        asyncio.to_thread(extract_query_entities, query),
-    )
-    log.info(f"L0 BM25 workspace: {len(l0_results)} results")
-    log.info(f"L1 System files: {len(system_results)} results")
-
-    # L3: Graph search (now native async via AsyncGraphDatabase).
-    graph_context = await search_neo4j_sequential(query, entities, arena_list, limit=8)
-    log.info(f"L3 Graph search: {len(graph_context['results'])} results, {graph_context['entity_count']} entities")
-
-    # HyDE expansion is sync (LLM call), offload to thread.
-    hyde_query = await asyncio.to_thread(hyde_expand, query)
-
-    # L4/L5/L6 fan out concurrently — L4 (sqlite) via to_thread, L5/L6
-    # native async via httpx.AsyncClient.
-    vector_results, l5_results, l6_results = await asyncio.gather(
-        asyncio.to_thread(search_qmd_informed, hyde_query, graph_context, 8),
-        search_l5_communications(hyde_query, limit=6, arenas=arena_list),
-        search_l6_documents(hyde_query, limit=6, arenas=arena_list),
-    )
-    log.info(f"L4 Vector search: {len(vector_results)} results (HyDE={'on' if hyde_query != query else 'off'})")
-    log.info(f"L5 Communications: {len(l5_results)} results")
-    log.info(f"L6 Documents: {len(l6_results)} results")
-
-    # L2: HybridRAG fusion (combines all layers with L1 priority)
-    all_results = l0_results + system_results + graph_context["results"] + vector_results + l5_results + l6_results
-
-    # Remove duplicates by path
-    seen_paths = set()
-    deduplicated = []
-    for result in all_results:
-        if result["path"] not in seen_paths:
-            deduplicated.append(result)
-            seen_paths.add(result["path"])
-
-    # Sort by layer priority: L1 System (1.0) > L3 Graph (0.9) > L4 Vector (0.7+)
-    deduplicated.sort(key=lambda x: x["score"], reverse=True)
-
-    # Cross-encoder reranking: re-embed top results and blend scores.
-    # PyTorch CrossEncoder.predict is sync — offload to thread.
-    deduplicated = await asyncio.to_thread(cross_encoder_rerank, query, deduplicated, limit)
-
-    # Track layer usage for evolution
-    search_time_ms = (time.time() - start_time) * 1000
-    layers_used = []
-    if len(l0_results) > 0:
-        layers_used.append("workspace_bm25")
-    if len(system_results) > 0:
-        layers_used.append("system")
-    if len(graph_context["results"]) > 0:
-        layers_used.append("graph")
-    if len(vector_results) > 0:
-        layers_used.append("vector")
-    if len(l5_results) > 0:
-        layers_used.append("communications")
-    if len(l6_results) > 0:
-        layers_used.append("documents")
-
-    log_memory_usage(
-        query=query,
-        layers_hit=layers_used,
-        l1_hits=len(system_results),
-        l3_hits=len(graph_context["results"]),
-        l4_hits=len(vector_results),
-        search_time_ms=search_time_ms,
-        entities_extracted=entities
-    )
-
-    log.info(f"L2 HybridRAG final: {len(deduplicated[:limit])} total results")
-    return deduplicated[:limit]
-
-# ---------------------------------------------------------------------------
-# API Endpoints
-# ---------------------------------------------------------------------------
-
-@app.post("/v1/search")
-async def search_endpoint(request: Request) -> dict:
-    """Direct L1→L3→L4 cascade search. Returns structured results for programmatic use."""
-    try:
-        body = await request.json()
-        query = body.get("query", "")
-        limit = body.get("limit", 16)
-        arena = body.get("arena") or None
-        arenas = body.get("arenas") or None
-        if not query:
-            raise HTTPException(status_code=400, detail="query is required")
-
-        results = await sequential_hybridrag_search(query, limit=limit, arena=arena, arenas=arenas)
-
-        # Also return raw graph entities for context enrichment.
-        # Same arena scope as the cascade search above — without it
-        # the entities returned could include cross-tenant rows.
-        arena_list = list(arenas) if arenas else ([arena] if arena else [])
-        entities = extract_query_entities(query)
-        graph_context = await search_neo4j_sequential(query, entities, arena_list, limit=8)
-
-        return {
-            "results": results,
-            "entities": entities,
-            "graph_nodes": graph_context.get("entity_count", 0),
-            "graph_entities": graph_context.get("graph_entities", []),
-            "layers_active": {
-                "L1_system": True,
-                "L3_graph": True,
-                "L4_vector": True,
-                "L5_communications": True
-            }
-        }
-    except HTTPException:
-        raise
-    except Exception as e:
-        log.error(f"Search endpoint error: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@app.get("/v1/models")
-async def list_models() -> dict:
-    """OpenAI-compatible model listing."""
-    return {
-        "object": "list",
-        "data": [
-            {"id": "gpt-3.5-turbo", "object": "model", "owned_by": "sequential-hybridrag"}
-        ]
-    }
-
-@app.post("/v1/embeddings")
-async def create_embeddings(request: EmbeddingRequest) -> dict:
-    """Pass-through to NV-Embed-v2 (4096-dim). Batch-native — forwards the full
-    input list in a single HTTP call instead of looping one-at-a-time.
-
-    Returns OpenAI-shaped response regardless of upstream provider, so
-    callers (including L4 search and external clients) get a consistent
-    contract from this proxy."""
-    try:
-        inputs = [request.input] if isinstance(request.input, str) else request.input
-        embeddings = await _embed_client().embed_batch_async(inputs)
-        return {
-            "object": "list",
-            "model": request.model or "nv-embed-v2",
-            "data": [
-                {"object": "embedding", "embedding": e, "index": i}
-                for i, e in enumerate(embeddings)
-            ],
-            "usage": {"prompt_tokens": 0, "total_tokens": 0},
-        }
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.post("/v1/chat/completions")
-async def chat_completions(request: ChatCompletionRequest) -> dict:
-    """Sequential HybridRAG memory search."""
-    try:
-        # Extract query from last user message
-        user_messages = [m for m in request.messages if m.role == "user"]
-        if not user_messages:
-            raise HTTPException(status_code=400, detail="No user message found")
-
-        query = user_messages[-1].content
-
-        # Perform sequential HybridRAG search — pass through tenant
-        # scope from the request so L3 graph traversal stays inside the
-        # caller's arena. The search function short-circuits L3 to
-        # empty when no arenas are supplied; callers that need L3 must
-        # pass `arena` or `arenas` on the request body.
-        start_time = time.time()
-        results = await sequential_hybridrag_search(
-            query, limit=16, arena=request.arena, arenas=request.arenas,
-        )
-        search_time = time.time() - start_time
-
-        # Format results with correct layer structure
-        context_parts = []
-        system_count = sum(1 for r in results if r["source"] == "core_memory")
-        graph_count = sum(1 for r in results if r["source"] == "graph")
-        vector_count = sum(1 for r in results if r["source"] == "vector")
-
-        context_parts.append(f"# HybridRAG Results (L1 System → L2 HybridRAG → L3 Graph → L4 Vector)")
-        context_parts.append(f"Query: {query}")
-        context_parts.append(f"Results: {system_count} system + {graph_count} graph + {vector_count} vector = {len(results)} total")
-        context_parts.append(f"Search time: {search_time:.3f}s")
-        context_parts.append("")
-
-        current_tier = None
-        for i, result in enumerate(results):
-            # Group by layer for clarity
-            source = result['source']
-            if source != current_tier:
-                if source == "core_memory":
-                    context_parts.append(f"## L1 SYSTEM FILES (Highest Priority)")
-                elif source == "graph":
-                    context_parts.append(f"## L3 GRAPH SEARCH (via L2 HybridRAG)")
-                elif source == "vector":
-                    context_parts.append(f"## L4 VECTOR SEARCH (via L2 HybridRAG)")
-                context_parts.append("")
-                current_tier = source
-
-            context_parts.append(f"**{result['path']}** (score: {result['score']:.3f})")
-            context_parts.append("")
-            context_parts.append(result['text'][:800])
-            context_parts.append("")
-
-        response_content = "\n".join(context_parts) if context_parts else "No relevant context found."
-
-        return {
-            "id": f"seq-hybridrag-{int(time.time())}",
-            "object": "chat.completion",
-            "choices": [{
-                "message": {
-                    "role": "assistant",
-                    "content": response_content
-                },
-                "index": 0,
-                "finish_reason": "stop"
-            }],
-            "usage": {"total_tokens": len(response_content)},
-            "model": request.model
-        }
-
-    except Exception as e:
-        log.error(f"HybridRAG search failed: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.get("/contradictions/{node_name}")
-async def check_contradictions(node_name: str, arena: Optional[str] = None) -> dict:
-    """Detect contradictions around a named node.
-
-    `arena` is required to scope the lookup to one tenant's graph. The
-    endpoint returns a 400 when called without it — silently spanning
-    the entire graph here would leak entity names across tenants via
-    the `node_name` lookup.
-    """
-    if not arena:
-        raise HTTPException(
-            status_code=400,
-            detail="arena query parameter is required to scope contradiction lookup",
-        )
-    try:
-        driver = get_neo4j_driver()
-        contradictions = []
-        async with driver.session() as session:
-            # Find the node — must be in the caller's arena.
-            result = await session.run(
-                """MATCH (n) WHERE toLower(n.name) = toLower($name) AND n.arena = $arena
-                   RETURN elementId(n) AS id""",
-                name=node_name, arena=arena,
-            )
-            node = await result.single()
-            if not node:
-                return {"node": node_name, "contradictions": [], "error": "Node not found"}
-            nid = node["id"]
-
-            # Explicit CONTRADICTS — both endpoints must be in the same arena.
-            result = await session.run(
-                """MATCH (a)-[r:CONTRADICTS]-(b)
-                   WHERE elementId(a) = $nid AND b.arena = $arena
-                   RETURN a.name AS a, b.name AS b, r.reason AS reason""",
-                nid=nid, arena=arena,
-            )
-            async for rec in result:
-                contradictions.append({"type": "explicit", "a": rec["a"], "b": rec["b"], "reason": rec["reason"]})
-
-            # Property conflicts via shared neighbour — every node along
-            # the (a)--(shared)--(b) path filtered by arena so a shared
-            # neighbour from another tenant can't trigger a false-positive
-            # conflict in this tenant's view.
-            result = await session.run(
-                """MATCH (a)--(shared)--(b)
-                   WHERE elementId(a) = $nid AND a <> b
-                     AND shared.arena = $arena AND b.arena = $arena
-                   WITH a, b, shared, properties(a) AS pa, properties(b) AS pb
-                   WITH a, b, shared,
-                        [k IN keys(pa) WHERE k IN keys(pb) AND pa[k] <> pb[k]
-                         AND NOT k IN ['last_accessed','embedding','created_at','updated_at','id','weight']] AS ck
-                   WHERE size(ck) > 0
-                   RETURN a.name AS a, b.name AS b, shared.name AS via, ck
-                   LIMIT 10""", nid=nid, arena=arena,
-            )
-            async for rec in result:
-                contradictions.append({
-                    "type": "property_conflict", "a": rec["a"], "b": rec["b"],
-                    "via": rec["via"], "conflicting_keys": rec["ck"]
-                })
-        return {"node": node_name, "contradictions": contradictions, "count": len(contradictions)}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-async def _check_l5_health() -> bool:
-    """Quick check if L5 Communications API is responding."""
-    try:
-        resp = await get_http_client().get(f"{L5_API_URL}/health", timeout=3)
-        return resp.status_code == 200
-    except Exception:
-        return False
-
-async def _check_l6_health() -> bool:
-    """Quick check if L6 Document Store is responding."""
-    try:
-        resp = await get_http_client().get(f"{L6_URL}/health", timeout=3)
-        return resp.status_code == 200 and resp.json().get("status") in ("ok", "degraded")
-    except Exception:
-        return False
-
-@app.get("/health")
-async def health() -> dict:
-    """System health check.
-
-    Reports "ok" iff every layer L2 directly owns is healthy: L0 BM25
-    (SQLite FTS5 file), L4 QMD vector store (sqlite file), and the
-    Neo4j connection. L5/L6 reachability is reported informationally
-    only — the compat shim probes them directly. Ollama is no longer
-    a hard dependency anywhere; the engine uses the configured
-    NV_EMBED_URL via _embed_post helpers in each layer.
-    """
-    qmd_healthy = os.path.exists(QMD_DB_PATH)
-    l0_healthy = L0_MEMORY_DB.exists()
-
-    neo4j_healthy = False
-    try:
-        driver = get_neo4j_driver()
-        async with driver.session() as session:
-            await session.run("RETURN 1")
-        neo4j_healthy = True
-    except Exception as e:
-        logging.debug(f"Suppressed: {e}")
-
-    l5_reachable = await _check_l5_health()
-    l6_reachable = await _check_l6_health()
-
-    # Top-level status: degrade only on layers L2 is the sole gatekeeper for.
-    # L5/L6 are independent services probed by the compat shim.
-    must_be_ok = [l0_healthy, qmd_healthy, neo4j_healthy]
-    overall = "ok" if all(must_be_ok) else "degraded"
-
-    return {
-        "status": overall,
-        "proxy": "healthy",
-        "architecture": "sequential-hybridrag-proper-layers",
-        "layers": {
-            "L0_workspace_bm25": {"status": "healthy" if l0_healthy else "unavailable", "backend": "sqlite-fts5"},
-            "L1_system_files": {"status": "healthy", "description": "MEMORY.md, plans.md, daily notes"},
-            "L2_hybridrag": {"status": "healthy", "description": "Orchestrates L3+L4 fusion"},
-            "L3_graph_search": {"status": "healthy" if neo4j_healthy else "unavailable", "backend": "neo4j"},
-            "L4_vector_search": {"status": "healthy" if qmd_healthy else "unavailable", "backend": "qmd"},
-            "L5_communications": {"status": "healthy" if l5_reachable else "unavailable", "backend": "milvus"},
-            "L6_document_store": {"status": "healthy" if l6_reachable else "unavailable", "backend": "milvus+fts5+reranker", "port": 8037},
-        }
-    }
-
-@app.get("/stats")
-async def layer_statistics() -> Any:
-    """Memory layer usage statistics for evolution tracking."""
-    return get_layer_stats()
-
-# ---------------------------------------------------------------------------
-# Internal write endpoints (L0 BM25 + L4 QMD + L3 KG)
-#
-# These let the compat shim populate the three layers that L2 reads from
-# directly (rather than via HTTP sidecars). Without these the L2-via-shim
-# path runs with empty L0/L4-qmd/L3 and RRF fusion is polluted by zero-
-# result rank lists.
-# ---------------------------------------------------------------------------
-
-_ENTITY_STOP = {
-    'what', 'who', 'where', 'when', 'how', 'does', 'did', 'the', 'and',
-    'for', 'with', 'from', 'about', 'this', 'that', 'have', 'has', 'are',
-    'was', 'were', 'been', 'will', 'would', 'could', 'should', 'into',
-}
-
-def _extract_entities_for_kg(text: str, max_entities: int = 32) -> List[str]:
-    """Mirror of extract_query_entities, but applied to ingest content.
-
-    Picks single-word title-case tokens + bigrams of consecutive title-case
-    tokens. Same heuristic as query-side so node names and search terms line
-    up. Caps at max_entities to keep ingest fast.
-    """
-    if not text:
-        return []
-    words = [re.sub(r'[^\w\s-]', '', w).strip() for w in text.split()]
-    words = [w for w in words if w]
-    found: List[str] = []
-    seen: Set[str] = set()
-    # Single-word title-case
-    for w in words:
-        if w.istitle() and len(w) > 2 and w.lower() not in _ENTITY_STOP:
-            key = w.lower()
-            if key not in seen:
-                found.append(w)
-                seen.add(key)
-    # Bigrams of consecutive title-case
-    for i in range(len(words) - 1):
-        a, b = words[i], words[i + 1]
-        if (a.istitle() and b.istitle()
-                and a.lower() not in _ENTITY_STOP
-                and b.lower() not in _ENTITY_STOP
-                and len(a) > 1 and len(b) > 1):
-            phrase = f"{a} {b}"
-            key = phrase.lower()
-            if key not in seen:
-                found.append(phrase)
-                seen.add(key)
-        if len(found) >= max_entities:
-            break
-    return found[:max_entities]
-
-
-async def _embed_batch_local(texts: List[str]) -> List[List[float]]:
-    """Batch embed via the shared EmbedClient (async). Returns vectors in input order."""
-    if not texts:
-        return []
-    try:
-        return await _embed_client().embed_batch_async(texts)
-    except Exception as e:
-        log.warning(f"NV-Embed batch failed: {e}; trying singletons")
-        # Singleton fallback stays sync (each one-shot embed is small);
-        # offload to thread so we don't block the loop.
-        return await asyncio.gather(
-            *(asyncio.to_thread(get_embedding, t) for t in texts)
-        )
-
-
-class IndexInternalBatchRequest(BaseModel):
-    records: List[Dict[str, Any]]  # [{"id": str, "content": str, "metadata": dict}, ...]
-    arena: Optional[str] = "general"
-    # When supplied (parallel to `records`), skip the L4-QMD embed call
-    # and use these vectors directly. Compat shim populates this when
-    # shared-embed mode is on so we don't duplicate embed work across
-    # layers. Length must match records — defensive bail-out below if
-    # it doesn't.
-    embeddings: Optional[List[List[float]]] = None
-
-
-@app.post("/index-internal-batch")
-async def index_internal_batch(req: IndexInternalBatchRequest) -> dict:
-    """Populate L0 BM25 + L4 QMD vec + L3 Neo4j KG from one ingest call.
-
-    Called by the compat shim on /store-batch so the L2 7-layer fusion
-    has real data in every layer (not just L5/L6). Sequential within the
-    handler since all three writes are local (SQLite + bolt to a sibling
-    container).
-    """
-    t0 = time.time()
-    records = [r for r in (req.records or [])
-               if (r.get("content") or r.get("text"))]
-    if not records:
-        return {"status": "ok", "inserted": 0, "l0": 0, "l4_qmd": 0, "l3_entities": 0,
-                "l3_chunks": 0, "duration_ms": 0.0}
-
-    # Normalise
-    norm = []
-    for r in records:
-        content = r.get("content") or r.get("text") or ""
-        rid = (r.get("id")
-               or hashlib.sha1(content.encode()).hexdigest()[:32])
-        meta = r.get("metadata") or {}
-        path = meta.get("path") or meta.get("doc_id") or rid
-        norm.append({"id": str(rid), "content": content, "path": str(path),
-                     "metadata": meta})
-
-    now_iso = datetime.utcnow().isoformat() + "Z"
-    arena = req.arena or "general"
-
-    # ---- L0 BM25 (workspace.db) -----------------------------------------
-    l0_inserted = 0
-    try:
-        l0_db = Path(os.environ.get("PME_MEMORY_DB", str(L0_MEMORY_DB)))
-        l0_db.parent.mkdir(parents=True, exist_ok=True)
-        conn = sqlite3.connect(str(l0_db), timeout=10)
-        conn.execute("PRAGMA journal_mode=WAL")
-        # Schema is created by init_databases.py at container start, but be
-        # defensive in case L2 is run standalone.
-        conn.execute("""
-            CREATE TABLE IF NOT EXISTS chunks (
-                id INTEGER PRIMARY KEY,
-                path TEXT,
-                text TEXT,
-                file_type TEXT,
-                chunk_index INTEGER,
-                created_at TEXT
-            )
-        """)
-        conn.execute("""
-            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
-                path, text, file_type,
-                content='chunks',
-                content_rowid='id'
-            )
-        """)
-        conn.execute("""
-            CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
-                INSERT INTO chunks_fts(rowid, path, text, file_type)
-                VALUES (new.id, new.path, new.text, new.file_type);
-            END
-        """)
-        for n in norm:
-            cur = conn.execute(
-                "INSERT INTO chunks (path, text, file_type, chunk_index, created_at) VALUES (?, ?, ?, ?, ?)",
-                (f"bench/{arena}/{n['path']}.md", n["content"], "md", 0, now_iso),
-            )
-            if cur.rowcount > 0:
-                l0_inserted += 1
-        conn.commit()
-        conn.close()
-    except Exception as e:
-        log.error(f"L0 BM25 write failed: {e}")
-
-    # ---- L4 QMD vec (qmd.sqlite) ----------------------------------------
-    l4_inserted = 0
-    try:
-        # Shared-embed shortcut: if the compat shim handed us pre-computed
-        # vectors that line up with our normalised records, use them and
-        # skip our own embed RPC. Fall back to per-layer embedding when
-        # the vectors are absent or the lengths don't match (defensive).
-        shared_embs = req.embeddings
-        if (
-            shared_embs is not None
-            and len(shared_embs) == len(records)
-            and len(records) == len(norm)
-        ):
-            embeddings = shared_embs
-        else:
-            embeddings = await _embed_batch_local([n["content"] for n in norm])
-        if len(embeddings) != len(norm):
-            log.warning(f"L4 embed count mismatch: {len(embeddings)} != {len(norm)}")
-        qmd_db = Path(QMD_DB_PATH)
-        qmd_db.parent.mkdir(parents=True, exist_ok=True)
-        # Open with sqlite-vec loaded so we can dual-write to vec_index
-        # below. If extension load fails, vec_index inserts silently no-op
-        # via the try/except — chunks (JSON) still gets the write so the
-        # corpus stays whole; search just degrades to the old path.
-        conn = _open_qmd_conn()
-        conn.execute("PRAGMA journal_mode=WAL")
-        conn.execute("""
-            CREATE TABLE IF NOT EXISTS chunks (
-                id INTEGER PRIMARY KEY,
-                path TEXT,
-                text TEXT,
-                embedding TEXT,
-                embedding_model TEXT,
-                embedding_dim INTEGER,
-                chunk_index INTEGER,
-                created_at TEXT
-            )
-        """)
-        try:
-            _ensure_vec_index(conn)
-        except Exception as e:
-            log.error(f"vec_index DDL failed: {e} — falling back to chunks-only write")
-        for n, vec in zip(norm, embeddings):
-            if not vec:
-                continue
-            cur = conn.execute(
-                "INSERT INTO chunks (path, text, embedding, embedding_model, embedding_dim, chunk_index, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
-                (f"bench/{arena}/{n['path']}.md", n["content"],
-                 json.dumps(vec), "nv-embed-v2", len(vec), 0, now_iso),
-            )
-            # Mirror into the vec0 KNN index so search_qmd_informed can
-            # MATCH on the f32-packed vector. Dim must match the vec0 DDL
-            # (EMBED_DIM); skip rows where the embedding shape disagrees
-            # so a single bad row doesn't poison the batch insert.
-            if cur.lastrowid is not None and len(vec) == EMBED_DIM:
-                try:
-                    conn.execute(
-                        "INSERT INTO vec_index(id, embedding) VALUES (?, ?)",
-                        (cur.lastrowid, struct.pack(f"{len(vec)}f", *vec)),
-                    )
-                except Exception as e:
-                    # vec_index dual-write is defensive — the JSON column
-                    # in chunks is still the source of truth until the
-                    # backfill task confirms vec_index is in sync.
-                    log.debug(f"vec_index insert skipped for row {cur.lastrowid}: {e}")
-            l4_inserted += 1
-        conn.commit()
-        conn.close()
-    except Exception as e:
-        log.error(f"L4 QMD write failed: {e}")
-
-    # ---- L3 Neo4j KG ----------------------------------------------------
-    # Every node and edge written here is arena-scoped. Two paths:
-    #
-    #   1. Heuristic Concept extraction — title-case + bigrams over the
-    #      chunk body, same as before. Concepts MERGE on (arena, name)
-    #      so two tenants can independently mint a "Pricing" concept
-    #      without colliding.
-    #
-    #   2. Metadata-driven Person extraction — when the chunk's metadata
-    #      carries contact_email / contact_name (Pip emits these from
-    #      its ingest pipeline; other clients can do the same), we MERGE
-    #      a typed (:Entity:Person) node and connect it to the chunk via
-    #      a (:COMMUNICATED) edge that carries channel + direction. This
-    #      is the path the relationships UI reads from — it's reliable
-    #      because the writer knows exactly who the person is, no NLP
-    #      guessing required.
-    #
-    # The compound (arena, name) MERGE guarantees no cross-tenant entity
-    # collapse. Pre-existing unscoped entities (arena IS NULL) are left
-    # alone; the wipe-legacy migration script handles them out of band.
-    l3_entities = 0
-    l3_chunks = 0
-    try:
-        driver = get_neo4j_driver()
-        async with driver.session() as session:
-            # Indexes — idempotent. The compound (arena, name) is the
-            # right shape now that entities are arena-scoped; the legacy
-            # entity_name index stays for the wipe-migration to work
-            # against pre-arena rows, then can be dropped in a follow-up.
-            try:
-                await session.run("CREATE INDEX entity_arena_name IF NOT EXISTS FOR (n:Entity) ON (n.arena, n.name)")
-                await session.run("CREATE INDEX person_arena_email IF NOT EXISTS FOR (n:Person) ON (n.arena, n.email)")
-                await session.run("CREATE INDEX chunk_arena IF NOT EXISTS FOR (c:Chunk) ON (c.arena)")
-                await session.run("CREATE INDEX chunk_id IF NOT EXISTS FOR (c:Chunk) ON (c.id)")
-                # ChannelStat is the denormalised aggregate read by
-                # /aggregate on the fast path. Compound index covers
-                # the (arena, person_email) lookup that the reader
-                # uses; the per-channel rows are returned in one
-                # range scan.
-                await session.run("CREATE INDEX channelstat_arena_email IF NOT EXISTS FOR (s:ChannelStat) ON (s.arena, s.person_email)")
-                # UNIQUE constraint on the writer's MERGE key. Without
-                # this, two concurrent index-internal-batch transactions
-                # can both decide a ChannelStat doesn't exist and create
-                # rival nodes — the index doesn't lock, the constraint
-                # does. The constraint also implies an index on the
-                # full key so the MERGE locks efficiently.
-                await session.run("CREATE CONSTRAINT channelstat_unique IF NOT EXISTS FOR (s:ChannelStat) REQUIRE (s.arena, s.person_email, s.channel) IS UNIQUE")
-            except Exception:
-                pass
-            for n in norm:
-                heuristic_entities = _extract_entities_for_kg(n["content"])
-                meta = n.get("metadata") or {}
-                contact_email = meta.get("contact_email")
-                contact_name = meta.get("contact_name")
-                channel = meta.get("channel")
-                direction = meta.get("direction")
-                occurred_at = meta.get("timestamp") or meta.get("occurred_at") or now_iso
-                # Skip the chunk only when there is genuinely nothing to
-                # connect — heuristic entities AND no person metadata.
-                if not heuristic_entities and not contact_email and not contact_name:
-                    continue
-                # Create the chunk node — arena property is the
-                # tenant-isolation anchor. Every read traverses through
-                # this node, so getting the arena right here is the
-                # single most important invariant of this whole block.
-                await session.run(
-                    """
-                    MERGE (c:Chunk {id: $cid})
-                    SET c.text = $text,
-                        c.path = $path,
-                        c.source_file = $path,
-                        c.arena = $arena,
-                        c.created_at = $now
-                    """,
-                    cid=n["id"], text=n["content"][:2000], path=n["path"],
-                    arena=arena, now=now_iso,
-                )
-                l3_chunks += 1
-
-                # Concept entities — heuristic, arena-scoped.
-                for ent in heuristic_entities:
-                    await session.run(
-                        """
-                        MERGE (e:Entity:Concept {arena: $arena, name: $name})
-                        ON CREATE SET e.type = 'Concept',
-                                      e.created_at = $now,
-                                      e.weight = 1.0
-                        WITH e
-                        MATCH (c:Chunk {arena: $arena, id: $cid})
-                        MERGE (e)-[r:MENTIONS]->(c)
-                        ON CREATE SET r.weight = 1.0, r.created_at = $now
-                        ON MATCH SET r.weight = coalesce(r.weight, 1.0) + 0.1
-                        """,
-                        arena=arena, name=ent, cid=n["id"], now=now_iso,
-                    )
-                    l3_entities += 1
-                # Concept-concept co-occurrence — same arena on both
-                # ends so cross-tenant CO_OCCURS edges can't form even
-                # if two tenants happen to extract the same concept name.
-                if len(heuristic_entities) >= 2:
-                    for i in range(len(heuristic_entities)):
-                        for j in range(i + 1, len(heuristic_entities)):
-                            await session.run(
-                                """
-                                MATCH (a:Entity:Concept {arena: $arena, name: $a})
-                                MATCH (b:Entity:Concept {arena: $arena, name: $b})
-                                MERGE (a)-[r:CO_OCCURS]->(b)
-                                ON CREATE SET r.weight = 0.5, r.created_at = $now
-                                ON MATCH SET r.weight = coalesce(r.weight, 0.5) + 0.05
-                                """,
-                                arena=arena, a=heuristic_entities[i],
-                                b=heuristic_entities[j], now=now_iso,
-                            )
-
-                # Person entities — typed via writer-supplied metadata.
-                # Email gets its own node (canonical id for a person);
-                # name gets its own node (display surface). When both
-                # are present they're linked via KNOWN_AS so a query
-                # against either resolves the same person.
-                person_email_node = None
-                if isinstance(contact_email, str) and contact_email.strip():
-                    norm_email = contact_email.strip().lower()
-                    # Two-phase write: MERGE the Person + COMMUNICATED
-                    # edge, then update the ChannelStat aggregate IFF
-                    # the edge was just created. The `r._counted` flag
-                    # is the idempotency rail — set false on CREATE and
-                    # flipped to true after the stat update, so replays
-                    # of the same eventId never double-count even when
-                    # the chunk already exists.
-                    await session.run(
-                        """
-                        MERGE (p:Entity:Person {arena: $arena, email: $email})
-                        ON CREATE SET p.created_at = $now,
-                                      p.first_seen = $occurred_at,
-                                      p.last_seen = $occurred_at
-                        ON MATCH SET p.last_seen = CASE
-                              WHEN $occurred_at > coalesce(p.last_seen, '')
-                                THEN $occurred_at
-                              ELSE p.last_seen END
-                        WITH p
-                        MATCH (c:Chunk {arena: $arena, id: $cid})
-                        MERGE (p)-[r:COMMUNICATED]->(c)
-                        ON CREATE SET r.channel = $channel,
-                                      r.direction = $direction,
-                                      r.occurred_at = $occurred_at,
-                                      r.weight = 1.0,
-                                      r._counted = false
-                        WITH p, r
-                        // ChannelStat denormalises Person-COMMUNICATED
-                        // edge counts so /aggregate becomes a property
-                        // read instead of a per-query Cypher walk over
-                        // every edge. Read path falls back to the edge
-                        // walk for older tenants whose stats haven't
-                        // been backfilled, so this is a forward-only
-                        // optimisation — no migration needed for stats
-                        // to start materialising.
-                        FOREACH (_ IN CASE WHEN r._counted = false THEN [1] ELSE [] END |
-                          MERGE (s:ChannelStat {arena: $arena, person_email: $email, channel: $channel})
-                          ON CREATE SET s.count = 0,
-                                        s.inbound = 0,
-                                        s.outbound = 0,
-                                        s.first_seen = $occurred_at,
-                                        s.last_seen = $occurred_at,
-                                        s.created_at = $now
-                          SET s.count = s.count + 1,
-                              s.inbound = s.inbound + (CASE WHEN $direction = 'inbound' THEN 1 ELSE 0 END),
-                              s.outbound = s.outbound + (CASE WHEN $direction = 'outbound' THEN 1 ELSE 0 END),
-                              s.first_seen = CASE
-                                WHEN $occurred_at < coalesce(s.first_seen, $occurred_at)
-                                  THEN $occurred_at
-                                ELSE s.first_seen END,
-                              s.last_seen = CASE
-                                WHEN $occurred_at > coalesce(s.last_seen, '')
-                                  THEN $occurred_at
-                                ELSE s.last_seen END,
-                              s.updated_at = $now
-                          MERGE (p)-[:HAS_STAT]->(s)
-                          SET r._counted = true
-                        )
-                        """,
-                        arena=arena, email=norm_email, cid=n["id"],
-                        channel=channel, direction=direction,
-                        occurred_at=occurred_at, now=now_iso,
-                    )
-                    person_email_node = norm_email
-                    l3_entities += 1
-                if isinstance(contact_name, str) and contact_name.strip():
-                    cname = contact_name.strip()
-                    await session.run(
-                        """
-                        MERGE (p:Entity:Person {arena: $arena, name: $name})
-                        ON CREATE SET p.created_at = $now,
-                                      p.first_seen = $occurred_at,
-                                      p.last_seen = $occurred_at
-                        ON MATCH SET p.last_seen = CASE
-                              WHEN $occurred_at > coalesce(p.last_seen, '')
-                                THEN $occurred_at
-                              ELSE p.last_seen END
-                        WITH p
-                        MATCH (c:Chunk {arena: $arena, id: $cid})
-                        MERGE (p)-[r:COMMUNICATED]->(c)
-                        ON CREATE SET r.channel = $channel,
-                                      r.direction = $direction,
-                                      r.occurred_at = $occurred_at,
-                                      r.weight = 1.0
-                        """,
-                        arena=arena, name=cname, cid=n["id"],
-                        channel=channel, direction=direction,
-                        occurred_at=occurred_at, now=now_iso,
-                    )
-                    l3_entities += 1
-                    # Link name→email node so the relationships query
-                    # can resolve either alias to the same person.
-                    if person_email_node:
-                        await session.run(
-                            """
-                            MATCH (n:Person {arena: $arena, name: $name})
-                            MATCH (e:Person {arena: $arena, email: $email})
-                            MERGE (n)-[:KNOWN_AS]->(e)
-                            """,
-                            arena=arena, name=cname, email=person_email_node,
-                        )
-    except Exception as e:
-        log.error(f"L3 KG write failed: {e}")
-
-    dur_ms = (time.time() - t0) * 1000.0
-    return {
-        "status": "ok",
-        "inserted": len(norm),
-        "l0": l0_inserted,
-        "l4_qmd": l4_inserted,
-        "l3_entities": l3_entities,
-        "l3_chunks": l3_chunks,
-        "duration_ms": round(dur_ms, 1),
-    }
-
-
-@app.post("/forget-internal")
-async def forget_internal(request: Request) -> dict:
-    """Wipe L0 + L4-qmd + L3.
-
-    Two modes:
-      - Tenant-scoped (default, safe): pass `{"arena": "<tenant>"}` and
-        only that tenant's rows are deleted. Used by tenant offboarding
-        and by tests.
-      - Global (unsafe): the bench harness needs to wipe everything
-        between runs. Require an explicit `{"confirm": "GLOBAL_WIPE"}`
-        flag — without it we refuse rather than nuke shared infra.
-
-    Pre-fix this endpoint silently ignored the arena param and always
-    deleted globally. That meant a tenant offboarding script — or any
-    caller that read the param-name and trusted it — would erase every
-    other tenant's L3 graph and wipe the shared sqlite stores. Hence
-    the explicit confirm gate now.
-    """
-    try:
-        body = await request.json()
-    except Exception:
-        body = {}
-    arena = body.get("arena")
-    confirm = body.get("confirm")
-    if not arena and confirm != "GLOBAL_WIPE":
-        raise HTTPException(
-            status_code=400,
-            detail="forget-internal requires either 'arena' (tenant-scoped) "
-                   "or 'confirm: GLOBAL_WIPE' (unsafe, deletes everything).",
-        )
-    deleted = {"l0": 0, "l4_qmd": 0, "l3_entities": 0, "l3_chunks": 0}
-
-    # ---- L0 BM25 (sqlite) ----------------------------------------------
-    # The L0 chunks table doesn't carry an arena column today, so we
-    # only support GLOBAL_WIPE here. Tenant-scoped L0 deletes are a
-    # follow-up (needs schema migration to add `arena` to L0 rows).
-    try:
-        l0_db = Path(os.environ.get("PME_MEMORY_DB", str(L0_MEMORY_DB)))
-        if l0_db.exists() and confirm == "GLOBAL_WIPE":
-            conn = sqlite3.connect(str(l0_db), timeout=5)
-            cur = conn.execute("DELETE FROM chunks")
-            deleted["l0"] = cur.rowcount
-            try:
-                conn.execute("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')")
-            except Exception:
-                pass
-            conn.commit(); conn.close()
-    except Exception as e:
-        log.error(f"L0 forget failed: {e}")
-
-    # ---- L4 sqlite-vec --------------------------------------------------
-    # Same situation as L0 — no per-arena column on chunks. Global only
-    # for now; tenant-scoped delete is a follow-up.
-    try:
-        if Path(QMD_DB_PATH).exists() and confirm == "GLOBAL_WIPE":
-            conn = sqlite3.connect(QMD_DB_PATH, timeout=5)
-            cur = conn.execute("DELETE FROM chunks")
-            deleted["l4_qmd"] = cur.rowcount
-            conn.commit(); conn.close()
-    except Exception as e:
-        log.error(f"L4 QMD forget failed: {e}")
-
-    # ---- L3 Neo4j -------------------------------------------------------
-    # Neo4j chunks AND entities both carry arena now, so tenant-scoped
-    # delete works correctly here even if L0/L4 still need a migration.
-    try:
-        driver = get_neo4j_driver()
-        async with driver.session() as session:
-            if arena:
-                r1 = await session.run(
-                    "MATCH (c:Chunk {arena: $arena}) DETACH DELETE c RETURN count(c) AS n",
-                    arena=arena,
-                )
-                rec = await r1.single()
-                deleted["l3_chunks"] = rec["n"]
-                r2 = await session.run(
-                    "MATCH (e:Entity {arena: $arena}) DETACH DELETE e RETURN count(e) AS n",
-                    arena=arena,
-                )
-                rec = await r2.single()
-                deleted["l3_entities"] = rec["n"]
-            else:  # confirm == "GLOBAL_WIPE", validated above
-                r1 = await session.run("MATCH (c:Chunk) DETACH DELETE c RETURN count(c) AS n")
-                rec = await r1.single()
-                deleted["l3_chunks"] = rec["n"]
-                r2 = await session.run("MATCH (e:Entity) DETACH DELETE e RETURN count(e) AS n")
-                rec = await r2.single()
-                deleted["l3_entities"] = rec["n"]
-    except Exception as e:
-        log.error(f"L3 forget failed: {e}")
-    return {"status": "ok", "deleted": deleted, "arena": arena, "global_wipe": confirm == "GLOBAL_WIPE"}
-
-
-class AggregateInternalRequest(BaseModel):
-    """Aggregate (:Person)-[:COMMUNICATED]->(:Chunk) edges by group_by keys.
-
-    The relationships UI pre-#28 went through a metadata-filtered
-    /search and grouped client-side, capped at the engine over-fetch
-    ceiling. With typed-Person nodes in L3 we can run a single Cypher
-    aggregate that scales to any volume — no over-fetch, no cap.
-
-    Required: arena (the tenant scope) plus enough metadata to identify
-    the Person node we're rolling up. Today that means contact_email
-    (the canonical Person key), but the shape leaves room for future
-    Person identifiers (e.g. slack_user_id, hubspot_contact_id) without
-    a wire change.
-    """
-
-    arena: str
-    contact_email: Optional[str] = None
-    contact_name: Optional[str] = None
-    # Group by these properties on the COMMUNICATED edge. Only the
-    # relationship-page-supported keys are honoured; unknown keys are
-    # silently dropped (no useful aggregate shape for them).
-    group_by: List[str] = ["channel"]
-
-
-class AggregateBucket(BaseModel):
-    keys: Dict[str, Optional[str]]
-    count: int
-    inbound: int
-    outbound: int
-    last_seen: Optional[str] = None
-    first_seen: Optional[str] = None
-
-
-class AggregateInternalResponse(BaseModel):
-    arena: str
-    total: int
-    last_seen: Optional[str] = None
-    buckets: List[AggregateBucket]
-
-
-# Whitelist of group_by keys we know how to project. Cypher
-# parameter-substitution doesn't work on property names, so we
-# template the keys into the query — this whitelist is the safety
-# rail that keeps the templating from accepting arbitrary input.
-_AGGREGATE_GROUP_BY_KEYS = {"channel", "direction"}
-
-
-@app.post("/aggregate-internal", response_model=AggregateInternalResponse)
-async def aggregate_internal(req: AggregateInternalRequest) -> AggregateInternalResponse:
-    """Aggregate Person→Chunk COMMUNICATED edges by edge properties.
-
-    Returns one bucket per (group_by key combination) with count,
-    inbound/outbound split, and time bounds. The Person match is
-    arena-scoped (mandatory) and additionally filtered by whatever
-    Person identifier the caller supplies.
-
-    No fallback to chunk scanning — if the typed-Person nodes don't
-    exist for this contact, the response is `total: 0` with no
-    buckets, and the caller falls back to whatever it had before.
-    That's intentional: the over-fetch path is in TES (#273); this
-    endpoint is the scaling answer that doesn't have one.
-    """
-    arena = (req.arena or "").strip()
-    if not arena:
-        raise HTTPException(status_code=400, detail="arena is required")
-    contact_email = (req.contact_email or "").strip().lower()
-    contact_name = (req.contact_name or "").strip()
-    if not contact_email and not contact_name:
-        raise HTTPException(
-            status_code=400,
-            detail="provide contact_email and/or contact_name to identify the Person",
-        )
-
-    # Filter group_by to the supported keys; preserve order so a caller
-    # asking for ["direction", "channel"] gets buckets keyed in that
-    # order on the response.
-    seen: set[str] = set()
-    safe_group_by: List[str] = []
-    for k in req.group_by or []:
-        if k in _AGGREGATE_GROUP_BY_KEYS and k not in seen:
-            seen.add(k)
-            safe_group_by.append(k)
-
-    driver = get_neo4j_driver()
-
-    try:
-        async with driver.session() as session:
-            # Fast path: read from the ChannelStat denormalisation
-            # whenever the caller has an email and is grouping by
-            # channel. ChannelStats are written by /index-internal-batch
-            # on every store with contact_email metadata, so any tenant
-            # with new ingest gets O(channels) reads instead of an
-            # edge walk over every COMMUNICATED relationship.
-            #
-            # Conditions for the fast path:
-            #   - contact_email set (stats are email-keyed; name-only
-            #     contacts fall through to the edge walk).
-            #   - group_by is exactly ["channel"] OR no group_by (single
-            #     bucket). Other group_by combinations (e.g. with
-            #     direction) need the edge granularity the stats
-            #     don't carry.
-            fast_path_eligible = bool(contact_email) and (
-                not safe_group_by or safe_group_by == ["channel"]
-            )
-            if fast_path_eligible:
-                _res = await session.run(
-                    "MATCH (s:ChannelStat {arena: $arena, person_email: $email})\n"
-                    "RETURN s.channel AS channel,\n"
-                    "       s.count AS count,\n"
-                    "       s.inbound AS inbound,\n"
-                    "       s.outbound AS outbound,\n"
-                    "       s.last_seen AS last_seen,\n"
-                    "       s.first_seen AS first_seen\n"
-                    "ORDER BY s.count DESC\n",
-                    arena=arena, email=contact_email,
-                )
-                stats_rows = [rec async for rec in _res]
-                if stats_rows:
-                    # Build buckets directly. When group_by=[] we
-                    # collapse to a single overall bucket; otherwise
-                    # one bucket per channel.
-                    if safe_group_by == ["channel"]:
-                        buckets = [
-                            AggregateBucket(
-                                keys={"channel": rec["channel"]},
-                                count=int(rec["count"] or 0),
-                                inbound=int(rec["inbound"] or 0),
-                                outbound=int(rec["outbound"] or 0),
-                                last_seen=str(rec["last_seen"]) if rec["last_seen"] else None,
-                                first_seen=str(rec["first_seen"]) if rec["first_seen"] else None,
-                            )
-                            for rec in stats_rows
-                        ]
-                        total = sum(b.count for b in buckets)
-                        latest = None
-                        for b in buckets:
-                            if b.last_seen and (latest is None or b.last_seen > latest):
-                                latest = b.last_seen
-                    else:
-                        # Single global bucket — sum across channels.
-                        total = sum(int(rec["count"] or 0) for rec in stats_rows)
-                        inbound = sum(int(rec["inbound"] or 0) for rec in stats_rows)
-                        outbound = sum(int(rec["outbound"] or 0) for rec in stats_rows)
-                        last_seens = [rec["last_seen"] for rec in stats_rows if rec["last_seen"]]
-                        first_seens = [rec["first_seen"] for rec in stats_rows if rec["first_seen"]]
-                        latest = max((str(x) for x in last_seens), default=None)
-                        earliest = min((str(x) for x in first_seens), default=None)
-                        buckets = [AggregateBucket(
-                            keys={},
-                            count=total,
-                            inbound=inbound,
-                            outbound=outbound,
-                            last_seen=latest,
-                            first_seen=earliest,
-                        )]
-                    return AggregateInternalResponse(
-                        arena=arena,
-                        total=total,
-                        last_seen=latest,
-                        buckets=buckets,
-                    )
-                # else: stats absent (older tenant pre-rollup, or this
-                # contact has no email-keyed Person yet) → fall through
-                # to the edge-walk path.
-
-            # Edge-walk path (original Cypher). Used when:
-            #   - caller has only contact_name (no email-keyed stats)
-            #   - caller asked for a group_by we don't denormalise (e.g.
-            #     direction)
-            #   - tenant predates the rollup writer (no stats nodes yet)
-            # Both paths return the same response shape, so callers
-            # don't need to know which served them.
-            #
-            # Build the Person match. We want either email-keyed or
-            # name-keyed Person nodes; when both are supplied we OR
-            # them so a caller can hit either alias. Both branches
-            # arena-scope the Person.
-            person_clauses: List[str] = []
-            params: Dict[str, Any] = {"arena": arena}
-            if contact_email:
-                person_clauses.append("(p.email = $contact_email)")
-                params["contact_email"] = contact_email
-            if contact_name:
-                person_clauses.append("(p.name = $contact_name)")
-                params["contact_name"] = contact_name
-            person_filter = " OR ".join(person_clauses)
-
-            # group_by keys go into the WITH clause. Cypher doesn't
-            # support property-name parameters, so we template them
-            # in — the whitelist above is the safety rail against
-            # injection. Built up separately rather than via f-string
-            # so the static MATCH clause stays a plain string and the
-            # arena-safety lint can parse it cleanly.
-            with_keys = ", ".join(f"r.{k} AS {k}" for k in safe_group_by)
-            return_keys = ", ".join(safe_group_by)
-
-            # Static base — arena scope on both Person and Chunk so the
-            # lint catches any future copy-paste that forgets it.
-            base = (
-                "MATCH (p:Person {arena: $arena})-[r:COMMUNICATED]->(c:Chunk {arena: $arena})\n"
-                "WHERE " + person_filter + "\n"
-            )
-            agg_select = (
-                "count(*) AS count,\n"
-                "sum(CASE WHEN _direction = 'inbound' THEN 1 ELSE 0 END) AS inbound,\n"
-                "sum(CASE WHEN _direction = 'outbound' THEN 1 ELSE 0 END) AS outbound,\n"
-                "max(_occurred_at) AS last_seen,\n"
-                "min(_occurred_at) AS first_seen\n"
-            )
-
-            if safe_group_by:
-                cypher = (
-                    base
-                    + f"WITH {with_keys}, r.direction AS _direction, r.occurred_at AS _occurred_at\n"
-                    + f"RETURN {return_keys},\n"
-                    + agg_select
-                    + "ORDER BY count DESC\n"
-                )
-            else:
-                # No group_by → one global bucket (just the overall
-                # totals for this Person). Useful for "total comms
-                # with X" without per-channel breakdown.
-                cypher = (
-                    base
-                    + "WITH r.direction AS _direction, r.occurred_at AS _occurred_at\n"
-                    + "RETURN " + agg_select
-                )
-
-            buckets: List[AggregateBucket] = []
-            total = 0
-            latest: Optional[str] = None
-            _res = await session.run(cypher, **params)
-            async for rec in _res:
-                count = int(rec["count"] or 0)
-                total += count
-                last_seen = rec["last_seen"]
-                if last_seen and (latest is None or str(last_seen) > latest):
-                    latest = str(last_seen)
-                bucket_keys: Dict[str, Optional[str]] = (
-                    {k: rec[k] for k in safe_group_by} if safe_group_by else {}
-                )
-                buckets.append(AggregateBucket(
-                    keys=bucket_keys,
-                    count=count,
-                    inbound=int(rec["inbound"] or 0),
-                    outbound=int(rec["outbound"] or 0),
-                    last_seen=str(last_seen) if last_seen else None,
-                    first_seen=str(rec["first_seen"]) if rec["first_seen"] else None,
-                ))
-            return AggregateInternalResponse(
-                arena=arena,
-                total=total,
-                last_seen=latest,
-                buckets=buckets,
-            )
-    except HTTPException:
-        raise
-    except Exception as e:
-        log.error(f"aggregate-internal failed: {e}")
-        raise HTTPException(status_code=500, detail=f"aggregate failed: {e}")
-
-
-# ── /people-list-internal ───────────────────────────────────────────────
-#
-# Corpus-level "all people in these arenas" projection backing the Pip
-# Relationships UI list page. Different aggregation level from
-# /aggregate-internal: that one returns buckets PER ONE PERSON; this one
-# returns one row PER PERSON across the whole arena, sorted by recency.
-#
-# Reads the ChannelStat denorm directly. ChannelStats are keyed on
-# (arena, person_email, channel) and maintained on every store by the
-# writer block above (~line 1645 onwards). Each person has one
-# ChannelStat row per channel they appear on. Aggregating over
-# person_email collapses to one row per person; the per-channel
-# breakdown rides along as a nested list for callers who want both.
-#
-# Falls back to an edge walk path only when ChannelStats are absent
-# for the arena (older tenant predating the rollup writer). In normal
-# operation the fast path serves every read.
-
-class PeopleListInternalRequest(BaseModel):
-    """List all people communicated with across one or more arenas.
-
-    Multi-arena support exists so the Pip Relationships UI can answer
-    "which colleagues do Philip AND Jeanne both know" without N round-
-    trips. The endpoint is invariant to arena ordering — duplicates
-    across arenas are collapsed by person_email.
-
-    `emails` is a batched-mode filter (return only these emails). Used
-    by Pip's nightly health-recompute to fetch facets for a known set
-    of contacts in one call.
-
-    `search` matches person_email and person_name as case-insensitive
-    substrings. Empty/null → no filter. Server-side, so the caller
-    doesn't have to overfetch.
-
-    Pagination is `limit`/`offset`. Default limit matches the Pip UI
-    page size; large limits are fine since the underlying read is one
-    Cypher query against an indexed denorm.
-    """
-
-    arenas: List[str]
-    emails: Optional[List[str]] = None
-    search: Optional[str] = None
-    limit: int = 200
-    offset: int = 0
-    # Sort key. Whitelisted; unknown → ValidationError so callers can't
-    # smuggle Cypher fragments into the ORDER BY.
-    order_by: str = "last_seen_desc"
-
-
-class ChannelStatBucket(BaseModel):
-    """Per-channel breakdown for one person, denormalised from ChannelStat."""
-
-    channel: str
-    count: int
-    inbound: int
-    outbound: int
-    last_seen: Optional[str] = None
-    first_seen: Optional[str] = None
-
-
-class PeopleListEntry(BaseModel):
-    """One person's rollup in the corpus-level list response."""
-
-    person_email: str
-    person_name: Optional[str] = None
-    total: int
-    inbound: int
-    outbound: int
-    last_seen: Optional[str] = None
-    first_seen: Optional[str] = None
-    channels: List[ChannelStatBucket]
-
-
-class PeopleListInternalResponse(BaseModel):
-    """Paginated people-list result.
-
-    `total_count` is the distinct-people count after filters, before
-    pagination — so the caller can paint a "Showing 1-200 of 1206"
-    header without a second round-trip. `has_more` is the derived
-    pagination signal.
-    """
-
-    total_count: int
-    has_more: bool
-    items: List[PeopleListEntry]
-
-
-# Whitelist for order_by to keep the ORDER BY clause injection-safe.
-# Values are templated into Cypher after `WITH row` so each one must
-# qualify properties with `row.` (or wrap in a function that does).
-_PEOPLE_LIST_ORDER_BY = {
-    "last_seen_desc": "row.last_seen DESC",
-    "last_seen_asc": "row.last_seen ASC",
-    "total_desc": "row.total DESC",
-    "total_asc": "row.total ASC",
-    "name_asc": "coalesce(row.person_name, row.person_email) ASC",
-    "name_desc": "coalesce(row.person_name, row.person_email) DESC",
-}
-
-
-@app.post("/people-list-internal", response_model=PeopleListInternalResponse)
-async def people_list_internal(
-    req: PeopleListInternalRequest,
-) -> PeopleListInternalResponse:
-    """Corpus-level "all people I've communicated with" projection.
-
-    Single Cypher pass over ChannelStat keyed on (arena, person_email).
-    Returns paginated, filtered, sorted rows with per-channel
-    breakdown for each person.
-
-    Authentication is enforced by the surrounding TES module
-    (`authorizeClient` in resolvers.js); this endpoint itself trusts
-    its caller by virtue of running inside the engine network.
-    """
-
-    arenas = [a.strip() for a in (req.arenas or []) if a and a.strip()]
-    if not arenas:
-        raise HTTPException(status_code=400, detail="at least one arena is required")
-
-    if req.order_by not in _PEOPLE_LIST_ORDER_BY:
-        raise HTTPException(
-            status_code=400,
-            detail=f"order_by must be one of: {sorted(_PEOPLE_LIST_ORDER_BY)}",
-        )
-    order_clause = _PEOPLE_LIST_ORDER_BY[req.order_by]
-
-    limit = max(1, min(req.limit or 200, 2000))
-    offset = max(0, req.offset or 0)
-    emails_filter = (
-        [e.strip().lower() for e in req.emails if e and e.strip()]
-        if req.emails
-        else None
-    )
-    search_pattern = (req.search or "").strip().lower() or None
-
-    driver = get_neo4j_driver()
-
-    try:
-        async with driver.session() as session:
-            # Fast path: read ChannelStat directly. Aggregation walks
-            # the (arena, person_email) compound index — O(channels)
-            # per person, one row per channel.
-            #
-            # Two-stage shape: first WITH collects per-person rollups
-            # (sum across channels + collect the per-channel buckets);
-            # second WITH applies the filter, sort, total, and pagination.
-            params: Dict[str, Any] = {
-                "arenas": arenas,
-                "limit": limit,
-                "offset": offset,
-            }
-
-            # Build the filter clause. `arena IN $arenas` is the
-            # multi-arena gate. `s.person_email IN $emails` is the
-            # batched-mode filter; toLower handles caller-side case
-            # drift. `search` matches against person_email or
-            # (joined) person_name.
-            email_filter_clause = ""
-            if emails_filter:
-                email_filter_clause = " AND s.person_email IN $emails"
-                params["emails"] = emails_filter
-
-            if search_pattern:
-                # Defer the whole search filter until after the Person
-                # join — person_name only exists after the OPTIONAL
-                # MATCH below. Pre-filtering ChannelStats on email
-                # alone (the earlier two-step approach) silently dropped
-                # name-only matches, e.g. email=ag@x.io / name="Alex
-                # Tong" with search="alex" — the early WHERE failed and
-                # the post-join filter never saw the row.
-                params["search"] = search_pattern
-
-            cypher = (
-                "MATCH (s:ChannelStat)\n"
-                "WHERE s.arena IN $arenas"
-                + email_filter_clause
-                + "\n"
-                "WITH s.person_email AS person_email,\n"
-                "     collect({channel: s.channel, count: s.count,\n"
-                "              inbound: s.inbound, outbound: s.outbound,\n"
-                "              last_seen: s.last_seen, first_seen: s.first_seen,\n"
-                "              arena: s.arena}) AS channels\n"
-                # Resolve display name from the typed-Person node when
-                # one exists. OPTIONAL so people who only have edge data
-                # (no Person node yet) still show up — name falls back
-                # to email at the caller.
-                "OPTIONAL MATCH (p:Person {email: person_email})\n"
-                "WHERE p.arena IN $arenas\n"
-                "WITH person_email,\n"
-                "     channels,\n"
-                "     head(collect(DISTINCT p.name)) AS person_name\n"
-                # Apply the search filter now that we have both the
-                # joined name and the email available. `coalesce(name,
-                # '')` keeps people without a Person node in the result
-                # set when their email matches — they fall through the
-                # name probe cleanly instead of bypassing the filter
-                # via a `person_name IS NULL` short-circuit, which was
-                # the prior bug (anyone without a Person node passed
-                # search regardless of term).
-                + (
-                    "WHERE toLower(coalesce(person_name, '')) CONTAINS $search\n"
-                    "       OR toLower(person_email) CONTAINS $search\n"
-                    if search_pattern
-                    else ""
-                )
-                + "WITH person_email, person_name, channels,\n"
-                "     reduce(t = 0, x IN channels | t + coalesce(x.count, 0)) AS total,\n"
-                "     reduce(t = 0, x IN channels | t + coalesce(x.inbound, 0)) AS inbound,\n"
-                "     reduce(t = 0, x IN channels | t + coalesce(x.outbound, 0)) AS outbound,\n"
-                "     reduce(latest = '', x IN channels |\n"
-                "         CASE WHEN x.last_seen IS NOT NULL AND toString(x.last_seen) > latest\n"
-                "              THEN toString(x.last_seen) ELSE latest END) AS last_seen_raw,\n"
-                "     reduce(earliest = '', x IN channels |\n"
-                "         CASE WHEN x.first_seen IS NOT NULL AND (earliest = '' OR toString(x.first_seen) < earliest)\n"
-                "              THEN toString(x.first_seen) ELSE earliest END) AS first_seen_raw\n"
-                "WITH person_email, person_name, channels, total, inbound, outbound,\n"
-                "     CASE WHEN last_seen_raw = '' THEN null ELSE last_seen_raw END AS last_seen,\n"
-                "     CASE WHEN first_seen_raw = '' THEN null ELSE first_seen_raw END AS first_seen\n"
-                # collect-into-list and then split is the standard
-                # Cypher trick for getting both the total count and a
-                # paginated slice from one query.
-                "WITH collect({person_email: person_email, person_name: person_name,\n"
-                "              channels: channels, total: total,\n"
-                "              inbound: inbound, outbound: outbound,\n"
-                "              last_seen: last_seen, first_seen: first_seen}) AS all_rows\n"
-                "WITH all_rows, size(all_rows) AS total_count\n"
-                "UNWIND all_rows AS row\n"
-                f"WITH row, total_count ORDER BY {order_clause}\n"
-                "SKIP $offset LIMIT $limit\n"
-                "RETURN total_count, collect(row) AS page\n"
-            )
-
-            res = await session.run(cypher, **params)
-            record = await res.single()
-            if record is None:
-                return PeopleListInternalResponse(
-                    total_count=0, has_more=False, items=[]
-                )
-
-            total_count = int(record["total_count"] or 0)
-            page = record["page"] or []
-
-            items: List[PeopleListEntry] = []
-            for row in page:
-                channels = [
-                    ChannelStatBucket(
-                        channel=ch.get("channel") or "unknown",
-                        count=int(ch.get("count") or 0),
-                        inbound=int(ch.get("inbound") or 0),
-                        outbound=int(ch.get("outbound") or 0),
-                        last_seen=str(ch.get("last_seen")) if ch.get("last_seen") else None,
-                        first_seen=str(ch.get("first_seen")) if ch.get("first_seen") else None,
-                    )
-                    for ch in (row.get("channels") or [])
-                ]
-                items.append(
-                    PeopleListEntry(
-                        person_email=row.get("person_email"),
-                        person_name=row.get("person_name"),
-                        total=int(row.get("total") or 0),
-                        inbound=int(row.get("inbound") or 0),
-                        outbound=int(row.get("outbound") or 0),
-                        last_seen=str(row.get("last_seen")) if row.get("last_seen") else None,
-                        first_seen=str(row.get("first_seen")) if row.get("first_seen") else None,
-                        channels=channels,
-                    )
-                )
-
-            return PeopleListInternalResponse(
-                total_count=total_count,
-                has_more=(offset + len(items)) < total_count,
-                items=items,
-            )
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        log.error(f"people-list-internal failed: {e}")
-        raise HTTPException(status_code=500, detail=f"people-list failed: {e}")
-
-
-@app.get("/index-internal-stats")
-async def index_internal_stats() -> dict:
-    """Quick sanity check that the L0/L4-qmd/L3 stores are populated."""
-    out = {"l0_chunks": 0, "l4_qmd_chunks": 0,
-           "l3_chunks": 0, "l3_entities": 0}
-    try:
-        l0_db = Path(os.environ.get("PME_MEMORY_DB", str(L0_MEMORY_DB)))
-        if l0_db.exists():
-            conn = sqlite3.connect(str(l0_db), timeout=5)
-            r = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()
-            out["l0_chunks"] = r[0] if r else 0
-            conn.close()
-    except Exception as e:
-        out["l0_error"] = str(e)
-    try:
-        if Path(QMD_DB_PATH).exists():
-            conn = sqlite3.connect(QMD_DB_PATH, timeout=5)
-            r = conn.execute("SELECT COUNT(*) FROM chunks WHERE embedding IS NOT NULL").fetchone()
-            out["l4_qmd_chunks"] = r[0] if r else 0
-            conn.close()
-    except Exception as e:
-        out["l4_qmd_error"] = str(e)
-    try:
-        driver = get_neo4j_driver()
-        async with driver.session() as session:
-            res = await session.run("MATCH (c:Chunk) RETURN count(c) AS n")
-            r = await res.single()
-            out["l3_chunks"] = r["n"] if r else 0
-            res = await session.run("MATCH (e:Entity) RETURN count(e) AS n")
-            r = await res.single()
-            out["l3_entities"] = r["n"] if r else 0
-    except Exception as e:
-        out["l3_error"] = str(e)
-    return out
-
-# ---------------------------------------------------------------------------
-# Main
-# ---------------------------------------------------------------------------
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--port", type=int, default=8031)
-    parser.add_argument("--host", type=str, default="127.0.0.1")
-    args = parser.parse_args()
-
-    log.info(f"Starting Sequential HybridRAG Proxy (L1 System → L2 HybridRAG → L3 Graph → L4 Vector) on {args.host}:{args.port}")
-    uvicorn.run(app, host=args.host, port=args.port)