ctxora 6.2.0

package/src/harness_context/server.py

@@ -0,0 +1,1077 @@
+"""
+server.py — CTXORA MCP
+======================
+Local-first context engine for coding agents.
+
+MCP Tools exposed:
+  retrieve_context       — hybrid retrieval (semantic + BM25 + graph)
+  handoff_conversation   — no-compression history handoff
+  restore_conversation_handoff — restore a raw handoff payload
+  memory_save            — save to episodic / semantic / procedural memory
+  memory_search          — search across memory tiers
+  memory_inject          — inject relevant memory into prompt
+  memory_delete          — delete by key
+  memory_list            — list recent entries
+  memory_evict           — LRU eviction
+  memory_stats           — DB statistics
+  estimate_tokens        — token count utility
+  get_token_budget       — model-specific budget info
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import time
+from copy import deepcopy
+
+from mcp.server.fastmcp import FastMCP
+
+from chunking.compressor import ContextCompressor
+from chunking.treesitter_chunker import ASTChunker, Chunk, count_tokens
+from compact.handoff import DEFAULT_HANDOFF_THRESHOLD_TOKENS, ConversationHandoffStore
+from context.assembler import ContextAssembler
+from context.budgeting import get_budget
+from context.sanitizer import sanitize, sanitize_chunks
+from harness_context.engine import ContextEngine
+from harness_context.schemas import HarnessError
+from memory.episodic import EpisodicMemory, MemoryStore, ProceduralMemory, SemanticMemory
+from memory.vector_store import VectorStore
+from retrieval.bm25 import BM25Index
+from retrieval.cache import RetrievalCache
+from retrieval.embeddings import EmbeddingEngine
+from retrieval.graph import DependencyGraph
+from retrieval.reranker import Reranker
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+
+# ── Sub-modules ─────────────────────────────────────────────────────────
+
+
+# ── Logging ─────────────────────────────────────────────────────────────
+# Primary CTXORA MCP log.
+LOG_DIR = os.path.expanduser("~/.ctxora/logs")
+LOG_FILE = os.path.join(LOG_DIR, "ctxora-mcp.log")
+os.makedirs(LOG_DIR, exist_ok=True)
+
+# Legacy symlink: keep ~/.gemini/mcp-harness-v3.log pointing to the real file
+# so existing `tail -f` commands still work.
+_LEGACY_DIR = os.path.expanduser("~/.gemini")
+_LEGACY_LINK = os.path.join(_LEGACY_DIR, "mcp-harness-v3.log")
+try:
+    os.makedirs(_LEGACY_DIR, exist_ok=True)
+    if not os.path.exists(_LEGACY_LINK):
+        os.symlink(LOG_FILE, _LEGACY_LINK)
+except OSError:
+    pass  # non-critical — skip if symlink can't be created
+
+_log_handlers: list[logging.Handler] = [
+    logging.FileHandler(LOG_FILE, encoding="utf-8"),
+]
+
+# If a stale legacy file already exists and is not the primary log symlink,
+# write to it as well so older `tail -f ~/.gemini/...` commands keep working.
+try:
+    if os.path.exists(_LEGACY_LINK) and not os.path.samefile(LOG_FILE, _LEGACY_LINK):
+        _log_handlers.append(logging.FileHandler(_LEGACY_LINK, encoding="utf-8"))
+except OSError:
+    pass
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(levelname)s] %(message)s",
+    handlers=_log_handlers,
+)
+logger = logging.getLogger("ctxora.server")
+
+DEFAULT_MAX_PROMPT_TOKENS = 4_000
+DEFAULT_INCLUDE_REPORT = False
+
+
+def _coerce_target_ratio(target_ratio: float) -> float:
+    """Return a sane prompt target ratio in (0, 1]."""
+    try:
+        ratio = float(target_ratio)
+    except (TypeError, ValueError):
+        return 0.20
+    if ratio <= 0:
+        return 0.20
+    return min(ratio, 1.0)
+
+
+def _prompt_token_limit(
+    *,
+    context_window: int,
+    inject_budget: int,
+    target_ratio: float,
+    max_prompt_tokens: int,
+) -> int:
+    """Resolve the global prompt cap, never exceeding the inject budget."""
+    if max_prompt_tokens and max_prompt_tokens > 0:
+        return max(1, min(int(max_prompt_tokens), inject_budget))
+
+    ratio_limit = int(context_window * _coerce_target_ratio(target_ratio))
+    return max(1, min(ratio_limit, inject_budget))
+
+
+def _optimizer_report_comment(
+    *,
+    before_tokens: int,
+    after_tokens: int,
+    target_tokens: int,
+    chunks_before: int,
+    chunks_after: int,
+    memory_before: int,
+    memory_after: int,
+    status: str,
+) -> str:
+    saved_tokens = max(before_tokens - after_tokens, 0)
+    saved_percent = (
+        round(saved_tokens * 100 / before_tokens, 1)
+        if before_tokens > 0 else 0.0
+    )
+    return (
+        "<!-- prompt_optimizer "
+        f"status={status} before={before_tokens} after={after_tokens} "
+        f"saved={saved_tokens} saved_percent={saved_percent}% "
+        f"target={target_tokens} chunks={chunks_after}/{chunks_before} "
+        f"memory={memory_after}/{memory_before} -->"
+    )
+
+
+def _with_optimizer_report(
+    prompt: str,
+    *,
+    before_tokens: int,
+    target_tokens: int,
+    chunks_before: int,
+    chunks_after: int,
+    memory_before: int,
+    memory_after: int,
+    status: str,
+) -> str:
+    """Prepend a compact report and make its `after` token count exact."""
+    provisional = _optimizer_report_comment(
+        before_tokens=before_tokens,
+        after_tokens=count_tokens(prompt),
+        target_tokens=target_tokens,
+        chunks_before=chunks_before,
+        chunks_after=chunks_after,
+        memory_before=memory_before,
+        memory_after=memory_after,
+        status=status,
+    )
+    after_tokens = count_tokens(f"{provisional}\n{prompt}")
+    final = _optimizer_report_comment(
+        before_tokens=before_tokens,
+        after_tokens=after_tokens,
+        target_tokens=target_tokens,
+        chunks_before=chunks_before,
+        chunks_after=chunks_after,
+        memory_before=memory_before,
+        memory_after=memory_after,
+        status=status,
+    )
+    return f"{final}\n{prompt}"
+
+
+def _fit_prompt_to_limit(
+    *,
+    base_prompt: str,
+    chunks: list[Chunk],
+    memory_entries: list[dict],
+    query: str,
+    output_mode: str,
+    model: str,
+    target_tokens: int,
+    before_tokens: int,
+    include_report: bool,
+    base_prompt_policy: str,
+) -> tuple[str, list[Chunk], list[dict], str]:
+    """
+    Enforce the global prompt cap against the fully assembled output.
+
+    If retrieved context and memory cannot be trimmed enough because the base
+    prompt itself exceeds the cap, return a clear rejection unless callers opt
+    into unsafe base prompt truncation with base_prompt_policy="truncate".
+    """
+    selected_chunks = list(chunks)
+    selected_memory = list(memory_entries)
+    chunks_before = len(selected_chunks)
+    memory_before = len(selected_memory)
+
+    def render(status: str) -> tuple[str, int]:
+        prompt = assembler.assemble(
+            base_prompt=base_prompt,
+            chunks=selected_chunks,
+            memory_entries=selected_memory,
+            query=query,
+            output_mode=output_mode,
+            model=model,
+        )
+        if include_report:
+            prompt = _with_optimizer_report(
+                prompt,
+                before_tokens=before_tokens,
+                target_tokens=target_tokens,
+                chunks_before=chunks_before,
+                chunks_after=len(selected_chunks),
+                memory_before=memory_before,
+                memory_after=len(selected_memory),
+                status=status,
+            )
+        return prompt, count_tokens(prompt)
+
+    did_trim = False
+    prompt, tokens = render("ok")
+    while tokens > target_tokens and (selected_chunks or selected_memory):
+        did_trim = True
+        if selected_chunks:
+            selected_chunks.pop()
+        elif selected_memory:
+            selected_memory.pop()
+        prompt, tokens = render("trimmed")
+
+    if tokens <= target_tokens:
+        return prompt, selected_chunks, selected_memory, "trimmed" if did_trim else "ok"
+
+    if base_prompt_policy.lower() == "truncate":
+        return _fit_with_truncated_base_prompt(
+            base_prompt=base_prompt,
+            chunks=[],
+            memory_entries=[],
+            query=query,
+            output_mode=output_mode,
+            model=model,
+            target_tokens=target_tokens,
+            before_tokens=before_tokens,
+            include_report=include_report,
+            chunks_before=chunks_before,
+            memory_before=memory_before,
+        )
+
+    base_tokens = count_tokens(base_prompt)
+    rejection = (
+        "Error: prompt_optimizer rejected output because the base prompt alone "
+        "does not fit the global prompt target.\n"
+        f"base_prompt_tokens={base_tokens:,}; target_tokens={target_tokens:,}; "
+        "set a higher max_prompt_tokens/target_ratio, reduce base_prompt, or "
+        "use base_prompt_policy='truncate' if lossy truncation is acceptable."
+    )
+    if include_report:
+        rejection = _with_optimizer_report(
+            rejection,
+            before_tokens=before_tokens,
+            target_tokens=target_tokens,
+            chunks_before=chunks_before,
+            chunks_after=0,
+            memory_before=memory_before,
+            memory_after=0,
+            status="rejected_base_prompt",
+        )
+    return rejection, [], [], "rejected_base_prompt"
+
+
+def _fit_with_truncated_base_prompt(
+    *,
+    base_prompt: str,
+    chunks: list[Chunk],
+    memory_entries: list[dict],
+    query: str,
+    output_mode: str,
+    model: str,
+    target_tokens: int,
+    before_tokens: int,
+    include_report: bool,
+    chunks_before: int,
+    memory_before: int,
+) -> tuple[str, list[Chunk], list[dict], str]:
+    """Last-resort lossy base prompt truncation."""
+    marker = "\n\n[base_prompt truncated by prompt_optimizer]\n"
+    available = max(1, target_tokens - count_tokens(query) - 120)
+    while available > 0:
+        truncated_base = _truncate_to_tokens(base_prompt, available) + marker
+        prompt = assembler.assemble(
+            base_prompt=truncated_base,
+            chunks=chunks,
+            memory_entries=memory_entries,
+            query=query,
+            output_mode=output_mode,
+            model=model,
+        )
+        if include_report:
+            prompt = _with_optimizer_report(
+                prompt,
+                before_tokens=before_tokens,
+                target_tokens=target_tokens,
+                chunks_before=chunks_before,
+                chunks_after=0,
+                memory_before=memory_before,
+                memory_after=0,
+                status="truncated_base_prompt",
+            )
+        if count_tokens(prompt) <= target_tokens:
+            return prompt, [], [], "truncated_base_prompt"
+        available = int(available * 0.8)
+
+    rejection = (
+        "Error: prompt_optimizer could not fit even the truncated base prompt "
+        f"under target_tokens={target_tokens:,}."
+    )
+    return rejection, [], [], "rejected_base_prompt"
+
+
+def _truncate_to_tokens(text: str, max_tokens: int) -> str:
+    """Token-aware truncation with a character fallback."""
+    if count_tokens(text) <= max_tokens:
+        return text
+    try:
+        import tiktoken as _tiktoken
+        enc = _tiktoken.get_encoding("cl100k_base")
+        return enc.decode(enc.encode(text, disallowed_special=())[:max_tokens])
+    except Exception:
+        return text[:max_tokens * 3]
+
+
+# ── Singletons ──────────────────────────────────────────────────────────
+mcp = FastMCP("CTXORA MCP")
+embedder = EmbeddingEngine()
+bm25 = BM25Index()
+reranker = Reranker()
+graph = DependencyGraph()
+cache = RetrievalCache(ttl=600)
+chunker = ASTChunker()
+compressor = ContextCompressor()
+handoff_store = ConversationHandoffStore()
+vec_store = VectorStore()
+assembler = ContextAssembler()
+mem_store = MemoryStore.instance()
+episodic = EpisodicMemory(mem_store)
+semantic = SemanticMemory(mem_store)
+procedural = ProceduralMemory(mem_store)
+context_engine = ContextEngine()
+
+# ── In-memory chunk index ───────────────────────────────────────────────
+_indexed_paths: set[str] = set()
+_indexed_fingerprints: dict[str, str] = {}
+_all_chunks: list[Chunk] = []
+
+
+def _normalise_path(path: str) -> str:
+    return os.path.abspath(path)
+
+
+def _chunk_belongs_to_path(chunk: Chunk, source_path: str) -> bool:
+    """Return whether a chunk must be replaced when source_path is reindexed."""
+    chunk_path = _normalise_path(chunk.path)
+    root = _normalise_path(source_path)
+    if os.path.isdir(root) or any(
+        _normalise_path(existing.path).startswith(f"{root}{os.sep}")
+        for existing in _all_chunks
+    ):
+        try:
+            return os.path.commonpath([chunk_path, root]) == root
+        except ValueError:
+            return False
+    return chunk_path == root
+
+
+def _chunks_for_paths(chunks: list[Chunk], paths: list[str]) -> list[Chunk]:
+    """Keep compatibility retrieval scoped to the current request's paths."""
+    return [chunk for chunk in chunks if any(_chunk_belongs_to_path(chunk, path) for path in paths)]
+
+
+def _rebuild_indexes() -> None:
+    """Rebuild every derived index from the canonical chunk list atomically."""
+    texts = [chunk.content[:512] for chunk in _all_chunks]
+    vectors = embedder.rebuild(texts)
+    for chunk, vector in zip(_all_chunks, vectors):
+        chunk.embedding = vector
+    vec_store.rebuild(_all_chunks)
+    bm25.build(_all_chunks)
+    graph.build(_all_chunks)
+
+
+def _ensure_indexed(
+        paths: list[str],
+        force_reindex: bool = False) -> list[Chunk]:
+    """
+    Build/update the chunk index for the given paths.
+    Skips paths already indexed unless force_reindex=True.
+    """
+    new_paths = []
+    for path in paths:
+        normalised = _normalise_path(path)
+        fingerprint = cache.path_fingerprint(path)
+        if (
+            force_reindex
+            or normalised not in _indexed_paths
+            or _indexed_fingerprints.get(normalised) != fingerprint
+        ):
+            new_paths.append(path)
+    if not new_paths and _all_chunks:
+        return _all_chunks
+
+    t0 = time.time()
+    chunks = chunker.chunk_paths(new_paths)
+    sanitize_chunks(chunks)
+
+    # Remove stale chunks before adding replacements. This handles changed and
+    # deleted files without accumulating duplicate UUID-based chunk records.
+    _all_chunks[:] = [
+        chunk for chunk in _all_chunks
+        if not any(_chunk_belongs_to_path(chunk, path) for path in new_paths)
+    ]
+    _all_chunks.extend(chunks)
+    _rebuild_indexes()
+
+    for path in new_paths:
+        normalised = _normalise_path(path)
+        _indexed_paths.add(normalised)
+        _indexed_fingerprints[normalised] = cache.path_fingerprint(path)
+    cache.invalidate_all()
+    logger.info(
+        f"[Server] refreshed {len(chunks)} chunks in "
+        f"{time.time() - t0:.1f}s | total={len(_all_chunks)}")
+    return _all_chunks
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# MCP TOOLS
+# ══════════════════════════════════════════════════════════════════════════════
+
+@mcp.tool(name="retrieve_context_legacy")
+def retrieve_context_legacy(
+    base_prompt: str,
+    paths: list[str],
+    query: str,
+    model: str = "claude-sonnet-4",
+    output_mode: str = "concise",
+    retrieve_top_n: int = 50,
+    rerank_top_k: int = 12,
+    memory_top_k: int = 4,
+    min_memory_sim: float = 0.18,
+    graph_expand: bool = True,
+    compress: bool = True,
+    force_reindex: bool = False,
+    target_ratio: float = 0.20,
+    max_prompt_tokens: int = DEFAULT_MAX_PROMPT_TOKENS,
+    include_report: bool = DEFAULT_INCLUDE_REPORT,
+    base_prompt_policy: str = "reject",
+) -> str:
+    """
+    DEPRECATED compatibility tool — assembled prompt retrieval pipeline.
+
+    Pipeline: index → hybrid score → rerank → graph expand → compress → assemble
+    → enforce global prompt budget.
+
+    :param base_prompt:    System prompt to build on.
+    :param paths:          File/folder paths to index and retrieve from.
+    :param query:          Current user task or question.
+    :param model:          Target model (affects token budget + output format).
+    :param output_mode:    "concise" | "structured" | "code_only" | "minimal"
+    :param retrieve_top_n: Candidate pool before reranking (default 50).
+    :param rerank_top_k:   Final chunks after reranking (default 12).
+    :param memory_top_k:   Memory entries to inject (default 4).
+    :param min_memory_sim: Minimum similarity for memory injection.
+    :param graph_expand:   Expand retrieval via dependency graph.
+    :param compress:       Compress long chunks before injecting.
+    :param force_reindex:  Force re-chunking even for already-indexed paths.
+    :param target_ratio:   Max share of model context window for final prompt.
+                            Ignored when max_prompt_tokens > 0.
+    :param max_prompt_tokens: Absolute final prompt cap. Defaults to 4,000;
+                              pass 0 to use target_ratio instead.
+    :param include_report: Prepend token telemetry only when explicitly enabled.
+    :param base_prompt_policy: "reject" or "truncate" if base_prompt alone
+                               exceeds the global cap.
+    """
+    try:
+        budget = get_budget(model)
+        target_tokens = _prompt_token_limit(
+            context_window=budget.context_window,
+            inject_budget=budget.inject_budget,
+            target_ratio=target_ratio,
+            max_prompt_tokens=max_prompt_tokens,
+        )
+        logger.info(
+            f"model={model!r} budget={budget.context_window:,} "
+            f"inject={budget.inject_budget:,} target={target_tokens:,}")
+        logger.info(f"retrieve_context | query={query[:100]!r} | paths={paths}")
+
+        # ── 1. Refresh the canonical index before looking up cached selections.
+        # A source fingerprint change invalidates cache and refreshes its chunks.
+        all_chunks = _chunks_for_paths(_ensure_indexed(paths, force_reindex), paths)
+        cache_options = {
+            "retrieve_top_n": retrieve_top_n,
+            "rerank_top_k": rerank_top_k,
+            "graph_expand": graph_expand,
+        }
+        cached = None if force_reindex else cache.get(query, paths, cache_options)
+        if cached:
+            logger.info(f"  [Cache HIT] {len(cached)} chunks")
+            final_chunks = cached
+        else:
+            # ── 2. Retrieve from the fresh index ──────────────────────────────
+            if not all_chunks:
+                logger.info("  No chunks indexed from provided paths")
+                final_chunks = []
+            else:
+                # ── 3. Hybrid scoring ────────────────────────────────────────
+                query_vec = embedder.embed_text(query)
+                vec_results = vec_store.search(query_vec, top_k=max(retrieve_top_n, len(all_chunks)))
+                bm25_scores = bm25.score(query)
+
+                # Build chunk lookup by id
+                chunk_by_id = {c.id: c for c in all_chunks}
+                bm25_by_idx = {c.id: s for c, s in zip(bm25.chunks, bm25_scores)}
+
+                requested_ids = {chunk.id for chunk in all_chunks}
+                candidates_by_id: dict[str, Chunk] = {}
+                for r in vec_results:
+                    c = chunk_by_id.get(r["id"])
+                    if c is None or c.id not in requested_ids:
+                        continue
+
+                    sem_score = float(r.get("score", 0))
+                    bm25_score = bm25_by_idx.get(c.id, 0.0)
+                    rec_score = 0.0    # recency not tracked per chunk
+                    prio_score = c.priority / 10.0
+                    grph_score = graph.symbol_score(query, c)
+
+                    final = (
+                        sem_score * 0.55 +
+                        bm25_score * 0.20 +
+                        rec_score * 0.10 +
+                        prio_score * 0.10 +
+                        grph_score * 0.05
+                    )
+                    c._score = final   # type: ignore[attr-defined]
+                    candidates_by_id[c.id] = c
+
+                # Keep a lexical-only candidate from being excluded by a
+                # semantic pre-filter. Both pools are fused before reranking.
+                lexical_order = sorted(
+                    (chunk for chunk in bm25.chunks if chunk.id in requested_ids),
+                    key=lambda chunk: bm25_by_idx.get(chunk.id, 0.0),
+                    reverse=True,
+                )[:retrieve_top_n]
+                for rank, c in enumerate(lexical_order, 1):
+                    if c.id not in candidates_by_id:
+                        c._score = bm25_by_idx.get(c.id, 0.0) / (60 + rank)  # type: ignore[attr-defined]
+                        candidates_by_id[c.id] = c
+
+                candidates = list(candidates_by_id.values())
+
+                candidates.sort(
+                    key=lambda x: getattr(
+                        x, "_score", 0), reverse=True)
+
+                # ── 4. Rerank top-N ──────────────────────────────────────────
+                reranked = reranker.rerank(
+                    query, candidates[:retrieve_top_n], top_k=rerank_top_k)
+
+                # ── 5. Graph expansion ───────────────────────────────────────
+                if graph_expand:
+                    extras = graph.expand(reranked, max_extra=4)
+                    for e in extras:
+                        if e.id not in {c.id for c in reranked}:
+                            e._score = 0.3   # type: ignore[attr-defined]
+                            reranked.append(e)
+
+                # Cache raw copies. Compression below is presentation-specific
+                # and must never mutate the source index or a cache entry.
+                cache.set(query, paths, reranked, cache_options)
+                final_chunks = deepcopy(reranked)
+
+        # ── 6. Compress candidates (before budget enforcement) ───────────────
+        # Compress first so budget is enforced on ACTUAL output token counts,
+        # not inflated pre-compression counts.
+        if compress:
+            compressed_strings = compressor.compress(final_chunks, model=model)
+            for c, cs in zip(final_chunks, compressed_strings):
+                c.content = cs
+                c.tokens = count_tokens(cs)  # re-count post-compression
+
+        # ── 7. Memory retrieval ──────────────────────────────────────────────
+        memory_entries = mem_store.search(
+            query, top_k=memory_top_k, min_sim=min_memory_sim)
+        logger.info(f"  {len(memory_entries)} memory entries injected")
+
+        # ── 8. Assemble once without trimming for before/after telemetry ─────
+        before_prompt = assembler.assemble(
+            base_prompt=base_prompt,
+            chunks=final_chunks,
+            memory_entries=memory_entries,
+            query=query,
+            output_mode=output_mode,
+            model=model,
+        )
+        before_tokens = count_tokens(before_prompt)
+
+        # ── 9. Enforce global prompt budget on the actual assembled output ───
+        result, selected_chunks, selected_memory, status = _fit_prompt_to_limit(
+            base_prompt=base_prompt,
+            chunks=final_chunks,
+            memory_entries=memory_entries,
+            query=query,
+            output_mode=output_mode,
+            model=model,
+            target_tokens=target_tokens,
+            before_tokens=before_tokens,
+            include_report=include_report,
+            base_prompt_policy=base_prompt_policy,
+        )
+        after_tokens = count_tokens(result)
+        saved_tokens = max(before_tokens - after_tokens, 0)
+        saved_percent = (
+            round(saved_tokens * 100 / before_tokens, 1)
+            if before_tokens > 0 else 0.0
+        )
+        logger.info(
+            "  prompt_optimizer | "
+            f"status={status} before={before_tokens:,} after={after_tokens:,} "
+            f"saved={saved_tokens:,} saved_percent={saved_percent}% "
+            f"target={target_tokens:,} chunks={len(selected_chunks)}/{len(final_chunks)} "
+            f"memory={len(selected_memory)}/{len(memory_entries)}"
+        )
+        return result
+
+    except Exception as e:
+        logger.error(f"  ❌ {e}", exc_info=True)
+        return f"Error in retrieve_context: {e}"
+
+
+@mcp.tool()
+def handoff_conversation(
+    messages_json: str,
+    threshold_tokens: int = DEFAULT_HANDOFF_THRESHOLD_TOKENS,
+    label: str = "",
+    workspace_id: str = "legacy_global",
+    retention_seconds: int = 604_800,
+    consent: bool = True,
+) -> str:
+    """
+    Prepare an automatic no-compression handoff when history is too long.
+
+    The original provider-format JSON is stored unchanged only after the token
+    threshold is reached. The MCP client creates the fresh task and can call
+    restore_conversation_handoff with the returned ID when it needs history.
+
+    :param messages_json: Original OpenAI, Anthropic, or Gemini message array.
+    :param threshold_tokens: Handoff threshold; defaults to the 30,000-token
+                             project session budget.
+    :param label: Optional client-visible label for the handoff record.
+    """
+    try:
+        result = handoff_store.prepare(messages_json, threshold_tokens, label, workspace_id, retention_seconds, consent)
+        logger.info(
+            "handoff_conversation | "
+            f"action={result['action']} tokens={result['history_tokens']:,} "
+            f"threshold={result['threshold_tokens']:,}"
+        )
+        return json.dumps(result, ensure_ascii=False)
+    except Exception as e:
+        logger.error(f"  ❌ {e}")
+        return f"Error: {e}"
+
+
+def _v2_result(operation):
+    try:
+        return operation()
+    except HarnessError as error:
+        raise ValueError(json.dumps({"code": error.code, "message": str(error)})) from error
+
+
+@mcp.tool()
+def register_workspace(
+    workspace_id: str,
+    roots: list[str],
+    max_files: int = 10_000,
+    max_file_bytes: int = 2_000_000,
+    max_total_bytes: int = 200_000_000,
+) -> dict:
+    """Register canonical allowed roots and indexing limits."""
+    return _v2_result(lambda: context_engine.register_workspace(
+        workspace_id, roots, max_files=max_files, max_file_bytes=max_file_bytes,
+        max_total_bytes=max_total_bytes,
+    ))
+
+
+@mcp.tool()
+def refresh_workspace(workspace_id: str, paths: list[str] | None = None) -> dict:
+    """Incrementally refresh authorized chunks, embeddings and code graph."""
+    return _v2_result(lambda: context_engine.refresh_workspace(workspace_id, paths))
+
+
+@mcp.tool()
+def plan_context(
+    workspace_id: str,
+    query: str,
+    available_input_tokens: int,
+    strategy_override: str = "",
+) -> dict:
+    """Choose an explainable, overridable context strategy."""
+    return _v2_result(lambda: context_engine.plan_context(
+        workspace_id, query, available_input_tokens, strategy_override,
+    ))
+
+
+@mcp.tool()
+def retrieve_context(
+    workspace_id: str,
+    query: str,
+    top_k: int = 12,
+    graph_expand: bool = True,
+    token_budget: int = 4_000,
+) -> dict:
+    """Return structured provider-neutral evidence and coverage diagnostics."""
+    return _v2_result(lambda: context_engine.retrieve_context(
+        workspace_id, query, top_k, graph_expand, token_budget,
+    ))
+
+
+@mcp.tool()
+def prepare_context(
+    workspace_id: str,
+    query: str,
+    available_input_tokens: int,
+    strategy_override: str = "",
+) -> dict:
+    """Execute the selected CAG, RAG, long-context or graph strategy."""
+    return _v2_result(lambda: context_engine.prepare_context(
+        workspace_id, query, available_input_tokens, strategy_override,
+    ))
+
+
+@mcp.tool()
+def context_stats(workspace_id: str) -> dict:
+    """Return workspace-scoped index, graph, bundle and token statistics."""
+    return _v2_result(lambda: context_engine.stats(workspace_id))
+
+
+@mcp.tool()
+def invalidate_context(workspace_id: str, target: str = "all") -> dict:
+    """Invalidate one workspace index or bundle store."""
+    return _v2_result(lambda: context_engine.invalidate(workspace_id, target))
+
+
+@mcp.tool()
+def restore_conversation_handoff(handoff_id: str, workspace_id: str = "legacy_global") -> str:
+    """Restore an explicitly requested, unmodified conversation handoff."""
+    try:
+        result = handoff_store.restore(handoff_id, workspace_id)
+        if result is None:
+            return f"Error: handoff not found: {handoff_id}"
+        return json.dumps(result, ensure_ascii=False)
+    except Exception as e:
+        logger.error(f"  ❌ {e}")
+        return f"Error: {e}"
+
+@mcp.tool()
+def list_conversation_handoffs(workspace_id: str = "legacy_global", limit: int = 30) -> str:
+    """List auditable handoffs for one workspace."""
+    return json.dumps(handoff_store.list(workspace_id, limit), ensure_ascii=False)
+
+@mcp.tool()
+def delete_conversation_handoff(handoff_id: str, workspace_id: str = "legacy_global") -> str:
+    """Delete one handoff without affecting other workspaces."""
+    return json.dumps({"deleted": handoff_store.delete(handoff_id, workspace_id), "handoff_id": handoff_id})
+
+@mcp.tool()
+def purge_expired_handoffs() -> str:
+    """Purge handoffs past their configured retention window."""
+    return json.dumps({"purged": handoff_store.purge_expired()})
+
+
+# ── Memory tools ────────────────────────────────────────────────────────
+
+@mcp.tool()
+def memory_save(
+    key: str,
+    value: str,
+    mtype: str = "semantic",
+    tags: str = "",
+    workspace_id: str = "legacy_global",
+    scope: str = "workspace",
+    source: str = "user",
+    confidence: float = 1.0,
+    expires_at: float | None = None,
+) -> str:
+    """
+    Save knowledge to long-term memory.
+
+    :param key:   Short identifying key (used for search).
+    :param value: Content to remember.
+    :param mtype: Memory tier — "episodic" | "semantic" | "procedural"
+    :param tags:  Comma-separated labels for filtering.
+    """
+    try:
+        value = sanitize(value, escape_xml=False)
+        from memory.episodic import MemoryType
+        result = mem_store.save(
+            MemoryType(mtype), key, value, tags, workspace_id=workspace_id,
+            scope=scope, source=source, confidence=confidence, expires_at=expires_at,
+        )
+        return json.dumps(result, ensure_ascii=False)
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_search(
+    query: str,
+    mtype: str = "",
+    top_k: int = 5,
+    min_sim: float = 0.12,
+    workspace_id: str = "legacy_global",
+) -> str:
+    """
+    Search long-term memory by semantic similarity.
+
+    :param query:   Search query.
+    :param mtype:   Filter by tier: "episodic" | "semantic" | "procedural" | "" (all)
+    :param top_k:   Max results.
+    :param min_sim: Minimum similarity threshold (0–1).
+    """
+    try:
+        from memory.episodic import MemoryType
+        mt = None
+        if mtype:
+            try:
+                mt = MemoryType(mtype)
+            except ValueError as error:
+                raise ValueError(f"invalid memory type: {mtype}") from error
+        results = mem_store.search(
+            query, mtype=mt, top_k=top_k, min_sim=min_sim,
+            workspace_id=workspace_id)
+        return json.dumps(results, ensure_ascii=False)
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_inject(
+    base_prompt: str,
+    query: str,
+    top_k: int = 4,
+    min_sim: float = 0.18,
+    output_mode: str = "concise",
+    model: str = "claude-sonnet-4",
+    target_ratio: float = 0.20,
+    max_prompt_tokens: int = DEFAULT_MAX_PROMPT_TOKENS,
+    include_report: bool = DEFAULT_INCLUDE_REPORT,
+    base_prompt_policy: str = "reject",
+    workspace_id: str = "legacy_global",
+) -> str:
+    """
+    Inject relevant memory entries into a system prompt (without file retrieval).
+    Use when you only need memory context, not workspace files.
+
+    :param base_prompt:  System prompt.
+    :param query:        Current task.
+    :param top_k:        Max memory entries.
+    :param min_sim:      Minimum similarity.
+    :param output_mode:  "concise" | "structured" | "code_only" | "minimal"
+    :param model:        Target model for format selection.
+    :param target_ratio: Max share of model context window for final prompt.
+    :param max_prompt_tokens: Absolute final prompt cap. Defaults to 4,000;
+                              pass 0 to use target_ratio instead.
+    :param include_report: Prepend token telemetry only when explicitly enabled.
+    :param base_prompt_policy: "reject" or "truncate" for oversized base_prompt.
+    """
+    try:
+        entries = mem_store.search(query, top_k=top_k, min_sim=min_sim, workspace_id=workspace_id)
+        budget = get_budget(model)
+        target_tokens = _prompt_token_limit(
+            context_window=budget.context_window,
+            inject_budget=budget.inject_budget,
+            target_ratio=target_ratio,
+            max_prompt_tokens=max_prompt_tokens,
+        )
+        before_prompt = assembler.assemble(
+            base_prompt=base_prompt, chunks=[], memory_entries=entries,
+            query=query, output_mode=output_mode, model=model,
+        )
+        result, _, _, status = _fit_prompt_to_limit(
+            base_prompt=base_prompt,
+            chunks=[],
+            memory_entries=entries,
+            query=query,
+            output_mode=output_mode,
+            model=model,
+            target_tokens=target_tokens,
+            before_tokens=count_tokens(before_prompt),
+            include_report=include_report,
+            base_prompt_policy=base_prompt_policy,
+        )
+        logger.info(
+            "  memory_inject optimizer | "
+            f"status={status} after={count_tokens(result):,} "
+            f"target={target_tokens:,}"
+        )
+        return result
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_delete(key: str, mtype: str = "semantic", workspace_id: str = "legacy_global") -> str:
+    """Delete a memory entry by exact key and type."""
+    try:
+        from memory.episodic import MemoryType
+        mt = MemoryType(mtype)
+        deleted = mem_store.delete(mt, key, workspace_id=workspace_id)
+        return json.dumps({"deleted": deleted, "key": key, "mtype": mtype})
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_list(mtype: str = "", limit: int = 30, workspace_id: str = "legacy_global") -> str:
+    """List recent memory entries, optionally filtered by type."""
+    try:
+        from memory.episodic import MemoryType
+        mt = MemoryType(mtype) if mtype else None
+        return json.dumps(mem_store.list_keys(mt, limit, workspace_id), ensure_ascii=False)
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_evict(keep_top: int = 300) -> str:
+    """Evict least-recently-used memory entries to free space."""
+    try:
+        before = mem_store.stats()
+        removed = mem_store.evict_lru(keep_top)
+        after = mem_store.stats()
+        return json.dumps(
+            {"removed": removed, "before": before, "after": after})
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def memory_stats() -> str:
+    """Return statistics per memory tier (count, tokens, hits)."""
+    try:
+        return json.dumps(mem_store.stats(), ensure_ascii=False)
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def estimate_tokens(text: str) -> str:
+    """
+    Estimate token count for any text.
+    Useful for checking prompt size before sending to model.
+    """
+    tok = count_tokens(text)
+    chars = len(text)
+    # Representative models across all supported providers
+    _benchmark_models = [
+        # Claude
+        "claude-sonnet-4",
+        "claude-opus-4",
+        # Gemini
+        "gemini-2.5-pro",
+        # OpenAI GPT-5
+        "gpt-5.5",
+        "gpt-5.4",
+        "gpt-5",
+        # OpenAI Chat
+        "gpt-4o",
+        "gpt-4.1",
+        "gpt-4o-mini",
+        # OpenAI Reasoning
+        "o3",
+        "o1",
+        "o4-mini",
+        # OpenAI Codex
+        "codex-mini-latest",
+    ]
+    budgets = {
+        m: get_budget(m).inject_budget
+        for m in _benchmark_models
+    }
+    return json.dumps({
+        "tokens": tok,
+        "chars": chars,
+        "ratio_chars_per_tok": round(chars / tok, 2) if tok > 0 else 0,
+        "fits_in": {
+            m: tok <= b
+            for m, b in budgets.items()
+        },
+    }, ensure_ascii=False)
+
+
+@mcp.tool()
+def get_token_budget(model: str = "claude-sonnet-4") -> str:
+    """
+    Return the dynamic token budget for a specific model.
+    Shows context window, reserved headroom, and available inject budget.
+    """
+    try:
+        b = get_budget(model)
+        return json.dumps({
+            "model": b.model_name,
+            "context_window": b.context_window,
+            "reserved_output": b.reserved_output,
+            "reserved_reasoning": b.reserved_reasoning,
+            "reserved_tools": b.reserved_tools,
+            "inject_budget": b.inject_budget,
+        }, ensure_ascii=False)
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def invalidate_cache() -> str:
+    """Clear the retrieval cache. Call when workspace files have changed."""
+    cache.invalidate_all()
+    return json.dumps({"status": "cache cleared"})
+
+
+@mcp.tool()
+def reindex_paths(paths: list[str]) -> str:
+    """
+    Force re-chunking and re-indexing of given paths.
+    Use after significant file edits to refresh the vector index.
+    """
+    logger.info(f"[Server] force reindex: {paths}")
+    try:
+        chunks = _ensure_indexed(paths, force_reindex=True)
+        return json.dumps({"status": "reindexed", "total_chunks": len(chunks)})
+    except Exception as e:
+        return f"Error: {e}"
+
+
+# ── Entrypoint ──────────────────────────────────────────────────────────
+
+def main() -> None:
+    logger.info("═" * 70)
+    logger.info("CTXORA MCP starting")
+    logger.info(f"  Log (primary) : {LOG_FILE}")
+    logger.info(f"  Log (legacy)  : {_LEGACY_LINK}  → symlink")
+    logger.info(f"  tail -f {LOG_FILE}")
+    logger.info("  Tools  : retrieve_context · handoff_conversation")
+    logger.info("           restore_conversation_handoff")
+    logger.info("           memory_save · memory_search · memory_inject")
+    logger.info(
+        "           memory_delete · memory_list · memory_evict · memory_stats")
+    logger.info("           estimate_tokens · get_token_budget")
+    logger.info("           invalidate_cache · reindex_paths")
+    logger.info(
+        "  Models : Claude · Gemini · GPT-4o/4.1/4.5 · o1/o3/o4 · Codex")
+    logger.info(
+        "  Stack  : numpy vector store + BM25 + AST chunker")
+    logger.info(
+        "           local reranker + dependency graph + safety layer")
+    logger.info("═" * 70)
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()