loki-mode 7.21.0 → 7.23.0

package/mcp/server.py

@@ -22,6 +22,7 @@ import logging
 import threading
 import uuid
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Optional, List, Dict, Any
 
 # Add parent directory to path for imports
@@ -1517,6 +1518,67 @@ CHROMA_HOST = os.environ.get("LOKI_CHROMA_HOST", "localhost")
 CHROMA_PORT = int(os.environ.get("LOKI_CHROMA_PORT", "8100"))
 CHROMA_COLLECTION = os.environ.get("LOKI_CHROMA_COLLECTION", "loki-codebase")
 
+# Code-index freshness manifest (written by tools/index-codebase.py). Resolved
+# relative to the repo root the same way the indexer resolves it, so the two
+# agree on a single location. mcp/server.py -> parent.parent == repo root.
+_CODE_INDEX_REPO_ROOT = Path(__file__).resolve().parent.parent
+CODE_INDEX_MANIFEST_PATH = _CODE_INDEX_REPO_ROOT / ".loki" / "state" / "code-index-manifest.json"
+
+
+def _code_index_staleness() -> dict:
+    """Compare the code-index manifest mtimes against current files on disk.
+
+    Self-contained (no import of tools/index-codebase.py, which loads chromadb
+    at module top under a possibly-different Python). Mirrors the mtime
+    staleness pattern in memory/retrieval.py. A missing manifest degrades to
+    not-stale so the happy path on a fresh repo is unaffected.
+
+    Returns {"stale": bool, "stale_files": int}.
+    """
+    try:
+        data = json.loads(CODE_INDEX_MANIFEST_PATH.read_text())
+        files = data.get("files", {})
+        if not isinstance(files, dict) or not files:
+            return {"stale": False, "stale_files": 0}
+    except Exception:
+        return {"stale": False, "stale_files": 0}
+
+    stale = 0
+    for rel, entry in files.items():
+        abs_path = _CODE_INDEX_REPO_ROOT / rel
+        try:
+            if not abs_path.exists():
+                stale += 1
+            elif os.path.getmtime(abs_path) != entry.get("mtime"):
+                stale += 1
+        except OSError:
+            stale += 1
+    return {"stale": stale > 0, "stale_files": stale}
+
+
+def _maybe_autoreindex_code() -> None:
+    """Opt-in incremental re-index when the manifest is stale.
+
+    Gated behind LOKI_CODE_INDEX_AUTOREINDEX=1 because embeddings cost compute.
+    Default behavior is warn-if-stale (the tools just report the staleness
+    fields). Best-effort: never raises into the caller.
+    """
+    if os.environ.get("LOKI_CODE_INDEX_AUTOREINDEX", "0") != "1":
+        return
+    if not _code_index_staleness().get("stale"):
+        return
+    try:
+        import subprocess
+        indexer = _CODE_INDEX_REPO_ROOT / "tools" / "index-codebase.py"
+        py = "/opt/homebrew/bin/python3.12"
+        if not Path(py).exists():
+            py = sys.executable
+        subprocess.run([py, str(indexer), "--changed"],
+                       cwd=str(_CODE_INDEX_REPO_ROOT),
+                       capture_output=True, timeout=300)
+    except Exception as e:
+        logger.warning(f"Auto-reindex (LOKI_CODE_INDEX_AUTOREINDEX) failed: {e}")
+
 
 def _get_chroma_collection():
     """Get or create ChromaDB collection (lazy connection).
@@ -1581,6 +1643,10 @@ async def loki_code_search(
                                        'language': language, 'file_filter': file_filter,
                                        'type_filter': type_filter})
 
+    # Warn-if-stale (default) or opt-in auto-reindex before querying.
+    _maybe_autoreindex_code()
+    _staleness = _code_index_staleness()
+
     collection = _get_chroma_collection()
     if collection is None:
         return json.dumps({
@@ -1637,7 +1703,13 @@ async def loki_code_search(
 
         _emit_tool_event_async('loki_code_search', 'complete',
                                result_status='success', result_count=len(output))
-        return json.dumps({"query": query, "results": output, "total": len(output)})
+        return json.dumps({
+            "query": query,
+            "results": output,
+            "total": len(output),
+            "stale": _staleness["stale"],
+            "stale_files": _staleness["stale_files"],
+        })
 
     except Exception as e:
         logger.error(f"Code search failed: {e}")
@@ -1659,6 +1731,8 @@ async def loki_code_search_stats() -> str:
     Shows total chunks, files indexed, breakdown by language and type.
     Useful for verifying the index is up to date.
     """
+    _staleness = _code_index_staleness()
+
     collection = _get_chroma_collection()
     if collection is None:
         return json.dumps({"error": "ChromaDB not available"})
@@ -1674,6 +1748,8 @@ async def loki_code_search_stats() -> str:
                 "by_language": {},
                 "by_type": {},
                 "reindex_command": "python3.12 tools/index-codebase.py --reset",
+                "stale": _staleness["stale"],
+                "stale_files": _staleness["stale_files"],
             })
 
         results = collection.get(limit=count, include=["metadatas"])
@@ -1694,6 +1770,8 @@ async def loki_code_search_stats() -> str:
             "by_language": langs,
             "by_type": types,
             "reindex_command": "python3.12 tools/index-codebase.py --reset",
+            "stale": _staleness["stale"],
+            "stale_files": _staleness["stale_files"],
         })
     except Exception as e:
         logger.error(f"Code search stats failed: {e}")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "loki-mode",
-  "version": "7.21.0",
+  "version": "7.23.0",
   "description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 11 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
   "keywords": [
     "agent",

package/references/mcp-integration.md

@@ -147,6 +147,71 @@ See `skills/documentation.md` for documentation generation using Repowise.
 
 ---
 
+## Built-in Hybrid Codebase Search (loki_code_search)
+
+Loki Mode ships its own codebase search that does not require any third-party MCP
+server. It is exposed both as MCP tools (`loki_code_search`,
+`loki_code_search_stats`) and as a CLI subcommand (`loki code search`). It combines
+lexical and semantic retrieval over the indexed codebase:
+
+- Lexical: ripgrep / grep (with a python scan fallback)
+- Semantic: ChromaDB over the `loki-codebase` collection
+- Fusion: reciprocal rank fusion (RRF) merges the two ranked lists
+- Truncation: results are deduped by file:line and trimmed to a token budget
+  (greedy, highest fused score first)
+
+When ChromaDB or its docker container is unreachable, search degrades to grep-only
+so it still returns results instead of erroring. The implementation lives in
+`tools/hybrid_search.py`.
+
+### Index freshness and staleness reporting
+
+The semantic index is backed by a manifest at
+`.loki/state/code-index-manifest.json`. The MCP tools (`loki_code_search` and
+`loki_code_search_stats`) compare the manifest against the files on disk and report
+two fields in their JSON output:
+
+- `stale`: boolean, true when one or more indexed files have changed since the last
+  index
+- `stale_files`: count of changed files
+
+Staleness is computed from the manifest alone (no ChromaDB call), and a missing
+manifest degrades to not-stale so a fresh repo is unaffected.
+
+Default behavior is warn-if-stale: the tools report staleness but do not re-index.
+Set `LOKI_CODE_INDEX_AUTOREINDEX=1` to opt into an automatic incremental re-index
+before querying (off by default because embeddings cost compute). The incremental
+re-index is driven by `tools/index-codebase.py --changed`, which re-chunks only
+files whose mtime or sha1 differ from the manifest, upserts the new chunks, deletes
+orphaned chunk IDs for changed files, and drops chunks for files removed from disk.
+
+### CLI usage: `loki code search`
+
+```bash
+loki code search "rate limit backoff"            # hybrid grep + semantic
+loki code search "council vote" --grep-only      # lexical only
+loki code search "memory retrieval" --semantic-only --top 15
+loki code search "build prompt" --budget 4000    # widen the token budget
+loki code search "save state" --json             # machine-readable output
+```
+
+Flags (see `loki code search --help`):
+
+| Flag | Default | Effect |
+|------|---------|--------|
+| `--grep-only` | off | lexical search only (skip semantic) |
+| `--semantic-only` | off | semantic search only (skip grep) |
+| `--budget N` | 3000 | token budget for the merged result set |
+| `--top N` | 10 | maximum number of results |
+| `--json` | off | emit JSON instead of formatted output |
+
+`loki code search` is one of the `loki code` codebase-intelligence subcommands
+(alongside `overview`, `symbols`, `deps`, `hotspots`, and `diff`). It falls back to
+grep-only when ChromaDB is unreachable, and requires python3.12 for the semantic
+path because that is what the chromadb client needs on this stack.
+
+---
+
 ## MCP Configuration Location
 
 Claude Code reads MCP configuration from:

package/skills/00-index.md

@@ -107,6 +107,8 @@
 - Inter-stream communication via signals
 - Auto-merge completed features
 - Orchestrator state management
+- Supervisor / judge pattern (CONTINUE / COMPLETE / ESCALATE / PIVOT)
+- Dynamic resource-aware session concurrency (LOKI_DYNAMIC_CONCURRENCY)
 
 ### github-integration.md
 **When:** Working with GitHub issues, creating PRs, syncing status

package/skills/parallel-workflows.md

@@ -128,6 +128,102 @@ orchestrator_workflow:
 
 ---
 
+## Supervisor / Judge Pattern
+
+A supervisor (or judge) is a decision step the orchestrator runs at milestones to
+decide whether the parallel run should keep going, wrap up, ask a human, or change
+course. It is a pattern, not a separate daemon: the orchestrator already makes this
+call at completion checkpoints. The judge verbs below are adopted from Cursor's
+multi-agent learnings (see `references/cursor-learnings.md`).
+
+### When the supervisor runs
+
+- After a major milestone (a stream merges, a phase completes)
+- When workers report completion
+- When progress stalls (diminishing returns across iterations)
+- Under resource pressure, before deciding to spawn more sessions
+
+### Inputs and outputs
+
+```yaml
+supervisor:
+  inputs:
+    - Current state            # .loki/state/ for each worktree/stream
+    - Original goal            # the PRD / spec / brief
+    - Recent progress          # checklist deltas, merged streams, test results
+    - Resource consumption     # .loki/state/resources.json (CPU, memory, status)
+  outputs:
+    - CONTINUE   # more work needed; keep streams running
+    - COMPLETE   # goal achieved; move to cleanup
+    - ESCALATE   # human intervention needed; raise a PAUSE / handoff
+    - PIVOT      # current approach is not converging; change strategy
+```
+
+The closest implemented analog is the completion council (`council_should_stop()`
+in `autonomy/completion-council.sh`), which decides COMPLETE vs CONTINUE from
+evidence rather than a single self-report. The supervisor pattern extends that
+mental model to the parallel case: it also reads `.loki/state/resources.json` so a
+machine under load steers toward CONTINUE-with-fewer-sessions (or ESCALATE) instead
+of oversubscribing the host.
+
+### Resource state as input
+
+The orchestrator persists a best-effort snapshot to
+`.loki/state/resources.json` (CPU usage percent, memory usage percent, and an
+`overall_status`). The dynamic-concurrency logic below reads the same file, so the
+supervisor's "can I spawn more?" decision and the spawn cap stay consistent.
+
+---
+
+## Dynamic Resource-Aware Session Concurrency
+
+By default Loki Mode caps parallel Claude sessions at a fixed
+`LOKI_MAX_PARALLEL_SESSIONS` (default 3). Opt-in dynamic concurrency lets that cap
+scale DOWN under load instead of oversubscribing the machine. It only ever reduces
+the cap; it never raises it above the configured ceiling.
+
+`effective_session_cap()` (`autonomy/run.sh`) is the single source of truth:
+
+- Default off (`LOKI_DYNAMIC_CONCURRENCY` unset or not `1`): returns exactly
+  `LOKI_MAX_PARALLEL_SESSIONS` with no file reads and no subprocesses, so the spawn
+  decision is byte-identical to the pre-feature behavior.
+- Enabled: starts from `LOKI_MAX_PARALLEL_SESSIONS_CEILING` and reads
+  `.loki/state/resources.json`. At/above the CPU or memory threshold (default 85
+  percent), or when `overall_status` is not `ok`, it halves the cap. At/above the
+  critical threshold (default 95 percent) it forces the cap to 1. The result is
+  always clamped to `[1, ceiling]`. A missing, empty, or unparseable resources file
+  is treated as no-pressure and leaves the cap at the ceiling.
+
+### Env knobs (all read in `autonomy/run.sh`)
+
+```bash
+LOKI_DYNAMIC_CONCURRENCY=1            # opt in; default 0 (off = today's behavior)
+LOKI_MAX_PARALLEL_SESSIONS_CEILING=N # upper bound when dynamic is on;
+                                     #   default = LOKI_MAX_PARALLEL_SESSIONS (3)
+LOKI_CONCURRENCY_CPU_THRESHOLD=85    # CPU percent at/above which the cap halves
+LOKI_CONCURRENCY_MEM_THRESHOLD=85    # memory percent at/above which the cap halves
+LOKI_CONCURRENCY_CRITICAL_THRESHOLD=95  # CPU or memory percent at/above which
+                                        #   the cap is forced to 1
+```
+
+### Honest ceiling: dozens, not thousands
+
+Raising `LOKI_MAX_PARALLEL_SESSIONS_CEILING` is safe because the system
+auto-throttles under pressure, but the realistic target is dozens of adaptive
+concurrent sessions, not hundreds or thousands. Two hard limits cap the useful
+ceiling on a single host:
+
+- The 3-reviewer council is a serialization point: every non-trivial change funnels
+  through blind review before merge, so review throughput, not spawn count, sets the
+  end-to-end pace.
+- Local resources (CPU, memory, API rate limits, disk for worktrees) bound how many
+  Claude sessions a single developer machine can usefully run at once.
+
+So treat dynamic concurrency as a way to set a higher ceiling safely and let the
+host throttle down, not as a path to a thousand subagents.
+
+---
+
 ## Claude Session Per Worktree
 
 Each worktree runs an independent Claude session:

package/skills/quality-gates.md

@@ -110,6 +110,24 @@ LOKI_HANDOFF_MD=1          # write a structured handoff doc to
 Optional: `LOKI_AUTO_LEARNINGS_EPISODE=1` also writes the learning into
 the Python episodic memory layer via `memory.engine.save_episode`.
 
+## Other opt-in environment flags (Release 3)
+
+Two more default-off flags added for hybrid search and parallel concurrency.
+Both are no-ops when unset (behavior identical to before).
+
+```bash
+LOKI_DYNAMIC_CONCURRENCY=1       # scale the parallel-session cap DOWN under
+                                 # CPU/memory pressure (default off). Full
+                                 # knobs and defaults: skills/parallel-workflows.md
+                                 # (Dynamic Resource-Aware Session Concurrency)
+
+LOKI_CODE_INDEX_AUTOREINDEX=1    # auto incremental re-index of the semantic
+                                 # code index before a search when stale
+                                 # (default off = warn-if-stale). Details:
+                                 # references/mcp-integration.md (Built-in
+                                 # Hybrid Codebase Search)
+```
+
 ## Verified-completion evidence gate (v7.19.1, default-on)
 
 The completion council will not accept a "done" claim without evidence. Before