loki-mode 7.12.0 → 7.14.0

package/autonomy/lib/wiki_index.py

@@ -0,0 +1,258 @@
+"""wiki_index.py -- dependency-free codebase index for the R5 auto-wiki.
+
+Builds a line-anchored chunk index over a project's source files and provides
+deterministic token-overlap retrieval. This is the grounding substrate for
+cited answers: every chunk carries the REAL repo-relative file path and the
+REAL start/end line numbers it came from, so a citation can always be checked
+against the filesystem.
+
+Reuse note: the token-overlap scoring (`_tokenize` + overlap weighting) is
+ported from memory/knowledge_graph.py (OrganizationKnowledgeGraph), which scores
+memory patterns the same way. knowledge_graph.py is NOT a code index (it
+aggregates .loki/memory/semantic patterns), so the code scanning/chunking here
+is new. retrieval.py is a memory retriever, not a code indexer, so it is not
+reused for code retrieval.
+
+No third-party dependencies. CI-safe (no Docker, no network, no provider).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import subprocess
+from pathlib import Path
+
+# Source extensions we index. Kept broad but excludes lockfiles/binaries.
+SOURCE_EXTS = {
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".mjs", ".cjs",
+    ".rs", ".go", ".rb", ".java", ".kt", ".kts", ".c", ".cc", ".cpp",
+    ".h", ".hpp", ".cs", ".php", ".swift", ".sh", ".bash", ".sql",
+    ".vue", ".svelte", ".scala", ".clj", ".ex", ".exs", ".lua", ".r",
+}
+
+# Directories never worth indexing.
+SKIP_DIRS = {
+    "node_modules", ".git", "vendor", "__pycache__", "dist", "build",
+    ".next", "target", ".venv", "venv", "coverage", ".loki", ".cache",
+    "out", ".turbo", ".pytest_cache", ".mypy_cache",
+}
+
+CHUNK_LINES = 60  # lines per chunk; overlap-free, line-anchored.
+MAX_FILES = 800   # safety cap so huge repos stay cheap.
+MAX_FILE_BYTES = 400_000  # skip very large generated/minified files.
+
+# Tokenizer ported from memory/knowledge_graph.py:_tokenize / _STOPWORDS.
+_STOPWORDS = {
+    "the", "a", "an", "to", "for", "of", "and", "or", "with", "without",
+    "is", "are", "be", "up", "on", "in", "by", "not", "this", "that",
+    "from", "as", "at", "it", "if", "do", "we", "my", "our", "how",
+    "def", "self", "return", "import", "const", "let", "var", "function",
+}
+
+
+def _tokenize(text):
+    """Lowercase, split on non-alphanumerics, drop stopwords + short tokens.
+
+    Ported from knowledge_graph.OrganizationKnowledgeGraph._tokenize so wiki
+    retrieval scores text the same way memory-pattern retrieval does.
+    """
+    toks = re.split(r"[^a-z0-9_]+", str(text or "").lower())
+    return {t for t in toks if len(t) > 2 and t not in _STOPWORDS}
+
+
+def _git_tracked_files(root):
+    """Return git-tracked file paths (repo-relative), or None if not a repo."""
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(root), "ls-files"],
+            capture_output=True, text=True, timeout=30,
+        )
+        if out.returncode != 0:
+            return None
+        files = [line.strip() for line in out.stdout.splitlines() if line.strip()]
+        return files or None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _walk_files(root):
+    """Filtered filesystem walk fallback (when not a git repo)."""
+    root = Path(root)
+    results = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
+        for fn in filenames:
+            rel = os.path.relpath(os.path.join(dirpath, fn), root)
+            results.append(rel)
+    return results
+
+
+def list_source_files(root):
+    """Return a sorted list of repo-relative source files under root.
+
+    Prefers git ls-files (respects .gitignore), falls back to a filtered walk.
+    """
+    root = Path(root)
+    candidates = _git_tracked_files(root)
+    if candidates is None:
+        candidates = _walk_files(root)
+
+    sources = []
+    for rel in candidates:
+        # Skip anything inside a skip dir (git tracked files can include them
+        # if they were committed; we still exclude noise dirs).
+        parts = set(Path(rel).parts)
+        if parts & SKIP_DIRS:
+            continue
+        ext = os.path.splitext(rel)[1].lower()
+        if ext not in SOURCE_EXTS:
+            continue
+        abs_path = root / rel
+        try:
+            if not abs_path.is_file():
+                continue
+            if abs_path.stat().st_size > MAX_FILE_BYTES:
+                continue
+        except OSError:
+            continue
+        sources.append(rel)
+    sources.sort()
+    return sources[:MAX_FILES]
+
+
+def _read_lines(abs_path):
+    try:
+        with open(abs_path, "r", encoding="utf-8", errors="replace") as f:
+            return f.read().splitlines()
+    except OSError:
+        return []
+
+
+def build_index(root):
+    """Build a line-anchored chunk index over the project's source files.
+
+    Returns a dict:
+      {
+        "root": <abs root>,
+        "files": [<repo-relative paths>],
+        "chunks": [
+          {"id": int, "file": <rel>, "start_line": int, "end_line": int,
+           "text": <chunk text>},
+          ...
+        ],
+      }
+    Paths are ALWAYS repo-relative (no PII, no absolute paths leak).
+    Line numbers are 1-based and inclusive.
+    """
+    root = Path(root).resolve()
+    files = list_source_files(root)
+    chunks = []
+    cid = 0
+    for rel in files:
+        lines = _read_lines(root / rel)
+        if not lines:
+            continue
+        for start in range(0, len(lines), CHUNK_LINES):
+            block = lines[start:start + CHUNK_LINES]
+            if not any(line.strip() for line in block):
+                continue  # skip all-blank chunks
+            chunks.append({
+                "id": cid,
+                "file": rel,
+                "start_line": start + 1,
+                "end_line": start + len(block),
+                "text": "\n".join(block),
+            })
+            cid += 1
+    return {"root": str(root), "files": files, "chunks": chunks}
+
+
+def retrieve(index, query, k=6):
+    """Deterministic top-K chunk retrieval by token overlap.
+
+    Scoring mirrors knowledge_graph.query_patterns: token overlap between the
+    query and the chunk text, plus a small bonus when the query substring
+    appears verbatim and when query tokens appear in the file path (so
+    "how does the cli dispatch" surfaces cli.* files). No LLM, no network.
+    Ties broken by chunk id for stable, reproducible ordering.
+    """
+    qtokens = _tokenize(query)
+    qlower = str(query or "").lower()
+    scored = []
+    for ch in index.get("chunks", []):
+        text = ch.get("text", "")
+        score = 0
+        overlap = qtokens & _tokenize(text)
+        score += 3 * len(overlap)
+        # Path tokens (file/dir names are strong signals).
+        path_overlap = qtokens & _tokenize(ch.get("file", ""))
+        score += 2 * len(path_overlap)
+        # Verbatim substring bonus.
+        if qlower and len(qlower) > 3 and qlower in text.lower():
+            score += 4
+        if score > 0:
+            scored.append((score, ch["id"], ch))
+    # Highest score first; stable tiebreak on id.
+    scored.sort(key=lambda t: (-t[0], t[1]))
+    return [ch for _, _, ch in scored[:k]]
+
+
+def compute_signature(root):
+    """Cheap-incremental signature over git HEAD + per-file content hashes.
+
+    Same idea as the proof/docs manifest: a deterministic hash that changes
+    iff the indexed source set changes. Used to skip regeneration when the
+    codebase is unchanged.
+    """
+    root = Path(root).resolve()
+    h = hashlib.sha256()
+    # git HEAD (if available) makes the signature cheap to invalidate on commit.
+    try:
+        head = subprocess.run(
+            ["git", "-C", str(root), "rev-parse", "HEAD"],
+            capture_output=True, text=True, timeout=15,
+        )
+        if head.returncode == 0:
+            h.update(b"head:" + head.stdout.strip().encode())
+    except (OSError, subprocess.SubprocessError):
+        pass
+    for rel in list_source_files(root):
+        try:
+            data = (root / rel).read_bytes()
+        except OSError:
+            continue
+        h.update(rel.encode("utf-8"))
+        h.update(hashlib.sha256(data).digest())
+    return h.hexdigest()
+
+
+def extract_definitions(root, rel, limit=12):
+    """Return real def/class/function lines for a file, for code-derived citations.
+
+    Returns a list of {"name": str, "line": int} where line is 1-based and
+    points at a real definition in the file. Language-agnostic via a small set
+    of regexes; only emits matches that actually exist in the file.
+    """
+    lines = _read_lines(Path(root) / rel)
+    patterns = [
+        re.compile(r"^\s*def\s+([A-Za-z_][A-Za-z0-9_]*)"),
+        re.compile(r"^\s*class\s+([A-Za-z_][A-Za-z0-9_]*)"),
+        re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)"),
+        re.compile(r"^\s*(?:export\s+)?const\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\("),
+        re.compile(r"^\s*(?:pub\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)"),
+        re.compile(r"^\s*func\s+(?:\([^)]*\)\s*)?([A-Za-z_][A-Za-z0-9_]*)"),
+        re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*\{"),  # bash funcs
+    ]
+    defs = []
+    for i, line in enumerate(lines, start=1):
+        for pat in patterns:
+            m = pat.match(line)
+            if m:
+                defs.append({"name": m.group(1), "line": i})
+                break
+        if len(defs) >= limit:
+            break
+    return defs

package/autonomy/lib/wiki_llm.py

@@ -0,0 +1,140 @@
+"""wiki_llm.py -- stub-aware LLM invocation + citation validation for R5 wiki.
+
+Keeps every paid-call and grounding-guarantee concern in one place so the
+generator and the ask script behave identically.
+
+LLM stub contract (CI-safe, zero paid calls in tests):
+  LOKI_WIKI_LLM_STUB unset       -> call the real provider (claude -p / codex / ...)
+                                    via the same mechanism loki docs uses; if no
+                                    provider is on PATH, return None (callers then
+                                    fall back to extractive/template output).
+  LOKI_WIKI_LLM_STUB=<file path> -> read the completion from that file.
+  LOKI_WIKI_LLM_STUB=<other>     -> use the value literally as the completion.
+
+No third-party dependencies.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+
+def invoke_llm(prompt, timeout=120):
+    """Return the LLM completion string, or None if unavailable.
+
+    Honors LOKI_WIKI_LLM_STUB for CI. Otherwise shells out to the configured
+    provider, mirroring loki docs `_docs_invoke_provider`.
+    """
+    stub = os.environ.get("LOKI_WIKI_LLM_STUB")
+    if stub is not None:
+        # A path to a file with the canned completion, else the literal value.
+        if os.path.sep in stub or stub.endswith(".txt"):
+            p = Path(stub)
+            if p.is_file():
+                try:
+                    return p.read_text(encoding="utf-8", errors="replace")
+                except OSError:
+                    return ""
+        return stub
+
+    provider = os.environ.get("LOKI_PROVIDER", "claude")
+    state_provider = Path(".loki/state/provider")
+    if state_provider.is_file():
+        try:
+            provider = state_provider.read_text().strip() or provider
+        except OSError:
+            pass
+
+    # Resolve a timeout wrapper if present (matches the bash docs helper).
+    timeout_cmd = None
+    for cand in ("timeout", "gtimeout"):
+        if _which(cand):
+            timeout_cmd = cand
+            break
+
+    cmds = {
+        "claude": ["claude", "-p", prompt],
+        "codex": ["codex", "exec", "--full-auto", prompt],
+        "cline": ["cline", "-y", prompt],
+        "aider": ["aider", "--message", prompt, "--yes-always", "--no-auto-commits"],
+    }
+    base = cmds.get(provider)
+    if base is None or not _which(base[0]):
+        return None
+    cmd = ([timeout_cmd, str(timeout)] + base) if timeout_cmd else base
+    try:
+        out = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=timeout + 10,
+            stdin=subprocess.DEVNULL,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if out.returncode != 0 and not out.stdout.strip():
+        return None
+    return out.stdout
+
+
+def _which(name):
+    for d in os.environ.get("PATH", "").split(os.pathsep):
+        cand = os.path.join(d, name)
+        if os.path.isfile(cand) and os.access(cand, os.X_OK):
+            return cand
+    return None
+
+
+_CITE_RE = re.compile(r"\[(\d+)\]")
+
+
+def map_and_validate_citations(answer_text, chunks, root):
+    """Map [n] indices in answer_text to real {file,line} citations.
+
+    chunks: the numbered chunk list shown to the LLM. chunks[n-1] is the chunk
+    referenced by [n] (1-based). A citation is kept only if:
+      - the index is in range (it references a chunk we actually supplied), and
+      - the file exists on disk AND start_line <= file length.
+    This makes a fabricated citation structurally impossible to survive.
+
+    Returns (clean_text, citations) where citations is a de-duplicated list of
+    {"file": rel, "line": int} in first-appearance order, and clean_text has the
+    [n] markers rewritten to [file:line] for human-readable output.
+    """
+    root = Path(root)
+    citations = []
+    seen = set()
+
+    def _resolve(idx):
+        if idx < 1 or idx > len(chunks):
+            return None
+        ch = chunks[idx - 1]
+        rel = ch.get("file")
+        line = int(ch.get("start_line", 1))
+        abs_path = root / rel
+        try:
+            if not abs_path.is_file():
+                return None
+            with open(abs_path, "r", encoding="utf-8", errors="replace") as f:
+                nlines = sum(1 for _ in f)
+        except OSError:
+            return None
+        if line < 1 or line > max(nlines, 1):
+            return None
+        return {"file": rel, "line": line}
+
+    def _sub(m):
+        idx = int(m.group(1))
+        cite = _resolve(idx)
+        if cite is None:
+            return ""  # drop a bogus/non-resolving citation marker
+        key = (cite["file"], cite["line"])
+        if key not in seen:
+            seen.add(key)
+            citations.append(cite)
+        return "[%s:%d]" % (cite["file"], cite["line"])
+
+    clean = _CITE_RE.sub(_sub, answer_text or "")
+    # Collapse any double spaces left by dropped markers.
+    clean = re.sub(r"[ \t]{2,}", " ", clean)
+    return clean, citations