loki-mode 7.129.5 → 8.0.0

package/autonomy/lib/tree_digest.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""Deterministic digest of the source tree verified by a Loki proof."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import stat
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+MANIFEST_VERSION = 3
+_EXCLUDED_TOP_LEVEL = {".git", ".loki"}
+_EXCLUDED_TOP_LEVEL_GENERATED = {".next", ".nuxt", "target"}
+_EXCLUDED_WALK_DIRS = {
+    ".git",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".venv",
+    "__pycache__",
+    "node_modules",
+    "venv",
+}
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _git_paths(root: Path) -> list[str] | None:
+    """Return tracked paths, or None when root is not the repository root."""
+    try:
+        inside = subprocess.run(
+            ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"],
+            capture_output=True,
+            timeout=10,
+        )
+        if inside.returncode != 0 or inside.stdout.strip() != b"true":
+            return None
+        top = subprocess.run(
+            ["git", "-C", str(root), "rev-parse", "--show-toplevel"],
+            capture_output=True,
+            timeout=10,
+        )
+        if top.returncode != 0:
+            return None
+        top_path = Path(os.fsdecode(top.stdout.rstrip(b"\n"))).resolve()
+        if top_path != root.resolve():
+            return None
+        result = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(root),
+                "ls-files",
+                "-z",
+                "--cached",
+            ],
+            capture_output=True,
+            timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if result.returncode != 0:
+        return None
+    return [
+        raw.decode("utf-8", errors="surrogateescape")
+        for raw in result.stdout.split(b"\0")
+        if raw
+    ]
+
+
+def _filesystem_paths(root: Path, gitlinks: set[str]) -> list[str]:
+    """Enumerate nondependency worktree bytes without consulting Git ignores."""
+    paths: list[str] = []
+
+    def visit(directory: Path, prefix: str) -> None:
+        try:
+            entries = sorted(os.scandir(directory), key=lambda item: item.name)
+        except OSError:
+            return
+        for entry in entries:
+            relative = f"{prefix}/{entry.name}" if prefix else entry.name
+            normalized = relative.replace(os.sep, "/")
+            if not prefix and entry.name in (
+                _EXCLUDED_TOP_LEVEL | _EXCLUDED_TOP_LEVEL_GENERATED
+            ):
+                continue
+            if normalized in gitlinks:
+                paths.append(relative)
+                continue
+            try:
+                is_directory = entry.is_dir(follow_symlinks=False)
+            except OSError:
+                paths.append(relative)
+                continue
+            if is_directory:
+                if entry.name in _EXCLUDED_WALK_DIRS:
+                    continue
+                visit(Path(entry.path), relative)
+            else:
+                paths.append(relative)
+
+    visit(root, "")
+    return paths
+
+
+def _gitlinks(root: Path) -> dict[str, list[dict[str, str]]]:
+    """Return staged gitlink identities keyed by their repository path."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(root), "ls-files", "--stage", "-z"],
+            capture_output=True,
+            timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return {}
+    if result.returncode != 0:
+        return {}
+    links: dict[str, list[dict[str, str]]] = {}
+    for record in result.stdout.split(b"\0"):
+        if not record or b"\t" not in record:
+            continue
+        metadata, raw_path = record.split(b"\t", 1)
+        fields = metadata.split()
+        if len(fields) != 3 or fields[0] != b"160000":
+            continue
+        relative = raw_path.decode("utf-8", errors="surrogateescape")
+        links.setdefault(relative, []).append(
+            {
+                "stage": fields[2].decode("ascii", errors="replace"),
+                "oid": fields[1].decode("ascii", errors="replace"),
+            }
+        )
+    for values in links.values():
+        values.sort(key=lambda item: (item["stage"], item["oid"]))
+    return links
+
+
+def _gitlink_entry(
+    root: Path, relative: str, index_entries: list[dict[str, str]]
+) -> dict[str, Any]:
+    """Bind both the staged gitlink and the checked-out submodule contents."""
+    normalized = relative.replace(os.sep, "/")
+    path = root / relative
+    head_oid = ""
+    try:
+        head = subprocess.run(
+            ["git", "-C", str(path), "rev-parse", "--verify", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if head.returncode == 0:
+            head_oid = (head.stdout or "").strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return {
+        "path": normalized,
+        "kind": "gitlink",
+        "index": index_entries,
+        "head_oid": head_oid,
+        "worktree_sha256": compute_tree_digest(path),
+    }
+
+
+def _entry(
+    root: Path,
+    relative: str,
+    gitlink_entries: list[dict[str, str]] | None = None,
+) -> dict[str, Any] | None:
+    normalized = relative.replace(os.sep, "/")
+    top = normalized.split("/", 1)[0]
+    if top in _EXCLUDED_TOP_LEVEL:
+        return None
+    if gitlink_entries is not None:
+        return _gitlink_entry(root, relative, gitlink_entries)
+
+    path = root / relative
+    try:
+        info = path.lstat()
+    except FileNotFoundError:
+        return {"path": normalized, "kind": "missing"}
+    except OSError:
+        return {"path": normalized, "kind": "unreadable"}
+
+    executable = bool(info.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+    if stat.S_ISLNK(info.st_mode):
+        try:
+            target = os.readlink(path)
+        except OSError:
+            target = ""
+        target_bytes = os.fsencode(target)
+        return {
+            "path": normalized,
+            "kind": "symlink",
+            "executable": executable,
+            "size": len(target_bytes),
+            "sha256": hashlib.sha256(target_bytes).hexdigest(),
+        }
+    if stat.S_ISREG(info.st_mode):
+        try:
+            content_hash = _sha256_file(path)
+        except OSError:
+            return {"path": normalized, "kind": "unreadable"}
+        return {
+            "path": normalized,
+            "kind": "file",
+            "executable": executable,
+            "size": info.st_size,
+            "sha256": content_hash,
+        }
+    return {
+        "path": normalized,
+        "kind": "other",
+        "executable": executable,
+        "size": info.st_size,
+    }
+
+
+def build_manifest(root: str | os.PathLike[str]) -> dict[str, Any] | None:
+    """Build the canonical Git worktree manifest used by proof binding."""
+    resolved = Path(root).resolve()
+    tracked_paths = _git_paths(resolved)
+    if tracked_paths is None:
+        return None
+    gitlinks = _gitlinks(resolved)
+    paths = set(tracked_paths)
+    paths.update(_filesystem_paths(resolved, set(gitlinks)))
+    entries = []
+    for relative in sorted(paths):
+        item = _entry(resolved, relative, gitlinks.get(relative))
+        if item is not None:
+            entries.append(item)
+    return {
+        "version": MANIFEST_VERSION,
+        "filesystem_walk_excluded_top_level_directories": sorted(
+            _EXCLUDED_TOP_LEVEL | _EXCLUDED_TOP_LEVEL_GENERATED
+        ),
+        "excluded_walk_directories": sorted(_EXCLUDED_WALK_DIRS),
+        "entries": entries,
+    }
+
+
+def compute_tree_digest(root: str | os.PathLike[str]) -> str:
+    """Return a SHA-256 digest, or an empty string when it cannot be derived."""
+    manifest = build_manifest(root)
+    if manifest is None:
+        return ""
+    encoded = json.dumps(
+        manifest,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=True,
+    ).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+__all__ = ["MANIFEST_VERSION", "build_manifest", "compute_tree_digest"]

package/autonomy/lib/voter-agents.sh

@@ -246,20 +246,66 @@ loki_council_dispatch_agents() {
     schema_content=$(cat "$schema_path" 2>/dev/null) || return 1
     [ -n "$schema_content" ] || return 1
 
-    # 3. Invoke claude. Guard against absent binary or non-zero exit.
-    command -v claude >/dev/null 2>&1 || return 1
-
     local prompt
     prompt=$(printf 'Loki council iteration %s. Run each declared agent against the current workspace, return one finding per agent matching the provided JSON Schema. Be terse, be honest.' "$iteration")
 
-    # Capture stderr to a per-iteration log so hung / failing claude
-    # invocations are diagnosable instead of silently swallowed. Per Opus #2
-    # LOW finding: stored under COUNCIL_STATE_DIR, no PII risk beyond the
-    # prompt itself which already lives in the dispatch log.
-    local response
+    # Shared dispatch state (declared once, before either the SDK or claude arm).
+    local response=""
     local rc=0
     local stderr_log="$COUNCIL_STATE_DIR/votes/dispatch-stderr-${iteration}.log"
     mkdir -p "$(dirname "$stderr_log")" 2>/dev/null || true
+    local _va_sdk_done=0
+
+    # v8 RAW-SDK COUNCIL PATH (opt-in LOKI_SDK_VOTER_AGENTS=1). The bash route
+    # uses `claude --agents <json> --json-schema` (one dispatch fanning out to N
+    # named agents). The raw @anthropic-ai/sdk has no --agents primitive, so we
+    # inline the agent roster into the prompt and constrain the SAME
+    # finding-schema.json via the sdk-judge bridge -- the model returns the same
+    # {findings:[...]} object the downstream parser (line 302+) consumes, one
+    # finding per declared agent. Runs BEFORE the claude-binary guard so the
+    # no-binary deploy win holds. Fail-closed: on ANY miss (flag off, no key,
+    # bun/entrypoint absent, non-zero, empty) fall through to the claude path
+    # below, then to the heuristic council on a claude miss -- a hung/failed
+    # council can never become a false COMPLETE (same guarantee as the claude arm).
+    if [ "${LOKI_SDK_VOTER_AGENTS:-0}" = "1" ]; then
+        local _va_root _va_loki _va_schema_f _va_pf _va_out _va_rc
+        _va_root="$(cd "${__LOKI_VA_REPO_ROOT}" 2>/dev/null && pwd)" || _va_root="${__LOKI_VA_REPO_ROOT}"
+        _va_loki="${_va_root}/bin/loki"
+        _va_schema_f="${_va_root}/loki-ts/data/finding-schema.json"
+        if [ -x "$_va_loki" ] && [ -f "$_va_schema_f" ] && command -v bun >/dev/null 2>&1; then
+            _va_pf="$(mktemp 2>/dev/null)" || _va_pf=""
+            if [ -n "$_va_pf" ]; then
+                # inline the agent roster so a single schema-constrained call
+                # produces one finding per declared council agent.
+                printf '%s\n\nDeclared council agents (produce exactly one finding per agent, in this order):\n%s\n' \
+                    "$prompt" "$agents_json" > "$_va_pf"
+                _va_rc=0
+                local _va_to_s="${LOKI_COUNCIL_REVIEW_TIMEOUT:-600}"
+                local _va_wrap
+                if command -v timeout >/dev/null 2>&1; then _va_wrap="timeout $(( _va_to_s + 15 ))"
+                elif command -v gtimeout >/dev/null 2>&1; then _va_wrap="gtimeout $(( _va_to_s + 15 ))"
+                else _va_wrap=""; fi
+                _va_out="$($_va_wrap "$_va_loki" internal sdk-judge \
+                    --prompt-file "$_va_pf" --schema-file "$_va_schema_f" \
+                    --model "${LOKI_SDK_COUNCIL_MODEL:-claude-sonnet-5}" --effort high \
+                    --timeout-ms "$(( _va_to_s * 1000 ))" 2>"$COUNCIL_STATE_DIR/votes/dispatch-stderr-${iteration}.log")" || _va_rc=$?
+                rm -f "$_va_pf" 2>/dev/null || true
+                if [ "$_va_rc" -eq 0 ] && [ -n "$_va_out" ]; then
+                    # feed the SDK object into the SAME parse+materialize path
+                    response="$_va_out"
+                    rc=0
+                    _va_sdk_done=1  # skip the claude dispatch below
+                fi
+            fi
+        fi
+        # fall through to the claude path (fail-closed) if not _va_sdk_done
+    fi
+
+    # 3. Invoke claude (only when the SDK path did not already produce a response).
+    # Guard against absent binary or non-zero exit.
+    if [ "$_va_sdk_done" != "1" ]; then
+    command -v claude >/dev/null 2>&1 || return 1
+
     # caveman HARD-SUPPRESS (parsed output, v7.41.0): the response is parsed for
     # findings[].vote against the JSON Schema. A globally-active caveman would
     # compress/reword it and break the schema match or flip a vote. The tree-wide
@@ -285,6 +331,10 @@ loki_council_dispatch_agents() {
                       -p "$prompt" \
                       --agents "$agents_json" \
                       --json-schema "$schema_content" 2>"$stderr_log") || rc=$?
+    fi  # end claude-dispatch guard (SDK path skips it, already has $response)
+
+    # Shared fail-closed check for BOTH paths: any non-zero exit (claude/SDK
+    # timeout 124 or failure) or empty response -> return 1 -> heuristic fallback.
     if [ "$rc" -ne 0 ] || [ -z "$response" ]; then
         return 1
     fi

package/autonomy/lib/workspace_diff.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Final-worktree diff facts shared by proof generation and verification."""
+
+from __future__ import annotations
+
+import subprocess
+
+
+_EMPTY = {"count": 0, "insertions": 0, "deletions": 0, "files": []}
+
+
+def _git(repo_dir, args, allowed=(0,)):
+    try:
+        result = subprocess.run(
+            ["git", "-C", repo_dir] + args,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="surrogateescape",
+            timeout=30,
+        )
+    except Exception:
+        return None
+    return result.stdout if result.returncode in allowed else None
+
+
+def _excluded(path):
+    return path == ".loki" or path.startswith(".loki/")
+
+
+def _parse_numstat(raw):
+    files = []
+    for record in (raw or "").split("\0"):
+        if not record:
+            continue
+        parts = record.split("\t", 2)
+        if len(parts) != 3 or _excluded(parts[2]):
+            continue
+        ins_s, del_s, path = parts
+        files.append({
+            "path": path,
+            "insertions": 0 if ins_s == "-" else int(ins_s),
+            "deletions": 0 if del_s == "-" else int(del_s),
+            "status": "binary" if ins_s == "-" else "modified",
+        })
+    return files
+
+
+def _split_patch(raw):
+    chunks = []
+    current = []
+    for line in (raw or "").splitlines(keepends=True):
+        if line.startswith("diff --git ") and current:
+            chunks.append("".join(current))
+            current = [line]
+        else:
+            current.append(line)
+    if current:
+        chunks.append("".join(current))
+    return chunks
+
+
+def collect_workspace_diff(repo_dir, base, include_diffs=False):
+    """Describe final tracked and untracked bytes relative to ``base``.
+
+    ``git diff <base>`` compares the base tree to the final working tree, so a
+    single result covers committed, staged, unstaged, and deleted tracked
+    files without double counting. Git omits untracked files, which are added
+    explicitly. Harness-owned ``.loki`` state is never product work.
+    """
+    if _git(repo_dir, ["rev-parse", "--is-inside-work-tree"]) is None:
+        return dict(_EMPTY), None
+
+    comparison = base or "HEAD~1"
+    raw = _git(repo_dir, ["diff", "--no-renames", "--numstat", "-z", comparison, "--"])
+    if raw is None:
+        comparison = "HEAD"
+        raw = _git(repo_dir, ["diff", "--no-renames", "--numstat", "-z", comparison, "--"])
+    if raw is None:
+        return dict(_EMPTY), None
+
+    files = _parse_numstat(raw)
+    diffs = [] if include_diffs else None
+    if diffs is not None:
+        patch = _git(repo_dir, ["diff", "--no-renames", comparison, "--"])
+        for chunk in _split_patch(patch):
+            first = chunk.splitlines()[0] if chunk else ""
+            path = first.split(" b/", 1)[1] if " b/" in first else ""
+            if not _excluded(path):
+                diffs.append({"path": path, "patch": chunk})
+
+    untracked = _git(repo_dir, ["ls-files", "--others", "--exclude-standard", "-z"])
+    for path in sorted(p for p in (untracked or "").split("\0") if p and not _excluded(p)):
+        stat = _git(
+            repo_dir,
+            ["diff", "--no-index", "--no-renames", "--numstat", "-z", "--", "/dev/null", path],
+            allowed=(0, 1),
+        )
+        parsed = _parse_numstat(stat)
+        entry = parsed[0] if parsed else {
+            "path": path, "insertions": 0, "deletions": 0, "status": "untracked"
+        }
+        entry["path"] = path
+        entry["status"] = "untracked_binary" if entry["status"] == "binary" else "untracked"
+        files.append(entry)
+        if diffs is not None:
+            patch = _git(
+                repo_dir,
+                ["diff", "--no-index", "--no-renames", "--", "/dev/null", path],
+                allowed=(0, 1),
+            )
+            if patch:
+                diffs.append({"path": path, "patch": patch})
+
+    files.sort(key=lambda item: item["path"])
+    return {
+        "count": len(files),
+        "insertions": sum(item["insertions"] for item in files),
+        "deletions": sum(item["deletions"] for item in files),
+        "files": files,
+    }, diffs
+
+
+__all__ = ["collect_workspace_diff"]