loki-mode 9.12.6 → 9.16.0

package/autonomy/lib/preedit_snapshot.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Pre-edit snapshot: score the AGENT, not the agent-plus-whoever-fixed-it.
+
+WHY THIS EXISTS. 8090 AI's medical-document evaluation framework -- the single
+most rigorous engineering artifact in any competitor corpus -- makes an argument
+we could not answer:
+
+    "A skilled writer rescues an unusable draft, and the post-edit score looks
+     acceptable; a junior writer leaves the model's failures visible, and the
+     same model appears to perform worse. In either case, the metric describes
+     the writer-plus-AI workflow, not the AI alone."
+
+Every quality number we publish today is measured AFTER a human may have touched
+the diff. So a strong reviewer flatters the agent and a weak one maligns it, and
+the number moves for reasons that have nothing to do with the agent. That makes
+our own headline partly a measurement of our users.
+
+The fix is not analysis, it is ORDER OF OPERATIONS. 8090's own words: the two
+scores must be "separated by the workflow itself, not reconstructed afterward
+from logs". So this captures an immutable snapshot of the agent's output at the
+moment it stops and before any human edit. Anything reconstructed later is a
+guess about which lines a human wrote, and a guess is exactly what this replaces.
+
+WHAT IT DELIBERATELY IS NOT. It does not score anything. It records WHAT WAS
+THERE, with a hash, so a later comparison is a fact rather than an inference. A
+low pre-edit score is a DIAGNOSTIC, not a failure -- 8090 again: "A preedit
+composite of twenty percent is not a failure of the eval. It is a diagnostic,
+document-by-document, of where the model is weak." Nothing here feeds a gate,
+because a gate that punished the agent for needing edits would train the agent to
+produce diffs nobody edits, which is not the same as good diffs.
+
+WHAT ALREADY EXISTS AND IS NOT REBUILT. `_compute_headline` in
+proof-generator.py already implements the asymmetric-risk half of this idea, in
+its own words: "A failed check is a stronger negative signal than a not-run one:
+amber means we did not check everything, red means something we checked did not
+pass." That is F2-style weighting -- treating a false negative as worse than a
+false positive -- already shipped. This module adds only the missing half.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+# Named refusals, mirroring the outcome ledger's ANCHOR_REASONS. A comparison we
+# cannot compute is reported BY NAME, never as zero and never as a pass.
+COMPARE_REASONS = {
+    "no_snapshot": "no pre-edit snapshot was captured for this run",
+    "snapshot_unreadable": "the snapshot exists but could not be parsed",
+    "not_a_git_repo": "not a git repository, so the current diff cannot be read",
+    "no_current_diff": "there is no current diff to compare the snapshot against",
+}
+
+
+def _git(args, cwd, timeout=60):
+    try:
+        p = subprocess.run(["git"] + list(args), cwd=cwd,
+                           capture_output=True, text=True, timeout=timeout)
+        return p.returncode, p.stdout
+    except (subprocess.TimeoutExpired, OSError):
+        return 1, ""
+
+
+def _sha256(text):
+    return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()
+
+
+def capture(loki_dir, run_id, cwd, base_sha=None):
+    """Freeze the agent's output before any human edit.
+
+    Stores the full unified diff plus its sha256. The hash is what makes a later
+    comparison evidence rather than an assertion: anyone can recompute it.
+
+    Called by the workflow at the moment the agent stops. Capturing it later --
+    after a review, from a log -- would record the human's work as the agent's,
+    which is the exact contamination this exists to prevent.
+    """
+    rc, _ = _git(["rev-parse", "--git-dir"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "not_a_git_repo"}
+
+    if base_sha:
+        rc, diff = _git(["diff", base_sha, "--"], cwd)
+    else:
+        # Uncommitted agent work is the normal case at stop time.
+        rc, diff = _git(["diff", "HEAD", "--"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "no_current_diff"}
+
+    rc2, names = _git(["diff", "--name-only"] + ([base_sha] if base_sha else ["HEAD"]), cwd)
+    files = [l.strip() for l in names.splitlines() if l.strip()] if rc2 == 0 else []
+
+    snap = {
+        "schema_version": SCHEMA_VERSION,
+        "run_id": run_id,
+        "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "base_sha": base_sha or "",
+        # The point of the whole file: this is the agent's output, uncontaminated.
+        "author": "agent",
+        "diff_sha256": _sha256(diff),
+        "diff_bytes": len(diff.encode("utf-8", "replace")),
+        "files": files,
+        "lines_added": sum(1 for l in diff.splitlines()
+                           if l.startswith("+") and not l.startswith("+++")),
+        "lines_removed": sum(1 for l in diff.splitlines()
+                             if l.startswith("-") and not l.startswith("---")),
+    }
+
+    out_dir = os.path.join(loki_dir, "preedit")
+    try:
+        os.makedirs(out_dir, exist_ok=True)
+        path = os.path.join(out_dir, f"{run_id}.json")
+        # Write-once. A snapshot that a later run can overwrite is not a
+        # snapshot -- re-capturing after a human edit would silently record the
+        # human's work as the agent's, which is the contamination this prevents.
+        if os.path.exists(path):
+            return {"status": "exists", "path": path}
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(snap, fh, indent=2)
+            fh.write("\n")
+        # The raw diff is kept beside the metadata so the hash is checkable by
+        # hand. Evidence nobody can recompute is a claim, not evidence.
+        with open(os.path.join(out_dir, f"{run_id}.diff"), "w", encoding="utf-8") as fh:
+            fh.write(diff)
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "snapshot_unreadable", "detail": str(exc)}
+
+    return {"status": "captured", "path": path, "diff_sha256": snap["diff_sha256"]}
+
+
+def compare(loki_dir, run_id, cwd):
+    """How much of what shipped was the agent's, and how much was human rescue.
+
+    Reports lines, never a score. The split is the useful fact; a composite
+    number invites exactly the ranking behaviour that made post-edit scores
+    misleading in the first place.
+    """
+    path = os.path.join(loki_dir, "preedit", f"{run_id}.json")
+    if not os.path.isfile(path):
+        return {"status": UNKNOWN, "reason": "no_snapshot",
+                "detail": COMPARE_REASONS["no_snapshot"]}
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            snap = json.load(fh)
+    except (OSError, ValueError):
+        return {"status": UNKNOWN, "reason": "snapshot_unreadable",
+                "detail": COMPARE_REASONS["snapshot_unreadable"]}
+
+    base = snap.get("base_sha") or "HEAD"
+    rc, diff_now = _git(["diff", base, "--"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "no_current_diff",
+                "detail": COMPARE_REASONS["no_current_diff"]}
+
+    now_hash = _sha256(diff_now)
+    added_now = sum(1 for l in diff_now.splitlines()
+                    if l.startswith("+") and not l.startswith("+++"))
+
+    untouched = now_hash == snap.get("diff_sha256")
+    return {
+        "status": "measured",
+        "run_id": run_id,
+        "agent_lines_added": snap.get("lines_added"),
+        "current_lines_added": added_now,
+        # A hash match is proof nobody edited it; a mismatch proves somebody did,
+        # but NOT how much -- attributing individual lines would be the guess
+        # this module exists to avoid.
+        "human_edited": (not untouched),
+        "preedit_diff_sha256": snap.get("diff_sha256"),
+        "current_diff_sha256": now_hash,
+        "captured_at": snap.get("captured_at"),
+        "note": ("the pre-edit diff is byte-identical to what shipped, so this "
+                 "receipt measures the agent alone"
+                 if untouched else
+                 "what shipped differs from the agent's output, so any score "
+                 "taken now measures the agent plus the human who edited it"),
+    }
+
+
+def main(argv):
+    if not argv:
+        print("usage: preedit_snapshot.py capture|compare <run_id> [--json]",
+              file=sys.stderr)
+        return 2
+    action = argv[0]
+    run_id = argv[1] if len(argv) > 1 else ""
+    if not run_id:
+        print("a run_id is required", file=sys.stderr)
+        return 2
+
+    cwd = os.environ.get("LOKI_PREEDIT_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    if action == "capture":
+        res = capture(loki_dir, run_id, cwd,
+                      base_sha=os.environ.get("LOKI_RUN_START_SHA") or None)
+    elif action == "compare":
+        res = compare(loki_dir, run_id, cwd)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("captured", "exists", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/verdict.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""One verdict a reviewer can read in ten seconds, assembled from measured parts.
+
+WHY THIS EXISTS. We now measure five things nobody else does -- did the work
+survive (outcome ledger), does the spec still match the intent (intent ledger),
+was this the agent's own output or a human rescue (pre-edit snapshot), does the
+completion claim name real work (claim grounding), and which model decided
+(decision record). Each is correct and each lives in its own file. A reviewer
+looking at a pull request reads none of them.
+
+A moat nobody sees is not a moat. The Evidence Receipt is already attached to
+PRs by default (LOKI_PROVEN_PR, autonomy/run.sh:3849), so the surface exists and
+this fills it: one block, five lines, each of which is either a measured fact or
+an explicit UNKNOWN.
+
+WHAT IT REFUSES, AND WHY THAT MATTERS MORE HERE THAN ANYWHERE. This is the only
+place where all five signals meet, which makes it the one place a composite score
+would be tempting: a single "trust: 87" would fit a PR comment beautifully. It is
+not offered. Averaging a revert count, a hash comparison, a diff hash, a path
+match and a model id produces a number whose movement nobody can explain, and a
+number nobody can explain is the thing our competitors already ship. Five honest
+lines beat one confident one.
+
+UNKNOWN IS PRINTED, NOT HIDDEN. A reviewer must be able to tell "we checked and
+it is fine" from "we could not check". Suppressing the unmeasured lines would
+make a receipt with one real signal look identical to one with five, which is
+exactly the false confidence the whole product exists to refuse.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+# Each signal: where it lives, and the one-line question it answers for a human
+# scanning a PR. The question text is load-bearing -- a reviewer who cannot tell
+# what a line MEANS will skip the block, and a skipped block is worth nothing.
+SIGNALS = (
+    ("outcome", "did previous work survive, or get reverted"),
+    ("intent", "does the spec still say what was actually wanted"),
+    ("authorship", "is this the agent's output, or a human rescue"),
+    ("grounding", "does the completion claim name work that exists in the diff"),
+    ("model", "which model decided, at what temperature"),
+)
+
+
+def _read_json(path):
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError):
+        return None
+
+
+def _outcome_line(loki_dir):
+    p = os.path.join(loki_dir, "proofs")
+    if not os.path.isdir(p):
+        return UNKNOWN, "no receipts yet, so nothing has an outcome to follow"
+    # Deliberately does not recompute: the outcome ledger owns that logic and two
+    # implementations of one number eventually disagree.
+    return UNKNOWN, "run `loki outcomes` for the measured post-merge result"
+
+
+def _intent_line(loki_dir):
+    p = os.path.join(loki_dir, "intent", "intent.json")
+    if not os.path.isfile(p):
+        return UNKNOWN, "no intent recorded, so spec-vs-intent drift is not measurable"
+    doc = _read_json(p)
+    if doc is None:
+        return UNKNOWN, "the intent record could not be read"
+    stmts = doc.get("statements") or []
+    if not stmts:
+        return UNKNOWN, "no intent statements recorded"
+    linked = sum(1 for s in stmts if s.get("links"))
+    return "measured", f"{len(stmts)} intent statement(s), {linked} linked to the spec"
+
+
+def _authorship_line(loki_dir, run_id=None):
+    d = os.path.join(loki_dir, "preedit")
+    if not os.path.isdir(d):
+        return UNKNOWN, "no pre-edit snapshot, so agent output cannot be told from human edits"
+    snaps = [f for f in os.listdir(d) if f.endswith(".json")]
+    if not snaps:
+        return UNKNOWN, "no pre-edit snapshot captured"
+    return "measured", f"{len(snaps)} run(s) have an immutable pre-edit snapshot"
+
+
+def _grounding_line(loki_dir):
+    # The completion route now persists the per-claim result here
+    # (run.sh:_loki_check_claim_grounding, written at the non-destructive claim
+    # peek). Absent file means the run never reached a completion claim, which
+    # is UNKNOWN -- not a pass, and not a failure.
+    d = _read_json(os.path.join(loki_dir, "state", "claim-grounding.json"))
+    if not isinstance(d, dict) or d.get("status") != "measured":
+        return UNKNOWN, "no completion claim was checked against the diff"
+    named = d.get("paths_named") or []
+    if not named:
+        # Reported by name rather than scored: a claim naming no path is
+        # UNGROUNDABLE, which is a different fact from a claim that checked out.
+        return UNKNOWN, "the completion claim named no file path, so it cannot be grounded"
+    ungrounded = d.get("ungrounded") or []
+    if d.get("has_ungrounded_claim"):
+        return "finding", (
+            f"the completion claim names {len(ungrounded)} path(s) absent from the diff: "
+            + ", ".join(str(p) for p in ungrounded[:3])
+        )
+    return "measured", f"all {len(named)} path(s) named in the completion claim are in the diff"
+
+
+def _model_line(loki_dir):
+    p = os.path.join(loki_dir, "decisions", "decisions.jsonl")
+    if not os.path.isfile(p):
+        return UNKNOWN, "no decision records, so a model swap would be undetectable"
+    models = set()
+    n = 0
+    try:
+        with open(p, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    r = json.loads(line)
+                except ValueError:
+                    continue
+                n += 1
+                if r.get("model_id"):
+                    models.add(r["model_id"])
+    except OSError:
+        return UNKNOWN, "the decision record could not be read"
+    if not n:
+        return UNKNOWN, "no decision records"
+    if len(models) > 1:
+        # The single most audit-relevant thing this block can say.
+        return "measured", f"{n} decisions across {len(models)} DIFFERENT models: {', '.join(sorted(models))}"
+    return "measured", f"{n} decisions, one model: {', '.join(models) or 'unnamed'}"
+
+
+def build(loki_dir, run_id=None):
+    rows = []
+    for key, question in SIGNALS:
+        if key == "outcome":
+            status, detail = _outcome_line(loki_dir)
+        elif key == "intent":
+            status, detail = _intent_line(loki_dir)
+        elif key == "authorship":
+            status, detail = _authorship_line(loki_dir, run_id)
+        elif key == "grounding":
+            status, detail = _grounding_line(loki_dir)
+        else:
+            status, detail = _model_line(loki_dir)
+        rows.append({"signal": key, "question": question,
+                     "status": status, "detail": detail})
+
+    measured = sum(1 for r in rows if r["status"] == "measured")
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "signals": rows,
+        "measured": measured,
+        "total": len(rows),
+        # No composite. See the module docstring: averaging a revert count, a
+        # hash comparison and a model id yields a number nobody can explain.
+    }
+
+
+def render_markdown(v):
+    """A PR comment block. Terse on purpose: a reviewer gives this ten seconds."""
+    out = ["### Loki verification signals", ""]
+    out.append(f"{v['measured']} of {v['total']} signals measured. "
+               "UNKNOWN means we could not check, not that it passed.")
+    out.append("")
+    out.append("| Signal | Status | Detail |")
+    out.append("|---|---|---|")
+    for r in v["signals"]:
+        # Three states, not two. Collapsing everything non-measured to UNKNOWN
+        # would render a real finding ("the claim names a file absent from the
+        # diff") identically to "we could not check" -- the false equivalence
+        # this whole module exists to refuse. Anything unrecognised still
+        # degrades to UNKNOWN, so an unmeasured signal can never read as a pass.
+        status = r["status"] if r["status"] in ("measured", "finding") else UNKNOWN
+        out.append(f"| {r['signal']} | {status} | {r['detail']} |")
+    out.append("")
+    out.append("Every line is derived from a file in `.loki/` that you can read "
+               "yourself. No score is offered: five honest lines beat one "
+               "confident number.")
+    return "\n".join(out)
+
+
+def main(argv):
+    as_json = "--json" in argv
+    cwd = os.environ.get("LOKI_VERDICT_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+    v = build(loki_dir)
+    print(json.dumps(v, indent=2) if as_json else render_markdown(v))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))