loki-mode 9.12.6 → 9.17.0

package/autonomy/lib/preedit_snapshot.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Pre-edit snapshot: score the AGENT, not the agent-plus-whoever-fixed-it.
+
+WHY THIS EXISTS. 8090 AI's medical-document evaluation framework -- the single
+most rigorous engineering artifact in any competitor corpus -- makes an argument
+we could not answer:
+
+    "A skilled writer rescues an unusable draft, and the post-edit score looks
+     acceptable; a junior writer leaves the model's failures visible, and the
+     same model appears to perform worse. In either case, the metric describes
+     the writer-plus-AI workflow, not the AI alone."
+
+Every quality number we publish today is measured AFTER a human may have touched
+the diff. So a strong reviewer flatters the agent and a weak one maligns it, and
+the number moves for reasons that have nothing to do with the agent. That makes
+our own headline partly a measurement of our users.
+
+The fix is not analysis, it is ORDER OF OPERATIONS. 8090's own words: the two
+scores must be "separated by the workflow itself, not reconstructed afterward
+from logs". So this captures an immutable snapshot of the agent's output at the
+moment it stops and before any human edit. Anything reconstructed later is a
+guess about which lines a human wrote, and a guess is exactly what this replaces.
+
+WHAT IT DELIBERATELY IS NOT. It does not score anything. It records WHAT WAS
+THERE, with a hash, so a later comparison is a fact rather than an inference. A
+low pre-edit score is a DIAGNOSTIC, not a failure -- 8090 again: "A preedit
+composite of twenty percent is not a failure of the eval. It is a diagnostic,
+document-by-document, of where the model is weak." Nothing here feeds a gate,
+because a gate that punished the agent for needing edits would train the agent to
+produce diffs nobody edits, which is not the same as good diffs.
+
+WHAT ALREADY EXISTS AND IS NOT REBUILT. `_compute_headline` in
+proof-generator.py already implements the asymmetric-risk half of this idea, in
+its own words: "A failed check is a stronger negative signal than a not-run one:
+amber means we did not check everything, red means something we checked did not
+pass." That is F2-style weighting -- treating a false negative as worse than a
+false positive -- already shipped. This module adds only the missing half.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+# Named refusals, mirroring the outcome ledger's ANCHOR_REASONS. A comparison we
+# cannot compute is reported BY NAME, never as zero and never as a pass.
+COMPARE_REASONS = {
+    "no_snapshot": "no pre-edit snapshot was captured for this run",
+    "snapshot_unreadable": "the snapshot exists but could not be parsed",
+    "not_a_git_repo": "not a git repository, so the current diff cannot be read",
+    "no_current_diff": "there is no current diff to compare the snapshot against",
+}
+
+
+def _git(args, cwd, timeout=60):
+    try:
+        p = subprocess.run(["git"] + list(args), cwd=cwd,
+                           capture_output=True, text=True, timeout=timeout)
+        return p.returncode, p.stdout
+    except (subprocess.TimeoutExpired, OSError):
+        return 1, ""
+
+
+def _sha256(text):
+    return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()
+
+
+def capture(loki_dir, run_id, cwd, base_sha=None):
+    """Freeze the agent's output before any human edit.
+
+    Stores the full unified diff plus its sha256. The hash is what makes a later
+    comparison evidence rather than an assertion: anyone can recompute it.
+
+    Called by the workflow at the moment the agent stops. Capturing it later --
+    after a review, from a log -- would record the human's work as the agent's,
+    which is the exact contamination this exists to prevent.
+    """
+    rc, _ = _git(["rev-parse", "--git-dir"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "not_a_git_repo"}
+
+    if base_sha:
+        rc, diff = _git(["diff", base_sha, "--"], cwd)
+    else:
+        # Uncommitted agent work is the normal case at stop time.
+        rc, diff = _git(["diff", "HEAD", "--"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "no_current_diff"}
+
+    rc2, names = _git(["diff", "--name-only"] + ([base_sha] if base_sha else ["HEAD"]), cwd)
+    files = [l.strip() for l in names.splitlines() if l.strip()] if rc2 == 0 else []
+
+    snap = {
+        "schema_version": SCHEMA_VERSION,
+        "run_id": run_id,
+        "captured_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "base_sha": base_sha or "",
+        # The point of the whole file: this is the agent's output, uncontaminated.
+        "author": "agent",
+        "diff_sha256": _sha256(diff),
+        "diff_bytes": len(diff.encode("utf-8", "replace")),
+        "files": files,
+        "lines_added": sum(1 for l in diff.splitlines()
+                           if l.startswith("+") and not l.startswith("+++")),
+        "lines_removed": sum(1 for l in diff.splitlines()
+                             if l.startswith("-") and not l.startswith("---")),
+    }
+
+    out_dir = os.path.join(loki_dir, "preedit")
+    try:
+        os.makedirs(out_dir, exist_ok=True)
+        path = os.path.join(out_dir, f"{run_id}.json")
+        # Write-once. A snapshot that a later run can overwrite is not a
+        # snapshot -- re-capturing after a human edit would silently record the
+        # human's work as the agent's, which is the contamination this prevents.
+        if os.path.exists(path):
+            return {"status": "exists", "path": path}
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(snap, fh, indent=2)
+            fh.write("\n")
+        # The raw diff is kept beside the metadata so the hash is checkable by
+        # hand. Evidence nobody can recompute is a claim, not evidence.
+        with open(os.path.join(out_dir, f"{run_id}.diff"), "w", encoding="utf-8") as fh:
+            fh.write(diff)
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "snapshot_unreadable", "detail": str(exc)}
+
+    return {"status": "captured", "path": path, "diff_sha256": snap["diff_sha256"]}
+
+
+def compare(loki_dir, run_id, cwd):
+    """How much of what shipped was the agent's, and how much was human rescue.
+
+    Reports lines, never a score. The split is the useful fact; a composite
+    number invites exactly the ranking behaviour that made post-edit scores
+    misleading in the first place.
+    """
+    path = os.path.join(loki_dir, "preedit", f"{run_id}.json")
+    if not os.path.isfile(path):
+        return {"status": UNKNOWN, "reason": "no_snapshot",
+                "detail": COMPARE_REASONS["no_snapshot"]}
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            snap = json.load(fh)
+    except (OSError, ValueError):
+        return {"status": UNKNOWN, "reason": "snapshot_unreadable",
+                "detail": COMPARE_REASONS["snapshot_unreadable"]}
+
+    base = snap.get("base_sha") or "HEAD"
+    rc, diff_now = _git(["diff", base, "--"], cwd)
+    if rc != 0:
+        return {"status": UNKNOWN, "reason": "no_current_diff",
+                "detail": COMPARE_REASONS["no_current_diff"]}
+
+    now_hash = _sha256(diff_now)
+    added_now = sum(1 for l in diff_now.splitlines()
+                    if l.startswith("+") and not l.startswith("+++"))
+
+    untouched = now_hash == snap.get("diff_sha256")
+    return {
+        "status": "measured",
+        "run_id": run_id,
+        "agent_lines_added": snap.get("lines_added"),
+        "current_lines_added": added_now,
+        # A hash match is proof nobody edited it; a mismatch proves somebody did,
+        # but NOT how much -- attributing individual lines would be the guess
+        # this module exists to avoid.
+        "human_edited": (not untouched),
+        "preedit_diff_sha256": snap.get("diff_sha256"),
+        "current_diff_sha256": now_hash,
+        "captured_at": snap.get("captured_at"),
+        "note": ("the pre-edit diff is byte-identical to what shipped, so this "
+                 "receipt measures the agent alone"
+                 if untouched else
+                 "what shipped differs from the agent's output, so any score "
+                 "taken now measures the agent plus the human who edited it"),
+    }
+
+
+def main(argv):
+    if not argv:
+        print("usage: preedit_snapshot.py capture|compare <run_id> [--json]",
+              file=sys.stderr)
+        return 2
+    action = argv[0]
+    run_id = argv[1] if len(argv) > 1 else ""
+    if not run_id:
+        print("a run_id is required", file=sys.stderr)
+        return 2
+
+    cwd = os.environ.get("LOKI_PREEDIT_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    if action == "capture":
+        res = capture(loki_dir, run_id, cwd,
+                      base_sha=os.environ.get("LOKI_RUN_START_SHA") or None)
+    elif action == "compare":
+        res = compare(loki_dir, run_id, cwd)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("captured", "exists", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/proof-generator.py

@@ -943,12 +943,46 @@ def _git_diffstat(target_dir, include_diffs):
 
     Order of preference:
       1. _LOKI_RUN_START_SHA -- the run's own baseline (run.sh exports it).
-      2. The empty tree -- correct for a GREENFIELD run (a repo with no commits
+      2. .loki/state/start-sha -- the SAME baseline, persisted to disk. The env
+         var is only exported inside run_autonomous (run.sh:21690), so a receipt
+         generated outside that scope -- `loki proof` run by hand, a resumed
+         run, a receipt written after the runner exited -- saw an empty env var
+         and fell straight through to the empty tree.
+      3. The empty tree -- correct for a GREENFIELD run (a repo with no commits
          at start), where "everything that now exists" IS the run's output and
          there is no earlier commit to diff against.
-      3. Empty string -- let workspace_diff apply its own fallbacks.
+      4. Empty string -- let workspace_diff apply its own fallbacks.
+
+    WHY STEP 2 EXISTS. Without it every receipt on this repo recorded
+    base_sha="" -- measured: 9 of 9 receipts in .loki/proofs/, and the dashboard
+    correctly reported "9 receipts, 0 verified". An empty base is unanchorable,
+    so outcome_ledger.resolve_anchor() returns unanchored/base_sha_empty and NO
+    receipt can be verified. The persisted file already existed and both other
+    consumers already read it (run.sh:7779, completion-council.sh:1834); this
+    reader was the only one that did not, so it silently lost the anchor.
+
+    The greenfield fallback is deliberately kept BELOW the file: falling back to
+    the empty tree when a real baseline exists would attest to "everything in
+    the repo" as this run's output, which is a much worse lie than an empty
+    base_sha. Order matters more than the addition.
     """
     base = os.environ.get("_LOKI_RUN_START_SHA", "").strip()
+    if not base:
+        # Same baseline, read from disk. Mirrors run.sh:7779 and
+        # completion-council.sh:1834 rather than inventing a third convention.
+        try:
+            with open(os.path.join(target_dir, ".loki", "state", "start-sha")) as fh:
+                _persisted = fh.read().strip()
+            # Only trust it if it names a commit that actually exists HERE. A
+            # stale or foreign SHA would produce a diff against nothing and an
+            # integrity hash nobody can recompute.
+            if _persisted and subprocess.run(
+                ["git", "cat-file", "-e", _persisted + "^{commit}"],
+                cwd=target_dir, capture_output=True, timeout=10,
+            ).returncode == 0:
+                base = _persisted
+        except (OSError, subprocess.SubprocessError, ValueError):
+            pass
     if not base:
         # Greenfield: no baseline commit existed when the run started.
         base = _empty_tree_sha(target_dir)
@@ -990,8 +1024,33 @@ def _collect_iterations(loki_dir):
     return {"count": count, "succeeded": n_completed, "failed": n_failed}
 
 
+# Acceptance-criterion ids as minted at intake ("- AC-<AXIS>-NNN: <text>", see
+# _brief_acceptance_criteria in autonomy/loki). Anchored and strict on purpose:
+# a loose pattern would count prose that merely mentions an id, and the whole
+# value of the id is that a citation points at exactly one criterion.
+_AC_ID_RE = re.compile(r"^- (AC-[A-Z]+-[0-9]{3}): ", re.MULTILINE)
+
+
+def _spec_criteria_declared(text):
+    """Return the acceptance-criterion ids the spec DECLARES, in spec order.
+
+    DECLARED, NOT SATISFIED. This records which criteria exist in the spec and
+    nothing more -- no check runs here, and no field in the receipt asserts that
+    any of these was met. Naming it criteria_met would be a lie we cannot back.
+
+    Empty list when the spec declares none (an older PRD, a hand-written spec,
+    or a run with no spec file at all). Never invented, never a placeholder.
+    """
+    if not text:
+        return []
+    # Deduped: an id is a citation target, so each must resolve to one criterion.
+    # A repeated id is a spec bug; counting it twice would not make it citable.
+    return list(dict.fromkeys(_AC_ID_RE.findall(text)))
+
+
 def _collect_spec(loki_dir, target_dir):
-    """Return spec dict {source, brief}. brief truncated to 600 chars."""
+    """Return spec dict {source, brief, criteria_declared}. brief truncated to
+    600 chars."""
     prd_path = os.environ.get("PRD_PATH", "").strip()
     source = ""
     brief = ""
@@ -1018,7 +1077,15 @@ def _collect_spec(loki_dir, target_dir):
     # Full brief here; the <=600 cap is applied AFTER redaction in generate()
     # so a secret straddling the cap cannot be sliced into an under-length
     # fragment that bypasses the redactor.
-    return {"source": source, "brief": brief}
+    #
+    # Criteria are parsed from the FULL spec text, not from the 600-char display
+    # cap: a PRD's acceptance-criteria block sits well past char 600, so parsing
+    # the capped brief would silently drop most of them.
+    return {
+        "source": source,
+        "brief": brief,
+        "criteria_declared": _spec_criteria_declared(brief),
+    }
 
 
 def _self_version():

package/autonomy/lib/verdict.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""One verdict a reviewer can read in ten seconds, assembled from measured parts.
+
+WHY THIS EXISTS. We now measure five things nobody else does -- did the work
+survive (outcome ledger), does the spec still match the intent (intent ledger),
+was this the agent's own output or a human rescue (pre-edit snapshot), does the
+completion claim name real work (claim grounding), and which model decided
+(decision record). Each is correct and each lives in its own file. A reviewer
+looking at a pull request reads none of them.
+
+A moat nobody sees is not a moat. The Evidence Receipt is already attached to
+PRs by default (LOKI_PROVEN_PR, autonomy/run.sh:3849), so the surface exists and
+this fills it: one block, five lines, each of which is either a measured fact or
+an explicit UNKNOWN.
+
+WHAT IT REFUSES, AND WHY THAT MATTERS MORE HERE THAN ANYWHERE. This is the only
+place where all five signals meet, which makes it the one place a composite score
+would be tempting: a single "trust: 87" would fit a PR comment beautifully. It is
+not offered. Averaging a revert count, a hash comparison, a diff hash, a path
+match and a model id produces a number whose movement nobody can explain, and a
+number nobody can explain is the thing our competitors already ship. Five honest
+lines beat one confident one.
+
+UNKNOWN IS PRINTED, NOT HIDDEN. A reviewer must be able to tell "we checked and
+it is fine" from "we could not check". Suppressing the unmeasured lines would
+make a receipt with one real signal look identical to one with five, which is
+exactly the false confidence the whole product exists to refuse.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+# Each signal: where it lives, and the one-line question it answers for a human
+# scanning a PR. The question text is load-bearing -- a reviewer who cannot tell
+# what a line MEANS will skip the block, and a skipped block is worth nothing.
+SIGNALS = (
+    ("outcome", "did previous work survive, or get reverted"),
+    ("intent", "does the spec still say what was actually wanted"),
+    ("authorship", "is this the agent's output, or a human rescue"),
+    ("grounding", "does the completion claim name work that exists in the diff"),
+    ("model", "which model decided, at what temperature"),
+)
+
+
+def _read_json(path):
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError):
+        return None
+
+
+def _outcome_line(loki_dir):
+    p = os.path.join(loki_dir, "proofs")
+    if not os.path.isdir(p):
+        return UNKNOWN, "no receipts yet, so nothing has an outcome to follow"
+    # Deliberately does not recompute: the outcome ledger owns that logic and two
+    # implementations of one number eventually disagree.
+    return UNKNOWN, "run `loki outcomes` for the measured post-merge result"
+
+
+def _intent_line(loki_dir):
+    p = os.path.join(loki_dir, "intent", "intent.json")
+    if not os.path.isfile(p):
+        return UNKNOWN, "no intent recorded, so spec-vs-intent drift is not measurable"
+    doc = _read_json(p)
+    if doc is None:
+        return UNKNOWN, "the intent record could not be read"
+    stmts = doc.get("statements") or []
+    if not stmts:
+        return UNKNOWN, "no intent statements recorded"
+    linked = sum(1 for s in stmts if s.get("links"))
+    return "measured", f"{len(stmts)} intent statement(s), {linked} linked to the spec"
+
+
+def _authorship_line(loki_dir, run_id=None):
+    d = os.path.join(loki_dir, "preedit")
+    if not os.path.isdir(d):
+        return UNKNOWN, "no pre-edit snapshot, so agent output cannot be told from human edits"
+    snaps = [f for f in os.listdir(d) if f.endswith(".json")]
+    if not snaps:
+        return UNKNOWN, "no pre-edit snapshot captured"
+    return "measured", f"{len(snaps)} run(s) have an immutable pre-edit snapshot"
+
+
+def _grounding_line(loki_dir):
+    # The completion route now persists the per-claim result here
+    # (run.sh:_loki_check_claim_grounding, written at the non-destructive claim
+    # peek). Absent file means the run never reached a completion claim, which
+    # is UNKNOWN -- not a pass, and not a failure.
+    d = _read_json(os.path.join(loki_dir, "state", "claim-grounding.json"))
+    if not isinstance(d, dict) or d.get("status") != "measured":
+        return UNKNOWN, "no completion claim was checked against the diff"
+    named = d.get("paths_named") or []
+    if not named:
+        # Reported by name rather than scored: a claim naming no path is
+        # UNGROUNDABLE, which is a different fact from a claim that checked out.
+        return UNKNOWN, "the completion claim named no file path, so it cannot be grounded"
+    ungrounded = d.get("ungrounded") or []
+    if d.get("has_ungrounded_claim"):
+        return "finding", (
+            f"the completion claim names {len(ungrounded)} path(s) absent from the diff: "
+            + ", ".join(str(p) for p in ungrounded[:3])
+        )
+    return "measured", f"all {len(named)} path(s) named in the completion claim are in the diff"
+
+
+def _model_line(loki_dir):
+    p = os.path.join(loki_dir, "decisions", "decisions.jsonl")
+    if not os.path.isfile(p):
+        return UNKNOWN, "no decision records, so a model swap would be undetectable"
+    models = set()
+    n = 0
+    try:
+        with open(p, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    r = json.loads(line)
+                except ValueError:
+                    continue
+                n += 1
+                if r.get("model_id"):
+                    models.add(r["model_id"])
+    except OSError:
+        return UNKNOWN, "the decision record could not be read"
+    if not n:
+        return UNKNOWN, "no decision records"
+    if len(models) > 1:
+        # The single most audit-relevant thing this block can say.
+        return "measured", f"{n} decisions across {len(models)} DIFFERENT models: {', '.join(sorted(models))}"
+    return "measured", f"{n} decisions, one model: {', '.join(models) or 'unnamed'}"
+
+
+def build(loki_dir, run_id=None):
+    rows = []
+    for key, question in SIGNALS:
+        if key == "outcome":
+            status, detail = _outcome_line(loki_dir)
+        elif key == "intent":
+            status, detail = _intent_line(loki_dir)
+        elif key == "authorship":
+            status, detail = _authorship_line(loki_dir, run_id)
+        elif key == "grounding":
+            status, detail = _grounding_line(loki_dir)
+        else:
+            status, detail = _model_line(loki_dir)
+        rows.append({"signal": key, "question": question,
+                     "status": status, "detail": detail})
+
+    measured = sum(1 for r in rows if r["status"] == "measured")
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "signals": rows,
+        "measured": measured,
+        "total": len(rows),
+        # No composite. See the module docstring: averaging a revert count, a
+        # hash comparison and a model id yields a number nobody can explain.
+    }
+
+
+def render_markdown(v):
+    """A PR comment block. Terse on purpose: a reviewer gives this ten seconds."""
+    out = ["### Loki verification signals", ""]
+    out.append(f"{v['measured']} of {v['total']} signals measured. "
+               "UNKNOWN means we could not check, not that it passed.")
+    out.append("")
+    out.append("| Signal | Status | Detail |")
+    out.append("|---|---|---|")
+    for r in v["signals"]:
+        # Three states, not two. Collapsing everything non-measured to UNKNOWN
+        # would render a real finding ("the claim names a file absent from the
+        # diff") identically to "we could not check" -- the false equivalence
+        # this whole module exists to refuse. Anything unrecognised still
+        # degrades to UNKNOWN, so an unmeasured signal can never read as a pass.
+        status = r["status"] if r["status"] in ("measured", "finding") else UNKNOWN
+        out.append(f"| {r['signal']} | {status} | {r['detail']} |")
+    out.append("")
+    out.append("Every line is derived from a file in `.loki/` that you can read "
+               "yourself. No score is offered: five honest lines beat one "
+               "confident number.")
+    return "\n".join(out)
+
+
+def main(argv):
+    as_json = "--json" in argv
+    cwd = os.environ.get("LOKI_VERDICT_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+    v = build(loki_dir)
+    print(json.dumps(v, indent=2) if as_json else render_markdown(v))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))