loki-mode 9.12.6 → 9.17.0

package/autonomy/lib/decision_record.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""LLM Decision Record: which model, at what temperature, decided what.
+
+WHY THIS EXISTS. Factory AI's audit log has eight event types and NOT ONE
+records an agent action -- they are all admin configuration (membership, API
+keys, integrations, managed settings). Agent forensics exists only as
+customer-built OTEL: metrics by default, message content opt-in, and the customer
+must stand up and retain the pipeline. So "which agent changed this line, on
+whose authority, and what did it verify" is answerable only if the buyer built
+the plumbing themselves. Devin's audit log is likewise session and admin scoped.
+
+8090's evaluation framework records, per AI operation: session id, pipeline
+stage, model id, temperature, timestamps, token counts, reasoning trace,
+confidence. Their stated reason is the one that matters to a regulated buyer:
+capturing model id and temperature makes a MODEL SWAP DETECTABLE. Without it,
+a provider silently changing a model underneath you is invisible in your own
+records, and "we used an approved configuration" becomes unfalsifiable.
+
+WHAT THIS IS. An append-only record of agent decisions, written next to the
+receipt so the chain of custody is complete: the receipt says what was proven,
+the outcome ledger says what happened afterwards, and this says what made the
+call. Append-only because an audit trail a later run can rewrite is not an audit
+trail.
+
+WHAT THIS DELIBERATELY DOES NOT CAPTURE. No prompt bodies, no file contents, no
+credentials, no environment. 8090's own boundary applies with more force to us
+than to them: an adoption tool that exfiltrates a user's environment would cost
+exactly the trust this product sells. Fields are a fixed allowlist, and a test
+asserts the allowlist so a future contributor cannot widen it casually.
+
+IT IS NOT TELEMETRY. Nothing here is transmitted. It writes a local JSONL file
+that the user owns and can read, diff, and delete. autonomy/telemetry.sh remains
+the single egress point.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+# The complete set of fields a record may carry. Anything not on this list is
+# dropped rather than written: a record that can grow a new field by accident is
+# how an audit log becomes a data-exfiltration surface. The test suite asserts
+# this exact set, so widening it is a deliberate, reviewed act.
+ALLOWED_FIELDS = (
+    "schema_version",
+    "recorded_at",
+    "session_id",
+    "run_id",
+    "stage",          # which part of the loop made the call
+    "model_id",       # THE field that makes a silent model swap detectable
+    "temperature",    # same: a config drift nobody announced
+    "provider",
+    "tokens_in",
+    "tokens_out",
+    "duration_ms",
+    "outcome",        # coarse: ok | error | refused | timeout
+    "confidence",     # self-reported, and labelled as such below
+)
+
+# Fields whose value is the AGENT'S OWN OPINION, never a measurement. Kept
+# separate so a reader cannot mistake a self-report for an observation -- the
+# same FACTS-vs-ASSESSMENTS split the receipts already enforce.
+SELF_REPORTED = ("confidence",)
+
+
+def _records_path(loki_dir):
+    return os.path.join(loki_dir, "decisions", "decisions.jsonl")
+
+
+def record(loki_dir, fields):
+    """Append one decision record. Returns the written record, or a reason.
+
+    Append-only by construction: opened "a", never "w". A trail a later run can
+    rewrite is not a trail, and the whole value here is that a reader can trust
+    what it says about a run that already finished.
+    """
+    clean = {}
+    dropped = []
+    for k, v in (fields or {}).items():
+        if k in ALLOWED_FIELDS:
+            clean[k] = v
+        else:
+            dropped.append(k)
+
+    if not clean.get("model_id"):
+        # A record without a model id cannot answer the question this exists to
+        # answer, so it is refused rather than written as a half-record.
+        return {"status": "UNKNOWN", "reason": "no_model_id",
+                "detail": "a decision record without model_id cannot make a swap detectable"}
+
+    clean["schema_version"] = SCHEMA_VERSION
+    clean["recorded_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    path = _records_path(loki_dir)
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(clean, sort_keys=True) + "\n")
+    except OSError as exc:
+        return {"status": "UNKNOWN", "reason": "unwritable", "detail": str(exc)}
+
+    out = {"status": "recorded", "path": path, "record": clean}
+    if dropped:
+        # Surfaced, not silent: a caller trying to log a prompt body should see
+        # that it was refused rather than assume it was stored.
+        out["dropped_fields"] = sorted(dropped)
+    return out
+
+
+def summarize(loki_dir):
+    """What models actually ran, and did the configuration change mid-flight?
+
+    The useful audit question is not "how many calls" but "did the thing that
+    decided change without anyone saying so". A run whose records name two model
+    ids is exactly the case a regulated buyer needs surfaced.
+    """
+    path = _records_path(loki_dir)
+    if not os.path.isfile(path):
+        return {"status": "UNKNOWN", "reason": "no_records",
+                "detail": "no decision records for this project"}
+
+    models, temps, stages, n, bad = {}, {}, {}, 0, 0
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                except ValueError:
+                    # A corrupt line is counted, never silently skipped: an audit
+                    # trail that quietly drops what it cannot parse is worse than
+                    # one that admits a gap.
+                    bad += 1
+                    continue
+                n += 1
+                m = rec.get("model_id")
+                if m:
+                    models[m] = models.get(m, 0) + 1
+                t = rec.get("temperature")
+                if t is not None:
+                    temps[str(t)] = temps.get(str(t), 0) + 1
+                s = rec.get("stage")
+                if s:
+                    stages[s] = stages.get(s, 0) + 1
+    except OSError as exc:
+        return {"status": "UNKNOWN", "reason": "unreadable", "detail": str(exc)}
+
+    return {
+        "status": "measured",
+        "records": n,
+        "unparseable_lines": bad,
+        "models": models,
+        "temperatures": temps,
+        "stages": stages,
+        # The headline fact. More than one model id, or more than one
+        # temperature, means the configuration changed during this project and
+        # the records prove it.
+        "model_changed": len(models) > 1,
+        "temperature_changed": len(temps) > 1,
+        "self_reported_fields": list(SELF_REPORTED),
+    }
+
+
+def main(argv):
+    if not argv:
+        print("usage: decision_record.py record|summary [--field=value ...]",
+              file=sys.stderr)
+        return 2
+    action = argv[0]
+    cwd = os.environ.get("LOKI_DECISION_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    if action == "record":
+        fields = {}
+        for arg in argv[1:]:
+            if arg.startswith("--") and "=" in arg:
+                k, v = arg[2:].split("=", 1)
+                fields[k] = v
+        res = record(loki_dir, fields)
+    elif action == "summary":
+        res = summarize(loki_dir)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("recorded", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/failure_memory.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""Failure memory: the agent gets better at YOUR repo by having been wrong in it.
+
+WHY THIS EXISTS. Neither competitor learns from being wrong, and one of them
+shipped a UI to manage that fact.
+
+Factory AI has no memory system at all -- verified by grep over their whole doc
+corpus. AGENTS.md is hand-authored by the human, AutoWiki regenerates from CODE
+rather than from outcomes, QA failure learning is `suggest_in_report` (a human
+reads it and decides), and mission-generated skills live under {missionDir} and
+are DISCARDED with the mission. Devin has a real memory layer, but every session
+starts from a fresh snapshot copy and all changes are discarded at teardown --
+and their Session Insights ships a "Misleading Knowledge" tab enumerating memory
+items that led Devin astray. They built an interface for their memory poisoning
+output.
+
+So on both systems, an agent that failed a gate in your repo yesterday starts
+today knowing nothing about it.
+
+WHAT MAKES THIS DIFFERENT, AND WHY IT IS NOT THE SAME TRAP. The reason Devin
+needed a "Misleading Knowledge" tab is that memory written from an agent's own
+narration is unfalsifiable: it records what the agent BELIEVED, which is exactly
+what was wrong when it failed. So nothing here is written from narration. A
+lesson is created only from a MEASURED event -- a named gate that failed, with
+its verdict -- and it carries the evidence that produced it. A lesson whose
+evidence no longer holds can be retired mechanically instead of accumulating.
+
+This is deliberately built on the outcome ledger's discipline: a lesson is
+recorded only when the event that justifies it is a fact, and everything else
+reports UNKNOWN with a named reason rather than being written as a weak guess.
+
+WHAT IT DOES NOT DO. It does not summarize, generalize, or ask a model what the
+lesson "means". A generalization is a judgement, and a judgement stored as memory
+is indistinguishable from a fact by the next reader -- which is the poisoning
+mechanism. It stores the gate, the verdict, the evidence, and a count. Deciding
+what that implies stays with whoever reads it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+REASONS = {
+    "no_gate": "no gate name was given, so there is nothing to attribute the failure to",
+    "no_evidence": "no evidence was given, so the lesson would be unfalsifiable",
+    "no_lessons": "no failure lessons recorded for this project",
+    "unreadable": "the lesson store exists but could not be parsed",
+}
+
+
+def _path(loki_dir):
+    return os.path.join(loki_dir, "memory", "failures.jsonl")
+
+
+def record_failure(loki_dir, gate, verdict, evidence, run_id=None):
+    """Turn a measured gate failure into a durable, falsifiable lesson.
+
+    Requires EVIDENCE. A lesson without it is the agent's own account of why it
+    failed, which is precisely the unfalsifiable memory that made Devin's need a
+    "Misleading Knowledge" tab. If we cannot say what was observed, we do not
+    write anything.
+    """
+    if not gate:
+        return {"status": UNKNOWN, "reason": "no_gate", "detail": REASONS["no_gate"]}
+    if not evidence:
+        return {"status": UNKNOWN, "reason": "no_evidence",
+                "detail": REASONS["no_evidence"]}
+
+    rec = {
+        "schema_version": SCHEMA_VERSION,
+        "recorded_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "gate": gate,
+        "verdict": verdict or "FAIL",
+        # The falsifiability anchor. A future reader can check whether this still
+        # holds instead of taking the lesson on faith.
+        "evidence": evidence,
+        "run_id": run_id or "",
+        # Deliberately absent: any summary, generalization, or "what this means".
+        # A judgement stored beside facts becomes indistinguishable from one.
+    }
+
+    p = _path(loki_dir)
+    try:
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with open(p, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(rec, sort_keys=True) + "\n")
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "unreadable", "detail": str(exc)}
+    return {"status": "recorded", "path": p, "record": rec}
+
+
+def recall(loki_dir, gate=None):
+    """What has actually gone wrong here before.
+
+    Returns counts per gate, most-failed first. Counts, not prose: "the mock
+    integrity gate has failed here 6 times" is a fact a reader can act on, while
+    "this repo tends to have mocking problems" is a generalization that sounds
+    the same and is not checkable.
+    """
+    p = _path(loki_dir)
+    if not os.path.isfile(p):
+        return {"status": UNKNOWN, "reason": "no_lessons",
+                "detail": REASONS["no_lessons"]}
+
+    per_gate, recent, bad = {}, [], 0
+    try:
+        with open(p, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    r = json.loads(line)
+                except ValueError:
+                    # Counted, never silently skipped: a memory that quietly
+                    # drops what it cannot read is worse than one that admits it.
+                    bad += 1
+                    continue
+                g = r.get("gate")
+                if not g or (gate and g != gate):
+                    continue
+                per_gate[g] = per_gate.get(g, 0) + 1
+                recent.append(r)
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "unreadable", "detail": str(exc)}
+
+    if not per_gate:
+        return {"status": UNKNOWN, "reason": "no_lessons",
+                "detail": REASONS["no_lessons"]}
+
+    return {
+        "status": "measured",
+        "unparseable_lines": bad,
+        "by_gate": dict(sorted(per_gate.items(), key=lambda kv: -kv[1])),
+        "total": sum(per_gate.values()),
+        # Newest last so a reader sees the current state at the bottom, matching
+        # how the file itself is ordered.
+        "recent": recent[-5:],
+    }
+
+
+def prompt_context(loki_dir, limit=3):
+    """The lines worth putting in front of the next run.
+
+    Returns bare facts. A repo where the same gate has failed repeatedly is
+    information the next iteration should have; what to DO about it is left to
+    the agent, because prescribing the fix from a count would be inventing a
+    causal claim the data does not contain.
+    """
+    r = recall(loki_dir)
+    if r.get("status") != "measured":
+        return {"status": r.get("status"), "reason": r.get("reason"), "lines": []}
+    lines = []
+    for gate, n in list(r["by_gate"].items())[:limit]:
+        if n > 1:
+            lines.append(f"the {gate} gate has failed {n} times in this repo before")
+        else:
+            lines.append(f"the {gate} gate has failed here before")
+    return {"status": "measured", "lines": lines}
+
+
+def main(argv):
+    if not argv:
+        print("usage: failure_memory.py record|recall|context [...]", file=sys.stderr)
+        return 2
+    action = argv[0]
+    cwd = os.environ.get("LOKI_FAILMEM_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    kv = {}
+    for a in argv[1:]:
+        if a.startswith("--") and "=" in a:
+            k, v = a[2:].split("=", 1)
+            kv[k] = v
+
+    if action == "record":
+        res = record_failure(loki_dir, kv.get("gate"), kv.get("verdict"),
+                             kv.get("evidence"), kv.get("run_id"))
+    elif action == "recall":
+        res = recall(loki_dir, kv.get("gate"))
+    elif action == "context":
+        res = prompt_context(loki_dir)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("recorded", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/gate_policy.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Which gates block, which only advise, and what promoting one would have cost.
+
+Ona's Veto Exec ships an audit-first ladder: "Start with audit rules, review
+matches, then promote confirmed rules to block." The load-bearing part is the
+MIDDLE step. A policy you cannot safely turn on is a policy nobody turns on, so
+before flipping a gate to blocking you get to see what it WOULD have blocked.
+
+We already had both ends and nothing in between: gates are advisory or blocking,
+three promotion knobs exist (LOKI_GATE_MAGIC_DEBATE_BLOCKING, LOKI_COV_ENFORCE,
+LOKI_POLICY_APPROVAL_ENFORCE), and .loki/quality/gate-failure-count.json has
+counted per-gate failures the whole time. Nothing joined them, so an operator
+deciding whether to promote a gate had to guess.
+
+DETERMINISTIC. Reads two files and the environment. No model, no network, no
+spend -- same repo state, same answer, and every number is a count a reader can
+recompute by opening the same file.
+
+NEVER PROMOTES ANYTHING. This reports; it does not change policy. Turning a gate
+blocking stays an explicit operator act via the named environment variable,
+because a tool that silently starts blocking is the thing operators most
+reasonably fear.
+
+Shape (assess() and --json, schema_version 1). This is the contract the
+dashboard endpoint GET /api/gate-policy and tests/test_gate_policy_endpoint.py
+both read against, so field names here are load-bearing:
+
+  schema_version int    1
+  status         str    "measured"
+  ledger         str    "present" | "absent" -- whether the per-gate failure
+                        ledger .loki/quality/gate-failure-count.json was read
+  gates          list   one record per known gate, blocking gates first, each
+                        group sorted by name:
+    gate         str    gate name (e.g. "code_review")
+    mode         str    "blocking" | "advisory" -- for a promotable gate this
+                        depends on the ENVIRONMENT at call time
+    promotable   bool   True when a real promotion knob exists in run.sh
+    audit_hits   int|null  failures counted for this gate. null means
+                        UNMEASURED -- no ledger, or no entry for this gate.
+                        NEVER 0 for an unmeasured gate: 0 is the positive claim
+                        that the gate ran and never fired, which is the false
+                        green an absent measurement always produces.
+    why          str    one-line description of what the gate checks
+    promote_with str|null  "VAR=value" to make an advisory gate blocking; null
+                        when the gate already blocks
+"""
+
+import json
+import os
+import sys
+
+SCHEMA_VERSION = 1
+
+# Gates that can be promoted from advisory to blocking, and the knob that does
+# it. Only gates with a REAL knob in run.sh appear here -- listing an aspiration
+# would tell an operator to set a variable nothing reads.
+PROMOTABLE = {
+    "magic_debate": ("LOKI_GATE_MAGIC_DEBATE_BLOCKING", "true",
+                     "spec-vs-implementation debate on generated modules"),
+    "test_coverage": ("LOKI_COV_ENFORCE", "1",
+                      "project test runner pass/fail"),
+    "policy_approval": ("LOKI_POLICY_APPROVAL_ENFORCE", "1",
+                        "staged-autonomy approval policy"),
+}
+
+# Gates that block unconditionally. Listed so the report is a complete picture
+# rather than only the promotable subset -- an operator asking "what blocks here"
+# should not have to read run.sh to find out.
+ALWAYS_BLOCKING = {
+    "static_analysis": "CodeQL, ESLint/Pylint, type-checker findings on the diff",
+    "code_review": "3-reviewer blind review; Critical/High = BLOCK",
+    "mock_integrity": "tautological-assertion and mock-ratio detection",
+    "mutation_integrity": "assertion-churn (test-fitting) detection",
+}
+
+
+def _counts(loki_dir):
+    """Per-gate failure counts, or {} when the ledger has not been written."""
+    path = os.path.join(loki_dir, "quality", "gate-failure-count.json")
+    try:
+        with open(path) as fh:
+            data = json.load(fh)
+        return data if isinstance(data, dict) else {}
+    except (OSError, ValueError):
+        return {}
+
+
+def assess(loki_dir=".loki", env=None):
+    env = os.environ if env is None else env
+    counts = _counts(loki_dir)
+    have_ledger = bool(counts)
+
+    gates = []
+    for name, why in sorted(ALWAYS_BLOCKING.items()):
+        gates.append({
+            "gate": name, "mode": "blocking", "promotable": False,
+            "audit_hits": counts.get(name) if have_ledger else None,
+            "why": why, "promote_with": None,
+        })
+    for name, (var, val, why) in sorted(PROMOTABLE.items()):
+        on = str(env.get(var, "")).lower() in ("1", "true", "yes")
+        gates.append({
+            "gate": name,
+            "mode": "blocking" if on else "advisory",
+            "promotable": True,
+            # None, not 0: an absent ledger means UNMEASURED, and reporting 0
+            # would read as "this gate never fired" -- the same false green a
+            # missing measurement always produces.
+            "audit_hits": counts.get(name) if have_ledger else None,
+            "why": why,
+            "promote_with": None if on else f"{var}={val}",
+        })
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "status": "measured",
+        "ledger": "present" if have_ledger else "absent",
+        "gates": gates,
+    }
+
+
+def render_text(res):
+    out = ["Gate policy -- what blocks here, and what only advises", ""]
+    for g in res["gates"]:
+        hits = g["audit_hits"]
+        if hits is None:
+            hit_s = "not measured"
+        elif hits == 0:
+            hit_s = "0 hits"
+        else:
+            hit_s = f"{hits} hit{'s' if hits != 1 else ''}"
+        mode = g["mode"].upper()
+        out.append(f"  {mode:9} {g['gate']:20} {hit_s}")
+        out.append(f"            {g['why']}")
+        if g["promote_with"]:
+            verb = "would have blocked" if (hits or 0) > 0 else "has not fired"
+            out.append(f"            advisory: {verb} -- promote with {g['promote_with']}")
+        out.append("")
+
+    if res["ledger"] == "absent":
+        out.append("  No gate ledger yet (.loki/quality/gate-failure-count.json).")
+        out.append("  Counts read 'not measured' rather than 0: an absent")
+        out.append("  measurement is not evidence a gate never fired.")
+    else:
+        out.append("  Hits come from .loki/quality/gate-failure-count.json.")
+        out.append("  Open it and count the same numbers yourself.")
+    out.append("")
+    out.append("  This command never promotes a gate. Promotion is an explicit")
+    out.append("  operator act via the variable named above.")
+    return "\n".join(out)
+
+
+def main(argv):
+    as_json = "--json" in argv
+    loki_dir = ".loki"
+    for a in argv:
+        if not a.startswith("-"):
+            loki_dir = a
+            break
+    res = assess(loki_dir)
+    print(json.dumps(res, indent=2) if as_json else render_text(res))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))