loki-mode 9.12.6 → 9.16.0

package/autonomy/lib/claim_grounding.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Claim grounding: a completion claim must name work that exists in the diff.
+
+WHY THIS EXISTS. 8090 AI's evaluation framework blocks ungrounded output AT
+GENERATION rather than catching it at review, and states the payoff plainly:
+
+    "Ungrounded claims are blocked at generation. The rubric measures the quality
+     of what survives the architectural filter, which is a much smaller and more
+     interesting space."
+
+Our evidence gate already has six axes -- diff non-empty, tests green, runtime
+boot, no-mock, authorization, secret leak. Every one is a REPO-LEVEL fact. None
+reads what the agent actually CLAIMED. So an agent can finish by asserting "added
+retry logic to the payment client and covered it with tests" while the diff shows
+a README edit, and every axis passes: the diff is non-empty, the tests are green,
+the app boots. The claim itself is the one artifact nobody checks.
+
+WHAT THIS CHECKS, AND WHAT IT REFUSES TO CHECK. It resolves file-path-shaped
+tokens in the claim against the actual changed-file set. That is a deterministic
+string-to-set comparison, and it is the only part of a natural-language claim
+that can be checked without a model.
+
+It does NOT judge whether the claim is semantically true. "Added retry logic" vs
+"added a retry constant" is a judgement, and asking an LLM to grade it would be
+the same LLM-judge-as-measurement this codebase refuses everywhere else. A claim
+naming no paths is UNGROUNDABLE, reported by name -- never scored, never failed.
+
+THE FAIL-OPEN DIRECTION IS DELIBERATE. Only a claim that names a path which is
+demonstrably NOT in the diff is a finding. A claim with no paths, a claim naming
+a path that exists but was not touched by this run, and an empty claim are all
+reported and none of them blocks. A grounding check that blocked on ambiguity
+would fire constantly on ordinary prose and be disabled within a week, which is
+worse than not having it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+GROUNDING_REASONS = {
+    "no_claim": "no completion claim text was provided",
+    "no_diff": "no changed-file set was provided to check against",
+    "ungroundable": "the claim names no file paths, so it cannot be checked mechanically",
+}
+
+# A path-shaped token: at least one slash or a known source extension, and no
+# spaces. Deliberately conservative -- a false "this is a path" produces a false
+# finding, and a check that cries wolf gets turned off.
+_PATH_RE = re.compile(
+    r"(?<![\w/.-])"
+    r"(?:[\w.-]+/)+[\w.-]+\.[A-Za-z0-9]{1,6}"
+    r"|(?<![\w/.-])[\w.-]+\.(?:py|ts|tsx|js|jsx|sh|go|rs|rb|java|c|h|cpp|md|json|ya?ml|toml)"
+    r"(?![\w/.-])"
+)
+
+
+def extract_paths(claim):
+    """Path-shaped tokens in a claim, deduped, order preserved."""
+    if not claim:
+        return []
+    seen, out = set(), []
+    for m in _PATH_RE.finditer(claim):
+        tok = m.group(0).strip(".,;:)(\"'`")
+        if tok and tok not in seen:
+            seen.add(tok)
+            out.append(tok)
+    return out
+
+
+def check(claim, changed_files):
+    """Are the paths a claim names present in the diff?
+
+    Matching is suffix-based on purpose: an agent writes `run.sh` or
+    `autonomy/run.sh` for the same file, and demanding an exact repo-relative
+    string would make every informal mention a false finding.
+    """
+    if not claim or not claim.strip():
+        return {"status": UNKNOWN, "reason": "no_claim",
+                "detail": GROUNDING_REASONS["no_claim"]}
+    if changed_files is None:
+        return {"status": UNKNOWN, "reason": "no_diff",
+                "detail": GROUNDING_REASONS["no_diff"]}
+
+    named = extract_paths(claim)
+    if not named:
+        # The common, benign case: "fixed the login bug". Nothing to check, and
+        # that is not a defect -- reported so the number of unverifiable claims
+        # is visible, rather than silently counted as grounded.
+        return {"status": UNKNOWN, "reason": "ungroundable",
+                "detail": GROUNDING_REASONS["ungroundable"],
+                "paths_named": []}
+
+    changed = list(changed_files)
+    grounded, ungrounded = [], []
+    for p in named:
+        norm = p.lstrip("./")
+        # Basename fallback is load-bearing, not laxity. An agent writes `run.sh`
+        # for `autonomy/run.sh`, and requiring the full repo-relative string made
+        # every informal mention a false finding -- measured: "patched run.sh"
+        # against a changed autonomy/run.sh was reported UNGROUNDED. A grounding
+        # check that flags correct claims is worse than none, because it gets
+        # disabled and takes the real detections with it.
+        base = os.path.basename(norm)
+        hit = any(
+            c == norm
+            or c.endswith("/" + norm)
+            or norm.endswith("/" + c)
+            or os.path.basename(c) == base
+            for c in changed
+        )
+        (grounded if hit else ungrounded).append(p)
+
+    return {
+        "status": "measured",
+        "paths_named": named,
+        "grounded": grounded,
+        "ungrounded": ungrounded,
+        # The single actionable signal. A claim that names a file the run never
+        # touched is the "agent says done, diff says otherwise" failure, caught
+        # from the claim side instead of the repo side.
+        "has_ungrounded_claim": bool(ungrounded),
+    }
+
+
+def main(argv):
+    claim = ""
+    files_arg = ""
+    files_from = ""
+    i = 0
+    while i < len(argv):
+        if argv[i] == "--claim" and i + 1 < len(argv):
+            claim = argv[i + 1]; i += 2; continue
+        if argv[i] == "--files" and i + 1 < len(argv):
+            files_arg = argv[i + 1]; i += 2; continue
+        if argv[i] == "--files-from" and i + 1 < len(argv):
+            files_from = argv[i + 1]; i += 2; continue
+        i += 1
+
+    # --files is ALWAYS a comma-separated list; --files-from is always a file to
+    # read. The first version overloaded one flag for both and picked with
+    # os.path.isfile(), which silently misread `--files autonomy/run.sh` -- a
+    # perfectly ordinary changed-file list -- as "open that file and treat its
+    # 20,000 lines as filenames". The result was a correct claim reported
+    # UNGROUNDED, i.e. the exact false positive this check must never produce.
+    # An ambiguous flag whose meaning depends on the filesystem is a bug, not a
+    # convenience.
+    if files_from and os.path.isfile(files_from):
+        with open(files_from, "r", encoding="utf-8") as fh:
+            changed = [l.strip() for l in fh if l.strip()]
+    elif files_arg:
+        changed = [c.strip() for c in files_arg.split(",") if c.strip()]
+    else:
+        changed = None
+
+    res = check(claim, changed)
+    res["schema_version"] = SCHEMA_VERSION
+    print(json.dumps(res, indent=2))
+    # Exit 1 ONLY on a demonstrably ungrounded path. Every UNKNOWN exits 0: a
+    # check that failed on ambiguity would fire on ordinary prose and be disabled.
+    return 1 if res.get("has_ungrounded_claim") else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/config-map.sh

@@ -863,13 +863,17 @@ loki_config_validate_file() {
     pairs="$(
         _loki_cfg_collect_pairs() {
             local f="$1" fm="$2"
+            # NOTE: every case pattern below carries a leading open-paren --
+            # bash 3.2 (macOS /bin/bash) ends a command substitution at the
+            # close-paren of a case label, truncating this function body. The
+            # balanced form parses identically on 3.2 and 4+. Do not strip it.
             case "$fm" in
-                env)
+                (env)
                     local line key val
                     while IFS= read -r line || [ -n "$line" ]; do
-                        case "$line" in ''|'#'*) continue ;; esac
+                        case "$line" in (''|'#'*) continue ;; esac
                         line="${line#export }"
-                        case "$line" in *=*) ;; *) continue ;; esac
+                        case "$line" in (*=*) ;; (*) continue ;; esac
                         key="${line%%=*}"; val="${line#*=}"
                         key="${key#"${key%%[![:space:]]*}"}"; key="${key%"${key##*[![:space:]]}"}"
                         val="${val#"${val%%[![:space:]]*}"}"
@@ -878,7 +882,7 @@ loki_config_validate_file() {
                         printf '%s\t%s\n' "$key" "$val"
                     done < "$f"
                     ;;
-                yaml)
+                (yaml)
                     local mapping yaml_path env_var value have_yq=0
                     if command -v yq >/dev/null 2>&1; then have_yq=1; fi
                     for mapping in "${LOKI_CONFIG_MAP[@]}"; do
@@ -894,8 +898,8 @@ loki_config_validate_file() {
                         printf '%s\t%s\n' "$env_var" "$value"
                     done
                     ;;
-                json)
-                    # Reuse the json parser's emit path by calling a print-only variant.
+                (json)
+                    # Reuse the JSON parser emit path via a print-only variant.
                     _loki_cfg_json_emit "$f"
                     ;;
             esac

package/autonomy/lib/decision_record.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""LLM Decision Record: which model, at what temperature, decided what.
+
+WHY THIS EXISTS. Factory AI's audit log has eight event types and NOT ONE
+records an agent action -- they are all admin configuration (membership, API
+keys, integrations, managed settings). Agent forensics exists only as
+customer-built OTEL: metrics by default, message content opt-in, and the customer
+must stand up and retain the pipeline. So "which agent changed this line, on
+whose authority, and what did it verify" is answerable only if the buyer built
+the plumbing themselves. Devin's audit log is likewise session and admin scoped.
+
+8090's evaluation framework records, per AI operation: session id, pipeline
+stage, model id, temperature, timestamps, token counts, reasoning trace,
+confidence. Their stated reason is the one that matters to a regulated buyer:
+capturing model id and temperature makes a MODEL SWAP DETECTABLE. Without it,
+a provider silently changing a model underneath you is invisible in your own
+records, and "we used an approved configuration" becomes unfalsifiable.
+
+WHAT THIS IS. An append-only record of agent decisions, written next to the
+receipt so the chain of custody is complete: the receipt says what was proven,
+the outcome ledger says what happened afterwards, and this says what made the
+call. Append-only because an audit trail a later run can rewrite is not an audit
+trail.
+
+WHAT THIS DELIBERATELY DOES NOT CAPTURE. No prompt bodies, no file contents, no
+credentials, no environment. 8090's own boundary applies with more force to us
+than to them: an adoption tool that exfiltrates a user's environment would cost
+exactly the trust this product sells. Fields are a fixed allowlist, and a test
+asserts the allowlist so a future contributor cannot widen it casually.
+
+IT IS NOT TELEMETRY. Nothing here is transmitted. It writes a local JSONL file
+that the user owns and can read, diff, and delete. autonomy/telemetry.sh remains
+the single egress point.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+# The complete set of fields a record may carry. Anything not on this list is
+# dropped rather than written: a record that can grow a new field by accident is
+# how an audit log becomes a data-exfiltration surface. The test suite asserts
+# this exact set, so widening it is a deliberate, reviewed act.
+ALLOWED_FIELDS = (
+    "schema_version",
+    "recorded_at",
+    "session_id",
+    "run_id",
+    "stage",          # which part of the loop made the call
+    "model_id",       # THE field that makes a silent model swap detectable
+    "temperature",    # same: a config drift nobody announced
+    "provider",
+    "tokens_in",
+    "tokens_out",
+    "duration_ms",
+    "outcome",        # coarse: ok | error | refused | timeout
+    "confidence",     # self-reported, and labelled as such below
+)
+
+# Fields whose value is the AGENT'S OWN OPINION, never a measurement. Kept
+# separate so a reader cannot mistake a self-report for an observation -- the
+# same FACTS-vs-ASSESSMENTS split the receipts already enforce.
+SELF_REPORTED = ("confidence",)
+
+
+def _records_path(loki_dir):
+    return os.path.join(loki_dir, "decisions", "decisions.jsonl")
+
+
+def record(loki_dir, fields):
+    """Append one decision record. Returns the written record, or a reason.
+
+    Append-only by construction: opened "a", never "w". A trail a later run can
+    rewrite is not a trail, and the whole value here is that a reader can trust
+    what it says about a run that already finished.
+    """
+    clean = {}
+    dropped = []
+    for k, v in (fields or {}).items():
+        if k in ALLOWED_FIELDS:
+            clean[k] = v
+        else:
+            dropped.append(k)
+
+    if not clean.get("model_id"):
+        # A record without a model id cannot answer the question this exists to
+        # answer, so it is refused rather than written as a half-record.
+        return {"status": "UNKNOWN", "reason": "no_model_id",
+                "detail": "a decision record without model_id cannot make a swap detectable"}
+
+    clean["schema_version"] = SCHEMA_VERSION
+    clean["recorded_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    path = _records_path(loki_dir)
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(clean, sort_keys=True) + "\n")
+    except OSError as exc:
+        return {"status": "UNKNOWN", "reason": "unwritable", "detail": str(exc)}
+
+    out = {"status": "recorded", "path": path, "record": clean}
+    if dropped:
+        # Surfaced, not silent: a caller trying to log a prompt body should see
+        # that it was refused rather than assume it was stored.
+        out["dropped_fields"] = sorted(dropped)
+    return out
+
+
+def summarize(loki_dir):
+    """What models actually ran, and did the configuration change mid-flight?
+
+    The useful audit question is not "how many calls" but "did the thing that
+    decided change without anyone saying so". A run whose records name two model
+    ids is exactly the case a regulated buyer needs surfaced.
+    """
+    path = _records_path(loki_dir)
+    if not os.path.isfile(path):
+        return {"status": "UNKNOWN", "reason": "no_records",
+                "detail": "no decision records for this project"}
+
+    models, temps, stages, n, bad = {}, {}, {}, 0, 0
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                except ValueError:
+                    # A corrupt line is counted, never silently skipped: an audit
+                    # trail that quietly drops what it cannot parse is worse than
+                    # one that admits a gap.
+                    bad += 1
+                    continue
+                n += 1
+                m = rec.get("model_id")
+                if m:
+                    models[m] = models.get(m, 0) + 1
+                t = rec.get("temperature")
+                if t is not None:
+                    temps[str(t)] = temps.get(str(t), 0) + 1
+                s = rec.get("stage")
+                if s:
+                    stages[s] = stages.get(s, 0) + 1
+    except OSError as exc:
+        return {"status": "UNKNOWN", "reason": "unreadable", "detail": str(exc)}
+
+    return {
+        "status": "measured",
+        "records": n,
+        "unparseable_lines": bad,
+        "models": models,
+        "temperatures": temps,
+        "stages": stages,
+        # The headline fact. More than one model id, or more than one
+        # temperature, means the configuration changed during this project and
+        # the records prove it.
+        "model_changed": len(models) > 1,
+        "temperature_changed": len(temps) > 1,
+        "self_reported_fields": list(SELF_REPORTED),
+    }
+
+
+def main(argv):
+    if not argv:
+        print("usage: decision_record.py record|summary [--field=value ...]",
+              file=sys.stderr)
+        return 2
+    action = argv[0]
+    cwd = os.environ.get("LOKI_DECISION_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    if action == "record":
+        fields = {}
+        for arg in argv[1:]:
+            if arg.startswith("--") and "=" in arg:
+                k, v = arg[2:].split("=", 1)
+                fields[k] = v
+        res = record(loki_dir, fields)
+    elif action == "summary":
+        res = summarize(loki_dir)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("recorded", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/failure_memory.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""Failure memory: the agent gets better at YOUR repo by having been wrong in it.
+
+WHY THIS EXISTS. Neither competitor learns from being wrong, and one of them
+shipped a UI to manage that fact.
+
+Factory AI has no memory system at all -- verified by grep over their whole doc
+corpus. AGENTS.md is hand-authored by the human, AutoWiki regenerates from CODE
+rather than from outcomes, QA failure learning is `suggest_in_report` (a human
+reads it and decides), and mission-generated skills live under {missionDir} and
+are DISCARDED with the mission. Devin has a real memory layer, but every session
+starts from a fresh snapshot copy and all changes are discarded at teardown --
+and their Session Insights ships a "Misleading Knowledge" tab enumerating memory
+items that led Devin astray. They built an interface for their memory poisoning
+output.
+
+So on both systems, an agent that failed a gate in your repo yesterday starts
+today knowing nothing about it.
+
+WHAT MAKES THIS DIFFERENT, AND WHY IT IS NOT THE SAME TRAP. The reason Devin
+needed a "Misleading Knowledge" tab is that memory written from an agent's own
+narration is unfalsifiable: it records what the agent BELIEVED, which is exactly
+what was wrong when it failed. So nothing here is written from narration. A
+lesson is created only from a MEASURED event -- a named gate that failed, with
+its verdict -- and it carries the evidence that produced it. A lesson whose
+evidence no longer holds can be retired mechanically instead of accumulating.
+
+This is deliberately built on the outcome ledger's discipline: a lesson is
+recorded only when the event that justifies it is a fact, and everything else
+reports UNKNOWN with a named reason rather than being written as a weak guess.
+
+WHAT IT DOES NOT DO. It does not summarize, generalize, or ask a model what the
+lesson "means". A generalization is a judgement, and a judgement stored as memory
+is indistinguishable from a fact by the next reader -- which is the poisoning
+mechanism. It stores the gate, the verdict, the evidence, and a count. Deciding
+what that implies stays with whoever reads it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+REASONS = {
+    "no_gate": "no gate name was given, so there is nothing to attribute the failure to",
+    "no_evidence": "no evidence was given, so the lesson would be unfalsifiable",
+    "no_lessons": "no failure lessons recorded for this project",
+    "unreadable": "the lesson store exists but could not be parsed",
+}
+
+
+def _path(loki_dir):
+    return os.path.join(loki_dir, "memory", "failures.jsonl")
+
+
+def record_failure(loki_dir, gate, verdict, evidence, run_id=None):
+    """Turn a measured gate failure into a durable, falsifiable lesson.
+
+    Requires EVIDENCE. A lesson without it is the agent's own account of why it
+    failed, which is precisely the unfalsifiable memory that made Devin's need a
+    "Misleading Knowledge" tab. If we cannot say what was observed, we do not
+    write anything.
+    """
+    if not gate:
+        return {"status": UNKNOWN, "reason": "no_gate", "detail": REASONS["no_gate"]}
+    if not evidence:
+        return {"status": UNKNOWN, "reason": "no_evidence",
+                "detail": REASONS["no_evidence"]}
+
+    rec = {
+        "schema_version": SCHEMA_VERSION,
+        "recorded_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "gate": gate,
+        "verdict": verdict or "FAIL",
+        # The falsifiability anchor. A future reader can check whether this still
+        # holds instead of taking the lesson on faith.
+        "evidence": evidence,
+        "run_id": run_id or "",
+        # Deliberately absent: any summary, generalization, or "what this means".
+        # A judgement stored beside facts becomes indistinguishable from one.
+    }
+
+    p = _path(loki_dir)
+    try:
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with open(p, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(rec, sort_keys=True) + "\n")
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "unreadable", "detail": str(exc)}
+    return {"status": "recorded", "path": p, "record": rec}
+
+
+def recall(loki_dir, gate=None):
+    """What has actually gone wrong here before.
+
+    Returns counts per gate, most-failed first. Counts, not prose: "the mock
+    integrity gate has failed here 6 times" is a fact a reader can act on, while
+    "this repo tends to have mocking problems" is a generalization that sounds
+    the same and is not checkable.
+    """
+    p = _path(loki_dir)
+    if not os.path.isfile(p):
+        return {"status": UNKNOWN, "reason": "no_lessons",
+                "detail": REASONS["no_lessons"]}
+
+    per_gate, recent, bad = {}, [], 0
+    try:
+        with open(p, "r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    r = json.loads(line)
+                except ValueError:
+                    # Counted, never silently skipped: a memory that quietly
+                    # drops what it cannot read is worse than one that admits it.
+                    bad += 1
+                    continue
+                g = r.get("gate")
+                if not g or (gate and g != gate):
+                    continue
+                per_gate[g] = per_gate.get(g, 0) + 1
+                recent.append(r)
+    except OSError as exc:
+        return {"status": UNKNOWN, "reason": "unreadable", "detail": str(exc)}
+
+    if not per_gate:
+        return {"status": UNKNOWN, "reason": "no_lessons",
+                "detail": REASONS["no_lessons"]}
+
+    return {
+        "status": "measured",
+        "unparseable_lines": bad,
+        "by_gate": dict(sorted(per_gate.items(), key=lambda kv: -kv[1])),
+        "total": sum(per_gate.values()),
+        # Newest last so a reader sees the current state at the bottom, matching
+        # how the file itself is ordered.
+        "recent": recent[-5:],
+    }
+
+
+def prompt_context(loki_dir, limit=3):
+    """The lines worth putting in front of the next run.
+
+    Returns bare facts. A repo where the same gate has failed repeatedly is
+    information the next iteration should have; what to DO about it is left to
+    the agent, because prescribing the fix from a count would be inventing a
+    causal claim the data does not contain.
+    """
+    r = recall(loki_dir)
+    if r.get("status") != "measured":
+        return {"status": r.get("status"), "reason": r.get("reason"), "lines": []}
+    lines = []
+    for gate, n in list(r["by_gate"].items())[:limit]:
+        if n > 1:
+            lines.append(f"the {gate} gate has failed {n} times in this repo before")
+        else:
+            lines.append(f"the {gate} gate has failed here before")
+    return {"status": "measured", "lines": lines}
+
+
+def main(argv):
+    if not argv:
+        print("usage: failure_memory.py record|recall|context [...]", file=sys.stderr)
+        return 2
+    action = argv[0]
+    cwd = os.environ.get("LOKI_FAILMEM_CWD") or os.getcwd()
+    loki_dir = os.environ.get("LOKI_DIR") or os.path.join(cwd, ".loki")
+
+    kv = {}
+    for a in argv[1:]:
+        if a.startswith("--") and "=" in a:
+            k, v = a[2:].split("=", 1)
+            kv[k] = v
+
+    if action == "record":
+        res = record_failure(loki_dir, kv.get("gate"), kv.get("verdict"),
+                             kv.get("evidence"), kv.get("run_id"))
+    elif action == "recall":
+        res = recall(loki_dir, kv.get("gate"))
+    elif action == "context":
+        res = prompt_context(loki_dir)
+    else:
+        print(f"unknown action: {action}", file=sys.stderr)
+        return 2
+
+    print(json.dumps(res, indent=2))
+    return 0 if res.get("status") in ("recorded", "measured") else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))