loki-mode 9.12.6 → 9.17.0

package/autonomy/lib/agent_readiness.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Agent readiness: can an autonomous agent verify its own work in THIS repo?
+
+WHY THIS EXISTS, AND WHY IT IS NOT A COPY. Factory AI's Agent Readiness Model is
+a genuinely good idea and a category-defining artifact -- 5 levels, 9 pillars,
+2 scopes -- and it makes competitor comparisons happen on Factory's chosen axes.
+It is also LLM-SCORED: their report objects record `modelUsed` and
+`reasoningEffort` per report. So the number is a model's opinion of a repo, and
+two runs can disagree about the same commit.
+
+Ours is a measurement. Every criterion below is a file that exists or does not,
+a command that is present or absent. Same commit, same answer, every time, on
+any machine, with no key and no spend. "Theirs is an opinion, ours is a
+measurement, here is the command" is the same wedge as the receipt, applied to
+their own differentiated concept.
+
+WHAT IT MEASURES, AND WHY THOSE. Not general code quality -- that is what
+`loki modernize heal --assess` already scores with its own deterministic 4-level
+maturity rubric, and duplicating it would create two numbers that eventually
+disagree. This asks the narrower question our product actually depends on:
+CAN AN AGENT CHECK ITSELF HERE? Factory concedes the same dependency from the
+other side -- their Missions docs say that without "an automated, scriptable way
+to exercise the app... the mission cannot reliably verify its own work", and
+recommend Level 4+ before using their flagship. A repo with no test command is
+one where every agent, ours included, is guessing.
+
+WHAT IT REFUSES. No percentage, no letter grade, no composite. A composite
+invites ranking, ranking invites gaming, and the individual signals are the
+actionable part: "there is no test command" tells you what to do, "readiness 62%"
+does not. Criteria that cannot be determined report UNKNOWN by name rather than
+counting as failures -- an absent measurement is not a bad score.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+# Each criterion is a pure filesystem fact plus the command a reader can run to
+# check it themselves. The `why` is not decoration: a signal whose consequence
+# for an agent is unstated becomes a checkbox someone games.
+CRITERIA = [
+    {
+        "id": "test_command",
+        "why": "without a runnable test command an agent cannot verify its own change",
+        "verify": "look for a test script in package.json, a Makefile test target, pytest.ini, or tests/",
+    },
+    {
+        "id": "dependency_lock",
+        "why": "unpinned dependencies make a green run unreproducible tomorrow",
+        "verify": "look for package-lock.json, bun.lockb, poetry.lock, requirements.txt, Cargo.lock, go.sum",
+    },
+    {
+        "id": "ci_config",
+        "why": "without CI, nothing re-checks the agent's work independently of the agent",
+        "verify": "look for .github/workflows, .gitlab-ci.yml, or a CI config at the repo root",
+    },
+    {
+        "id": "agent_brief",
+        "why": "without a briefing file an agent rediscovers conventions every run and gets them wrong",
+        "verify": "look for AGENTS.md, CLAUDE.md, CONTRIBUTING.md",
+    },
+    {
+        "id": "readme",
+        "why": "without a README an agent has no statement of what the project is for",
+        "verify": "look for README.md or README",
+    },
+    {
+        "id": "gitignore",
+        "why": "without ignores an agent's diff fills with build output and the real change is buried",
+        "verify": "look for .gitignore",
+    },
+]
+
+
+def _any_exists(root, names):
+    for n in names:
+        if os.path.exists(os.path.join(root, n)):
+            return n
+    return None
+
+
+def _has_test_command(root):
+    pkg = os.path.join(root, "package.json")
+    if os.path.isfile(pkg):
+        try:
+            with open(pkg, "r", encoding="utf-8") as fh:
+                data = json.load(fh)
+            if (data.get("scripts") or {}).get("test"):
+                return "package.json scripts.test"
+        except (OSError, ValueError):
+            # A malformed package.json is not evidence either way. Fall through
+            # to the other signals rather than scoring it as absent.
+            pass
+    found = _any_exists(root, ["pytest.ini", "tox.ini", "Makefile", "tests", "test"])
+    return found
+
+
+def assess(root):
+    """Evaluate every criterion. Returns facts, never a score."""
+    if not os.path.isdir(root):
+        return {"status": UNKNOWN, "reason": "no_such_directory", "path": root}
+    if not os.path.isdir(os.path.join(root, ".git")):
+        # Not fatal: readiness is about the working tree. Recorded so a reader
+        # knows the repo context was absent rather than assumed.
+        git_present = False
+    else:
+        git_present = True
+
+    checks = []
+    for c in CRITERIA:
+        cid = c["id"]
+        if cid == "test_command":
+            hit = _has_test_command(root)
+        elif cid == "dependency_lock":
+            hit = _any_exists(root, ["package-lock.json", "bun.lockb", "yarn.lock",
+                                     "poetry.lock", "requirements.txt", "Cargo.lock",
+                                     "go.sum", "Pipfile.lock"])
+        elif cid == "ci_config":
+            hit = _any_exists(root, [".github/workflows", ".gitlab-ci.yml",
+                                     ".circleci", "azure-pipelines.yml", "Jenkinsfile"])
+        elif cid == "agent_brief":
+            hit = _any_exists(root, ["AGENTS.md", "CLAUDE.md", "CONTRIBUTING.md"])
+        elif cid == "readme":
+            hit = _any_exists(root, ["README.md", "README", "README.rst"])
+        elif cid == "gitignore":
+            hit = _any_exists(root, [".gitignore"])
+        else:
+            hit = None
+
+        checks.append({
+            "id": cid,
+            "present": bool(hit),
+            "found": hit or None,
+            "why": c["why"],
+            "verify": c["verify"],
+        })
+
+    present = [c for c in checks if c["present"]]
+    missing = [c for c in checks if not c["present"]]
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "status": "measured",
+        "path": os.path.abspath(root),
+        "git_repo": git_present,
+        # Counts, not a percentage. A composite invites ranking, ranking invites
+        # gaming, and "there is no test command" is the actionable part anyway.
+        "criteria_total": len(checks),
+        "criteria_present": len(present),
+        "checks": checks,
+        "missing": [c["id"] for c in missing],
+        # The single most consequential signal, surfaced on its own: this is the
+        # one Factory's own docs concede their flagship depends on.
+        "can_self_verify": any(c["id"] == "test_command" and c["present"]
+                               for c in checks),
+    }
+
+
+def render_text(res):
+    if res.get("status") != "measured":
+        return f"Agent readiness: UNKNOWN ({res.get('reason', 'unmeasurable')})"
+    out = ["Agent readiness -- can an agent verify its own work here?", ""]
+    for c in res["checks"]:
+        mark = "yes" if c["present"] else "NO "
+        line = f"  {mark}  {c['id']:20}"
+        if c["present"]:
+            line += f"({c['found']})"
+        else:
+            line += c["why"]
+        out.append(line)
+    out.append("")
+    out.append(f"  {res['criteria_present']} of {res['criteria_total']} present")
+    if not res["can_self_verify"]:
+        out.append("")
+        out.append("  No test command found. Every agent working here, ours")
+        out.append("  included, is guessing whether its change worked.")
+    out.append("")
+    out.append("  Every line above is a file that exists or does not. Check any of")
+    out.append("  them by hand; no model was asked for an opinion.")
+    return "\n".join(out)
+
+
+# What --fix writes for each missing criterion. Only files whose CORRECT content
+# can be derived from the repo itself appear here.
+#
+# test_command and dependency_lock are deliberately absent: guessing a test
+# command writes a line that lies (`npm test` in a repo with no runner exits
+# non-zero forever, and the readiness check would then report "present" for
+# something that does not work), and a lockfile must come from the real package
+# manager or it is worse than none. Those stay REPORTED, never generated.
+#
+# This is the difference between a scorecard and an executable one -- Factory's
+# /readiness-report -> /readiness-fix -- without the failure mode where the fix
+# makes the score green while the underlying capability is still missing.
+FIXABLE = {
+    "gitignore": (".gitignore", "node_modules/\n__pycache__/\n.env\n.venv/\ndist/\n*.log\n"),
+    "readme": ("README.md", None),        # content derived below
+    "agent_brief": ("AGENTS.md", None),   # content derived below
+}
+
+
+def _fix(root, res):
+    """Create the missing files whose content can be derived honestly.
+
+    Returns (written, skipped) where skipped names criteria that a generated
+    file could not honestly satisfy.
+    """
+    written, skipped = [], []
+    project = os.path.basename(os.path.abspath(root)) or "this project"
+    # assess() reports `missing` as a list of criterion IDs (strings), not the
+    # full check dicts. Tolerate both so a later shape change does not silently
+    # fix nothing.
+    for entry in res.get("missing", []):
+        cid = entry["id"] if isinstance(entry, dict) else entry
+        if cid not in FIXABLE:
+            skipped.append((cid, "must come from the real toolchain, not a guess"))
+            continue
+        name, body = FIXABLE[cid]
+        path = os.path.join(root, name)
+        if os.path.exists(path):
+            continue
+        if cid == "readme":
+            body = (f"# {project}\n\n"
+                    "## What this is\n\n_TODO: one paragraph._\n\n"
+                    "## Run it\n\n```sh\n# TODO: the command that starts this project\n```\n\n"
+                    "## Test it\n\n```sh\n# TODO: the command that runs the tests\n```\n")
+        elif cid == "agent_brief":
+            body = (f"# AGENTS.md\n\nBriefing for coding agents working in {project}.\n\n"
+                    "## Commands\n\n- Build: _TODO_\n- Test: _TODO_\n- Lint: _TODO_\n\n"
+                    "## Conventions\n\n_TODO: what a reviewer would flag._\n\n"
+                    "## Do not\n\n_TODO: the things that break this repo._\n")
+        try:
+            with open(path, "w") as fh:
+                fh.write(body)
+            written.append(name)
+        except OSError as exc:
+            skipped.append((cid, f"could not write {name}: {exc}"))
+    return written, skipped
+
+
+def main(argv):
+    as_json = "--json" in argv
+    do_fix = "--fix" in argv
+    root = "."
+    for a in argv:
+        if not a.startswith("-"):
+            root = a
+            break
+    res = assess(root)
+
+    if do_fix and res.get("status") == "measured":
+        written, skipped = _fix(root, res)
+        res = assess(root)  # re-measure: report what is true AFTER the fix
+        res["fix"] = {"written": written,
+                      "skipped": [{"id": i, "reason": r} for i, r in skipped]}
+
+    if as_json:
+        print(json.dumps(res, indent=2))
+    else:
+        print(render_text(res))
+        if do_fix:
+            fx = res.get("fix", {})
+            print("")
+            for name in fx.get("written", []):
+                print(f"  wrote    {name} (a stub -- fill in the TODOs)")
+            for s in fx.get("skipped", []):
+                print(f"  skipped  {s['id']}: {s['reason']}")
+            if not fx.get("written") and not fx.get("skipped"):
+                print("  nothing to fix.")
+    return 0 if res.get("status") == "measured" else 3
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/claim_grounding.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Claim grounding: a completion claim must name work that exists in the diff.
+
+WHY THIS EXISTS. 8090 AI's evaluation framework blocks ungrounded output AT
+GENERATION rather than catching it at review, and states the payoff plainly:
+
+    "Ungrounded claims are blocked at generation. The rubric measures the quality
+     of what survives the architectural filter, which is a much smaller and more
+     interesting space."
+
+Our evidence gate already has six axes -- diff non-empty, tests green, runtime
+boot, no-mock, authorization, secret leak. Every one is a REPO-LEVEL fact. None
+reads what the agent actually CLAIMED. So an agent can finish by asserting "added
+retry logic to the payment client and covered it with tests" while the diff shows
+a README edit, and every axis passes: the diff is non-empty, the tests are green,
+the app boots. The claim itself is the one artifact nobody checks.
+
+WHAT THIS CHECKS, AND WHAT IT REFUSES TO CHECK. It resolves file-path-shaped
+tokens in the claim against the actual changed-file set. That is a deterministic
+string-to-set comparison, and it is the only part of a natural-language claim
+that can be checked without a model.
+
+It does NOT judge whether the claim is semantically true. "Added retry logic" vs
+"added a retry constant" is a judgement, and asking an LLM to grade it would be
+the same LLM-judge-as-measurement this codebase refuses everywhere else. A claim
+naming no paths is UNGROUNDABLE, reported by name -- never scored, never failed.
+
+THE FAIL-OPEN DIRECTION IS DELIBERATE. Only a claim that names a path which is
+demonstrably NOT in the diff is a finding. A claim with no paths, a claim naming
+a path that exists but was not touched by this run, and an empty claim are all
+reported and none of them blocks. A grounding check that blocked on ambiguity
+would fire constantly on ordinary prose and be disabled within a week, which is
+worse than not having it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+
+SCHEMA_VERSION = "1.0"
+
+UNKNOWN = "UNKNOWN"
+
+GROUNDING_REASONS = {
+    "no_claim": "no completion claim text was provided",
+    "no_diff": "no changed-file set was provided to check against",
+    "ungroundable": "the claim names no file paths, so it cannot be checked mechanically",
+}
+
+# A path-shaped token: at least one slash or a known source extension, and no
+# spaces. Deliberately conservative -- a false "this is a path" produces a false
+# finding, and a check that cries wolf gets turned off.
+_PATH_RE = re.compile(
+    r"(?<![\w/.-])"
+    r"(?:[\w.-]+/)+[\w.-]+\.[A-Za-z0-9]{1,6}"
+    r"|(?<![\w/.-])[\w.-]+\.(?:py|ts|tsx|js|jsx|sh|go|rs|rb|java|c|h|cpp|md|json|ya?ml|toml)"
+    r"(?![\w/.-])"
+)
+
+
+def extract_paths(claim):
+    """Path-shaped tokens in a claim, deduped, order preserved."""
+    if not claim:
+        return []
+    seen, out = set(), []
+    for m in _PATH_RE.finditer(claim):
+        tok = m.group(0).strip(".,;:)(\"'`")
+        if tok and tok not in seen:
+            seen.add(tok)
+            out.append(tok)
+    return out
+
+
+def check(claim, changed_files):
+    """Are the paths a claim names present in the diff?
+
+    Matching is suffix-based on purpose: an agent writes `run.sh` or
+    `autonomy/run.sh` for the same file, and demanding an exact repo-relative
+    string would make every informal mention a false finding.
+    """
+    if not claim or not claim.strip():
+        return {"status": UNKNOWN, "reason": "no_claim",
+                "detail": GROUNDING_REASONS["no_claim"]}
+    if changed_files is None:
+        return {"status": UNKNOWN, "reason": "no_diff",
+                "detail": GROUNDING_REASONS["no_diff"]}
+
+    named = extract_paths(claim)
+    if not named:
+        # The common, benign case: "fixed the login bug". Nothing to check, and
+        # that is not a defect -- reported so the number of unverifiable claims
+        # is visible, rather than silently counted as grounded.
+        return {"status": UNKNOWN, "reason": "ungroundable",
+                "detail": GROUNDING_REASONS["ungroundable"],
+                "paths_named": []}
+
+    changed = list(changed_files)
+    grounded, ungrounded = [], []
+    for p in named:
+        norm = p.lstrip("./")
+        # Basename fallback is load-bearing, not laxity. An agent writes `run.sh`
+        # for `autonomy/run.sh`, and requiring the full repo-relative string made
+        # every informal mention a false finding -- measured: "patched run.sh"
+        # against a changed autonomy/run.sh was reported UNGROUNDED. A grounding
+        # check that flags correct claims is worse than none, because it gets
+        # disabled and takes the real detections with it.
+        base = os.path.basename(norm)
+        hit = any(
+            c == norm
+            or c.endswith("/" + norm)
+            or norm.endswith("/" + c)
+            or os.path.basename(c) == base
+            for c in changed
+        )
+        (grounded if hit else ungrounded).append(p)
+
+    return {
+        "status": "measured",
+        "paths_named": named,
+        "grounded": grounded,
+        "ungrounded": ungrounded,
+        # The single actionable signal. A claim that names a file the run never
+        # touched is the "agent says done, diff says otherwise" failure, caught
+        # from the claim side instead of the repo side.
+        "has_ungrounded_claim": bool(ungrounded),
+    }
+
+
+def main(argv):
+    claim = ""
+    files_arg = ""
+    files_from = ""
+    i = 0
+    while i < len(argv):
+        if argv[i] == "--claim" and i + 1 < len(argv):
+            claim = argv[i + 1]; i += 2; continue
+        if argv[i] == "--files" and i + 1 < len(argv):
+            files_arg = argv[i + 1]; i += 2; continue
+        if argv[i] == "--files-from" and i + 1 < len(argv):
+            files_from = argv[i + 1]; i += 2; continue
+        i += 1
+
+    # --files is ALWAYS a comma-separated list; --files-from is always a file to
+    # read. The first version overloaded one flag for both and picked with
+    # os.path.isfile(), which silently misread `--files autonomy/run.sh` -- a
+    # perfectly ordinary changed-file list -- as "open that file and treat its
+    # 20,000 lines as filenames". The result was a correct claim reported
+    # UNGROUNDED, i.e. the exact false positive this check must never produce.
+    # An ambiguous flag whose meaning depends on the filesystem is a bug, not a
+    # convenience.
+    if files_from and os.path.isfile(files_from):
+        with open(files_from, "r", encoding="utf-8") as fh:
+            changed = [l.strip() for l in fh if l.strip()]
+    elif files_arg:
+        changed = [c.strip() for c in files_arg.split(",") if c.strip()]
+    else:
+        changed = None
+
+    res = check(claim, changed)
+    res["schema_version"] = SCHEMA_VERSION
+    print(json.dumps(res, indent=2))
+    # Exit 1 ONLY on a demonstrably ungrounded path. Every UNKNOWN exits 0: a
+    # check that failed on ambiguity would fire on ordinary prose and be disabled.
+    return 1 if res.get("has_ungrounded_claim") else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/autonomy/lib/config-map.sh

@@ -863,13 +863,17 @@ loki_config_validate_file() {
     pairs="$(
         _loki_cfg_collect_pairs() {
             local f="$1" fm="$2"
+            # NOTE: every case pattern below carries a leading open-paren --
+            # bash 3.2 (macOS /bin/bash) ends a command substitution at the
+            # close-paren of a case label, truncating this function body. The
+            # balanced form parses identically on 3.2 and 4+. Do not strip it.
             case "$fm" in
-                env)
+                (env)
                     local line key val
                     while IFS= read -r line || [ -n "$line" ]; do
-                        case "$line" in ''|'#'*) continue ;; esac
+                        case "$line" in (''|'#'*) continue ;; esac
                         line="${line#export }"
-                        case "$line" in *=*) ;; *) continue ;; esac
+                        case "$line" in (*=*) ;; (*) continue ;; esac
                         key="${line%%=*}"; val="${line#*=}"
                         key="${key#"${key%%[![:space:]]*}"}"; key="${key%"${key##*[![:space:]]}"}"
                         val="${val#"${val%%[![:space:]]*}"}"
@@ -878,7 +882,7 @@ loki_config_validate_file() {
                         printf '%s\t%s\n' "$key" "$val"
                     done < "$f"
                     ;;
-                yaml)
+                (yaml)
                     local mapping yaml_path env_var value have_yq=0
                     if command -v yq >/dev/null 2>&1; then have_yq=1; fi
                     for mapping in "${LOKI_CONFIG_MAP[@]}"; do
@@ -894,8 +898,8 @@ loki_config_validate_file() {
                         printf '%s\t%s\n' "$env_var" "$value"
                     done
                     ;;
-                json)
-                    # Reuse the json parser's emit path by calling a print-only variant.
+                (json)
+                    # Reuse the JSON parser emit path via a print-only variant.
                     _loki_cfg_json_emit "$f"
                     ;;
             esac