loki-mode 9.12.4 → 9.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/VERSION +1 -1
- package/autonomy/loki +20 -8
- package/autonomy/verify.sh +10 -1
- package/dashboard/__init__.py +1 -1
- package/dashboard/api_operator.py +15 -2
- package/dashboard/api_v2.py +6 -1
- package/dashboard/control.py +62 -10
- package/dashboard/server.py +26 -5
- package/dashboard/static/index.html +1 -1
- package/docs/LOOP-HARNESS-AUDIT.md +631 -0
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/loop-harness-report.py +216 -0
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
9.12.
|
|
1
|
+
9.12.6
|
package/autonomy/loki
CHANGED
|
@@ -27881,11 +27881,17 @@ except: pass
|
|
|
27881
27881
|
|
|
27882
27882
|
# --- Build directory tree ---
|
|
27883
27883
|
local tree_output=""
|
|
27884
|
-
|
|
27884
|
+
# -e not -d: in a git WORKTREE, .git is a pointer FILE, not a directory.
|
|
27885
|
+
# -d sent every worktree down the find fallback below.
|
|
27886
|
+
if command -v git &>/dev/null && [ -e "$target_path/.git" ]; then
|
|
27885
27887
|
# Use git ls-files for accurate tree (respects .gitignore)
|
|
27886
27888
|
tree_output=$(cd "$target_path" && git ls-files 2>/dev/null | head -200 || true)
|
|
27887
27889
|
else
|
|
27888
|
-
# Fallback: find with common excludions
|
|
27890
|
+
# Fallback: find with common excludions.
|
|
27891
|
+
# `|| true` is load-bearing under `set -euo pipefail` (line 22): head -200
|
|
27892
|
+
# exits after 200 lines, find keeps writing, gets SIGPIPE, and the pipeline
|
|
27893
|
+
# returns 141 -- which -e turned into a silent abort producing 0 bytes on
|
|
27894
|
+
# any repo with more than 200 files. The git branch above is already guarded.
|
|
27889
27895
|
tree_output=$(find "$target_path" -maxdepth 4 -type f \
|
|
27890
27896
|
-not -path '*/node_modules/*' \
|
|
27891
27897
|
-not -path '*/.git/*' \
|
|
@@ -27895,7 +27901,7 @@ except: pass
|
|
|
27895
27901
|
-not -path '*/build/*' \
|
|
27896
27902
|
-not -path '*/.next/*' \
|
|
27897
27903
|
-not -path '*/target/*' \
|
|
27898
|
-
2>/dev/null | sed "s|$target_path/||" | sort | head -200)
|
|
27904
|
+
2>/dev/null | sed "s|$target_path/||" | sort | head -200 || true)
|
|
27899
27905
|
fi
|
|
27900
27906
|
|
|
27901
27907
|
# Categorize files
|
|
@@ -28652,8 +28658,13 @@ $devdeps_list"
|
|
|
28652
28658
|
fi
|
|
28653
28659
|
|
|
28654
28660
|
# --- File counts ---
|
|
28661
|
+
# Same SIGPIPE pipeline as cmd_onboard, but a DIFFERENT symptom here: this
|
|
28662
|
+
# assigns into a bare `tree_output=` after the local declaration, so under
|
|
28663
|
+
# -e a 141 aborts. Where a site instead writes `local x=$(...)`, `local`
|
|
28664
|
+
# resets $? and the 141 is swallowed -- silent truncation, no abort. Both
|
|
28665
|
+
# are wrong; `|| true` plus -e (worktree .git is a FILE) fixes both.
|
|
28655
28666
|
local tree_output=""
|
|
28656
|
-
if command -v git &>/dev/null && [ -
|
|
28667
|
+
if command -v git &>/dev/null && [ -e "$target_path/.git" ]; then
|
|
28657
28668
|
tree_output=$(cd "$target_path" && git ls-files 2>/dev/null | head -500 || true)
|
|
28658
28669
|
else
|
|
28659
28670
|
tree_output=$(find "$target_path" -maxdepth 4 -type f \
|
|
@@ -28661,7 +28672,7 @@ $devdeps_list"
|
|
|
28661
28672
|
-not -path '*/vendor/*' -not -path '*/__pycache__/*' \
|
|
28662
28673
|
-not -path '*/dist/*' -not -path '*/build/*' \
|
|
28663
28674
|
-not -path '*/.next/*' -not -path '*/target/*' \
|
|
28664
|
-
2>/dev/null | sed "s|$target_path/||" | sort | head -500)
|
|
28675
|
+
2>/dev/null | sed "s|$target_path/||" | sort | head -500 || true)
|
|
28665
28676
|
fi
|
|
28666
28677
|
local total_files src_count test_count doc_count config_count
|
|
28667
28678
|
total_files=$(echo "$tree_output" | { grep -c . || true; })
|
|
@@ -29065,9 +29076,10 @@ cmd_docs() {
|
|
|
29065
29076
|
_docs_scan_project() {
|
|
29066
29077
|
local target_path="$1"
|
|
29067
29078
|
|
|
29068
|
-
# Build file tree (excluding common noise)
|
|
29079
|
+
# Build file tree (excluding common noise).
|
|
29080
|
+
# Same SIGPIPE pipeline and same fix as cmd_onboard; see the comment there.
|
|
29069
29081
|
local tree_output=""
|
|
29070
|
-
if command -v git &>/dev/null && [ -
|
|
29082
|
+
if command -v git &>/dev/null && [ -e "$target_path/.git" ]; then
|
|
29071
29083
|
tree_output=$(cd "$target_path" && git ls-files 2>/dev/null | head -500 || true)
|
|
29072
29084
|
else
|
|
29073
29085
|
tree_output=$(find "$target_path" -maxdepth 4 -type f \
|
|
@@ -29075,7 +29087,7 @@ _docs_scan_project() {
|
|
|
29075
29087
|
-not -path '*/vendor/*' -not -path '*/__pycache__/*' \
|
|
29076
29088
|
-not -path '*/dist/*' -not -path '*/build/*' \
|
|
29077
29089
|
-not -path '*/.next/*' -not -path '*/target/*' \
|
|
29078
|
-
2>/dev/null | sed "s|$target_path/||" | sort | head -500)
|
|
29090
|
+
2>/dev/null | sed "s|$target_path/||" | sort | head -500 || true)
|
|
29079
29091
|
fi
|
|
29080
29092
|
echo "$tree_output"
|
|
29081
29093
|
}
|
package/autonomy/verify.sh
CHANGED
|
@@ -2522,9 +2522,18 @@ try:
|
|
|
2522
2522
|
# secrets/lint/deps). It must NEVER claim "verified" when the authoritative
|
|
2523
2523
|
# verdict is not VERIFIED -- a tool named Verify printing "-> verified" next
|
|
2524
2524
|
# to a BLOCKED result is a fake-green-shaped surface, and this product never
|
|
2525
|
-
# lies about done. So the banner reports the
|
|
2525
|
+
# lies about done. So the banner reports the partial finding of the engine,
|
|
2526
2526
|
# explicitly subordinated to the authoritative verdict, and never the word
|
|
2527
2527
|
# "verified" on its own when the verdict disagrees.
|
|
2528
|
+
#
|
|
2529
|
+
# NO APOSTROPHES IN THIS BODY. This heredoc sits inside "$( ... )", and bash
|
|
2530
|
+
# 3.2 -- the /bin/bash that every stock macOS ships -- tracks quoting THROUGH
|
|
2531
|
+
# the command substitution even though the heredoc is quoted with <<PYEOF.
|
|
2532
|
+
# One apostrophe opens a quote that never closes, so the parser swallows the
|
|
2533
|
+
# rest of the file and reports "syntax error near unexpected token (" at a
|
|
2534
|
+
# line ~270 later that is perfectly valid. The whole file then fails to
|
|
2535
|
+
# parse: `loki verify --help` exits 2 printing NOTHING on a stock Mac, while
|
|
2536
|
+
# working fine under Homebrew bash 5. Verified by minimal reproduction.
|
|
2528
2537
|
verdict = (os.environ.get("_V_VERDICT") or "").strip().upper()
|
|
2529
2538
|
if e.get("inconclusive"):
|
|
2530
2539
|
finding = "inconclusive (%s)" % (e.get("inconclusive_reason") or "no reason")
|
package/dashboard/__init__.py
CHANGED
|
@@ -77,11 +77,24 @@ def _loki_dir() -> str:
|
|
|
77
77
|
others, and an import-time snapshot would pin whichever was true when the
|
|
78
78
|
module first loaded.
|
|
79
79
|
"""
|
|
80
|
-
return os.environ.get("LOKI_DIR") or os.path.join(
|
|
80
|
+
return os.environ.get("LOKI_DIR") or os.path.join(_safe_getcwd(), ".loki")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _safe_getcwd() -> str:
|
|
84
|
+
"""os.getcwd() raises FileNotFoundError once the cwd is deleted.
|
|
85
|
+
|
|
86
|
+
These helpers run per request, so letting that propagate turns a stale
|
|
87
|
+
working directory into a blanket 500 across the whole API. Fall back to
|
|
88
|
+
this package's tree, which is on disk by definition.
|
|
89
|
+
"""
|
|
90
|
+
try:
|
|
91
|
+
return os.getcwd()
|
|
92
|
+
except OSError:
|
|
93
|
+
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
81
94
|
|
|
82
95
|
|
|
83
96
|
def _repo_dir() -> str:
|
|
84
|
-
return os.environ.get("LOKI_REPO_DIR") or
|
|
97
|
+
return os.environ.get("LOKI_REPO_DIR") or _safe_getcwd()
|
|
85
98
|
|
|
86
99
|
|
|
87
100
|
# Ceilings for the receipt walk on the HTTP path. Generous enough that a real
|
package/dashboard/api_v2.py
CHANGED
|
@@ -527,7 +527,12 @@ async def list_runs(
|
|
|
527
527
|
try:
|
|
528
528
|
from . import api_runs as _fs_runs
|
|
529
529
|
|
|
530
|
-
|
|
530
|
+
# A deleted cwd raises from os.getcwd(); this runs per request.
|
|
531
|
+
try:
|
|
532
|
+
_cwd = os.getcwd()
|
|
533
|
+
except OSError:
|
|
534
|
+
_cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
535
|
+
_loki = os.environ.get("LOKI_DIR") or os.path.join(_cwd, ".loki")
|
|
531
536
|
_fs = _fs_runs.list_runs(_loki)
|
|
532
537
|
# The adapter returns an envelope carrying source, freshness
|
|
533
538
|
# and an explicit reason when empty. Never fabricate a row:
|
package/dashboard/control.py
CHANGED
|
@@ -72,22 +72,64 @@ def find_skill_dir() -> Path:
|
|
|
72
72
|
return configured
|
|
73
73
|
raise RuntimeError(f"LOKI_SKILL_DIR is not a Loki source tree: {configured}")
|
|
74
74
|
|
|
75
|
+
# os.getcwd() RAISES FileNotFoundError when the working directory has been
|
|
76
|
+
# deleted out from under the process. That is not hypothetical: a dashboard
|
|
77
|
+
# started inside a temp workspace keeps running after the workspace is
|
|
78
|
+
# cleaned up, and then EVERY endpoint that resolves a path 500s at once --
|
|
79
|
+
# observed as ~60 simultaneous 500s including /api/status, whose handler
|
|
80
|
+
# touches almost nothing. A long-lived server must survive losing its cwd.
|
|
81
|
+
def _cwd_or_none() -> "Path | None":
|
|
82
|
+
try:
|
|
83
|
+
return Path.cwd()
|
|
84
|
+
except OSError:
|
|
85
|
+
return None
|
|
86
|
+
|
|
75
87
|
candidates = [
|
|
76
88
|
Path.home() / ".claude" / "skills" / "loki-mode",
|
|
77
89
|
Path(__file__).parent.parent,
|
|
78
|
-
|
|
90
|
+
_cwd_or_none(),
|
|
79
91
|
]
|
|
80
92
|
for candidate in candidates:
|
|
81
|
-
if
|
|
82
|
-
|
|
83
|
-
|
|
93
|
+
if candidate is None:
|
|
94
|
+
continue
|
|
95
|
+
try:
|
|
96
|
+
if ((candidate / "SKILL.md").exists()
|
|
97
|
+
and (candidate / "autonomy" / "run.sh").exists()):
|
|
98
|
+
return candidate
|
|
99
|
+
except OSError:
|
|
100
|
+
continue
|
|
101
|
+
|
|
102
|
+
# The fallback must not be the one path that can still raise. When the cwd
|
|
103
|
+
# is gone, resolve to this file's own tree -- it is on disk by definition,
|
|
104
|
+
# since we are executing out of it.
|
|
105
|
+
return _cwd_or_none() or Path(__file__).resolve().parent.parent
|
|
84
106
|
|
|
85
107
|
SKILL_DIR = find_skill_dir()
|
|
86
108
|
RUN_SH = SKILL_DIR / "autonomy" / "run.sh"
|
|
87
109
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
110
|
+
|
|
111
|
+
def _cwd_or_skill_dir() -> Path:
|
|
112
|
+
"""The cwd, or the skill tree when the cwd has been deleted.
|
|
113
|
+
|
|
114
|
+
Used where a real directory is required (subprocess cwd=), as opposed to
|
|
115
|
+
the confinement check in validate(), which must fail closed instead.
|
|
116
|
+
"""
|
|
117
|
+
try:
|
|
118
|
+
return Path.cwd()
|
|
119
|
+
except OSError:
|
|
120
|
+
return SKILL_DIR
|
|
121
|
+
|
|
122
|
+
# Ensure directories exist.
|
|
123
|
+
# Best-effort at IMPORT time. LOKI_DIR defaults to the relative path ".loki",
|
|
124
|
+
# so when the working directory has been deleted these mkdirs raise
|
|
125
|
+
# FileNotFoundError and the whole module fails to import -- which takes the
|
|
126
|
+
# entire dashboard down rather than the one feature that needs the directory.
|
|
127
|
+
# Whoever actually writes into these paths still surfaces its own error.
|
|
128
|
+
for _startup_dir in (STATE_DIR, LOG_DIR):
|
|
129
|
+
try:
|
|
130
|
+
_startup_dir.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
except OSError:
|
|
132
|
+
pass
|
|
91
133
|
|
|
92
134
|
# Utility: atomic write with optional file locking
|
|
93
135
|
def atomic_write_json(file_path: Path, data: dict, use_lock: bool = True):
|
|
@@ -202,8 +244,16 @@ class StartRequest(BaseModel):
|
|
|
202
244
|
if not prd_path.is_file():
|
|
203
245
|
raise ValueError(f"PRD path is not a file: {self.prd}")
|
|
204
246
|
|
|
205
|
-
# Verify path resolves within CWD or a reasonable parent
|
|
206
|
-
cwd
|
|
247
|
+
# Verify path resolves within CWD or a reasonable parent.
|
|
248
|
+
# FAIL CLOSED if the cwd is gone: this is a path-confinement check, so
|
|
249
|
+
# an unresolvable base must reject the path, never skip the comparison.
|
|
250
|
+
try:
|
|
251
|
+
cwd = Path.cwd().resolve()
|
|
252
|
+
except OSError as exc:
|
|
253
|
+
raise ValueError(
|
|
254
|
+
"cannot validate PRD path: the working directory no longer "
|
|
255
|
+
"exists (%s)" % exc
|
|
256
|
+
) from exc
|
|
207
257
|
try:
|
|
208
258
|
prd_path.relative_to(cwd)
|
|
209
259
|
except ValueError:
|
|
@@ -450,7 +500,9 @@ async def start_session(request: StartRequest):
|
|
|
450
500
|
stdout=subprocess.DEVNULL,
|
|
451
501
|
stderr=subprocess.DEVNULL,
|
|
452
502
|
start_new_session=True,
|
|
453
|
-
cwd
|
|
503
|
+
# A deleted cwd would make Popen itself raise; fall back to the
|
|
504
|
+
# skill tree rather than failing to launch the run at all.
|
|
505
|
+
cwd=str(_cwd_or_skill_dir())
|
|
454
506
|
)
|
|
455
507
|
|
|
456
508
|
# Save provider for status tracking
|
package/dashboard/server.py
CHANGED
|
@@ -4417,7 +4417,8 @@ async def start_build(request: Request, body: StartBuildRequest):
|
|
|
4417
4417
|
loki_dir = workspace_dir / ".loki"
|
|
4418
4418
|
else:
|
|
4419
4419
|
loki_dir = _get_loki_dir()
|
|
4420
|
-
project_dir = loki_dir.parent if loki_dir.name == ".loki"
|
|
4420
|
+
project_dir = (loki_dir.parent if loki_dir.name == ".loki"
|
|
4421
|
+
else (_safe_cwd() or _Path(".")))
|
|
4421
4422
|
project_dir = project_dir.resolve()
|
|
4422
4423
|
|
|
4423
4424
|
# Legacy no-workspace starts keep their existing single-flight behavior.
|
|
@@ -5637,6 +5638,22 @@ _active_project_dir: Optional[str] = None
|
|
|
5637
5638
|
_DASHBOARD_AUTOSTARTED: bool = os.environ.get("LOKI_DASHBOARD_AUTOSTARTED") == "1"
|
|
5638
5639
|
|
|
5639
5640
|
|
|
5641
|
+
def _safe_cwd() -> "_Path | None":
|
|
5642
|
+
"""The current directory, or None if it no longer exists.
|
|
5643
|
+
|
|
5644
|
+
os.getcwd() RAISES FileNotFoundError when the working directory has been
|
|
5645
|
+
deleted out from under a live process. A dashboard started inside a temp
|
|
5646
|
+
workspace keeps serving after that workspace is cleaned up, and every
|
|
5647
|
+
endpoint resolving a path then 500s simultaneously -- observed as ~60
|
|
5648
|
+
concurrent 500s including /api/status, whose handler touches almost
|
|
5649
|
+
nothing. Losing the cwd must degrade to a fallback, never to a stack trace.
|
|
5650
|
+
"""
|
|
5651
|
+
try:
|
|
5652
|
+
return _Path.cwd()
|
|
5653
|
+
except OSError:
|
|
5654
|
+
return None
|
|
5655
|
+
|
|
5656
|
+
|
|
5640
5657
|
def _get_loki_dir() -> _Path:
|
|
5641
5658
|
"""Get LOKI_DIR, refreshing from env on each call for consistency.
|
|
5642
5659
|
|
|
@@ -5661,10 +5678,14 @@ def _get_loki_dir() -> _Path:
|
|
|
5661
5678
|
if env_dir and _Path(env_dir).is_absolute():
|
|
5662
5679
|
return _Path(env_dir)
|
|
5663
5680
|
|
|
5664
|
-
# Check CWD first
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
5681
|
+
# Check CWD first. _safe_cwd() rather than _Path.cwd(): this function runs
|
|
5682
|
+
# on nearly every request, and a deleted working directory would otherwise
|
|
5683
|
+
# raise FileNotFoundError here and 500 the entire API at once.
|
|
5684
|
+
_cwd = _safe_cwd()
|
|
5685
|
+
if _cwd is not None:
|
|
5686
|
+
cwd_loki = _cwd / ".loki"
|
|
5687
|
+
if cwd_loki.is_dir():
|
|
5688
|
+
return cwd_loki
|
|
5668
5689
|
|
|
5669
5690
|
# Check home directory fallback
|
|
5670
5691
|
home_loki = _Path.home() / ".loki"
|
|
@@ -11989,7 +11989,7 @@ var LokiDashboard=(()=>{var Be=Object.defineProperty;var zt=Object.getOwnPropert
|
|
|
11989
11989
|
${o}
|
|
11990
11990
|
${this._error?`<div class="error-banner">${this._escapeHtml(this._error)}</div>`:""}
|
|
11991
11991
|
</div>
|
|
11992
|
-
`,this._bindEvents()}};customElements.get("loki-rarv-timeline")||customElements.define("loki-rarv-timeline",be);function xt(d){if(!d||typeof d!="object")return null;let e=d.freshness_s;return typeof e!="number"||!Number.isFinite(e)||e<0?null:e}function _t(d){if(!Array.isArray(d))return null;let e=null;for(let t of d){let i=xt(t);i!==null&&(e===null||i<e)&&(e=i)}return e}function yt(d={}){let{payload:e=null,receivedAtMs:t=null,nowMs:i=Date.now(),staleAfterS:a=120}=d,s=xt(e),r=null,o="none";return s!==null?(r=s,o="server"):typeof t=="number"&&Number.isFinite(t)&&(r=Math.max(0,Math.floor((i-t)/1e3)),o="client"),{known:r!==null,ageS:r,source:o,isStale:r===null?null:r>=a,staleAfterS:a}}function ti(d){if(!d||!d.known||typeof d.ageS!="number")return"unknown";let e=d.ageS;return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`}function Ze(d){if(!d||!d.known)return"Data age unknown";let e=d.source==="client"?" (receive time)":"",t=d.isStale?" - STALE":"";return`Updated ${ti(d)}${e}${t}`}var wt={running:{color:"var(--loki-green, #22c55e)",bg:"var(--loki-green-muted, rgba(34, 197, 94, 0.15))",label:"Running"},completed:{color:"var(--loki-blue, #3b82f6)",bg:"var(--loki-blue-muted, rgba(59, 130, 246, 0.15))",label:"Completed"},failed:{color:"var(--loki-red, #ef4444)",bg:"var(--loki-red-muted, rgba(239, 68, 68, 0.15))",label:"Failed"},cancelled:{color:"var(--loki-yellow, #eab308)",bg:"var(--loki-yellow-muted, rgba(234, 179, 8, 0.15))",label:"Cancelled"},pending:{color:"var(--loki-text-muted, #939084)",bg:"var(--loki-bg-tertiary, #ECEAE3)",label:"Pending"},queued:{color:"var(--loki-text-muted, #939084)",bg:"var(--loki-bg-tertiary, #ECEAE3)",label:"Queued"}};function ii(d,e,t){let i=d;if(i==null&&e){let l=new Date(e).getTime();i=(t?new Date(t).getTime():Date.now())-l}if(i==null||i<0)return"--";if(i<1e3)return`${i}ms`;let a=Math.floor(i/1e3);if(a<60)return`${a}s`;let s=Math.floor(a/60),r=a%60;if(s<60)return`${s}m ${r}s`;let o=Math.floor(s/60),n=s%60;return`${o}h ${n}m`}function ai(d){if(!d)return"--";try{return new Date(d).toLocaleString([],{month:"short",day:"numeric",hour:"2-digit",minute:"2-digit"})}catch{return String(d)}}var ke=class extends v{static get observedAttributes(){return["api-url","project-id","theme"]}constructor(){super(),this._loading=!1,this._error=null,this._api=null,this._runs=[],this._pollInterval=null,this._lastDataHash=null,this._emptyReason=null,this._source=null,this._freshPayload=null,this._changedAtMs=null}_freshness(){let e=t=>{if(!t)return!1;if(t.current===!0)return!0;let i=String(t.status||"").toLowerCase();return i==="running"||i==="in_progress"||i==="active"};return yt({payload:this._freshPayload,receivedAtMs:this._changedAtMs,staleAfterS:this._runs.some(e)?void 0:1/0})}_renderFreshness(){let e=this.shadowRoot&&this.shadowRoot.getElementById("freshness");if(!e)return;let t=this._freshness();e.className=`freshness ${t.known?t.isStale?"is-stale":"":"is-unknown"}`,e.dataset.source=t.source,e.dataset.stale=t.isStale===null?"unknown":String(t.isStale),e.textContent=Ze(t)}get projectId(){let e=this.getAttribute("project-id");return e?parseInt(e,10):null}set projectId(e){e!=null?this.setAttribute("project-id",String(e)):this.removeAttribute("project-id")}connectedCallback(){super.connectedCallback(),this._setupApi(),this._loadData(),this._startPolling()}disconnectedCallback(){super.disconnectedCallback(),this._stopPolling()}attributeChangedCallback(e,t,i){t!==i&&(e==="api-url"&&this._api&&(this._api=h({baseUrl:i}),this._loadData()),e==="project-id"&&this._loadData(),e==="theme"&&this._applyTheme())}_setupApi(){let e=this.getAttribute("api-url")||window.location.origin;this._api=h({baseUrl:e})}_startPolling(){this._poll=b({loadFn:()=>this._loadData(),intervalMs:5e3,element:this,immediate:!1})}_stopPolling(){this._poll&&(this._poll.stop(),this._poll=null)}async _loadData(){let e=this._api;try{let t=this.projectId,i=t!=null?`?project_id=${t}`:"",a=await e._get(`/api/v2/runs${i}`);if(e!==this._api)return;let s=a?.runs||a||[],r=Array.isArray(s)?s:[];this._freshPayload=a&&typeof a=="object"&&!Array.isArray(a)&&"freshness_s"in a?a:{freshness_s:_t(r)};let o=JSON.stringify(s),n=o!==this._lastDataHash||!!this._error;n&&(this._lastDataHash=o,this._runs=r,this._changedAtMs=Date.now()),this._emptyReason=a&&!Array.isArray(a)&&a.reason||null,this._source=a&&!Array.isArray(a)&&a.source||null,this._error=null,this._loading=!1,n?this.render():this._renderFreshness();return}catch(t){if(e!==this._api)return;this._error||(this._error=`Failed to load runs: ${t.message}`)}finally{this._loading=!1}this.render()}async _cancelRun(e){try{await this._api._post(`/api/v2/runs/${e}/cancel`),await this._loadData()}catch(t){this._error=`Cancel failed: ${t.message}`,this.render()}}async _replayRun(e){try{await this._api._post(`/api/v2/runs/${e}/replay`),await this._loadData()}catch(t){this._error=`Replay failed: ${t.message}`,this.render()}}_escapeHtml(e){return e?String(e).replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,"""):""}_getStyles(){return`
|
|
11992
|
+
`,this._bindEvents()}};customElements.get("loki-rarv-timeline")||customElements.define("loki-rarv-timeline",be);function xt(d){if(!d||typeof d!="object")return null;let e=d.freshness_s;return typeof e!="number"||!Number.isFinite(e)||e<0?null:e}function _t(d){if(!Array.isArray(d))return null;let e=null;for(let t of d){let i=xt(t);i!==null&&(e===null||i<e)&&(e=i)}return e}function yt(d={}){let{payload:e=null,receivedAtMs:t=null,nowMs:i=Date.now(),staleAfterS:a=120}=d,s=xt(e),r=null,o="none";return s!==null?(r=s,o="server"):typeof t=="number"&&Number.isFinite(t)&&(r=Math.max(0,Math.floor((i-t)/1e3)),o="client"),{known:r!==null,ageS:r,source:o,isStale:r===null?null:r>=a,staleAfterS:a}}function ti(d){if(!d||!d.known||typeof d.ageS!="number")return"unknown";let e=d.ageS;return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`}function Ze(d){if(!d||!d.known)return"Data age unknown";let e=d.source==="client"?" (receive time)":"",t=d.isStale?" - STALE":"";return`Updated ${ti(d)}${e}${t}`}var wt={running:{color:"var(--loki-green, #22c55e)",bg:"var(--loki-green-muted, rgba(34, 197, 94, 0.15))",label:"Running"},completed:{color:"var(--loki-blue, #3b82f6)",bg:"var(--loki-blue-muted, rgba(59, 130, 246, 0.15))",label:"Completed"},failed:{color:"var(--loki-red, #ef4444)",bg:"var(--loki-red-muted, rgba(239, 68, 68, 0.15))",label:"Failed"},cancelled:{color:"var(--loki-yellow, #eab308)",bg:"var(--loki-yellow-muted, rgba(234, 179, 8, 0.15))",label:"Cancelled"},pending:{color:"var(--loki-text-muted, #939084)",bg:"var(--loki-bg-tertiary, #ECEAE3)",label:"Pending"},queued:{color:"var(--loki-text-muted, #939084)",bg:"var(--loki-bg-tertiary, #ECEAE3)",label:"Queued"},building:{color:"var(--loki-green, #22c55e)",bg:"var(--loki-green-muted, rgba(34, 197, 94, 0.15))",label:"Building"},bootstrap:{color:"var(--loki-green, #22c55e)",bg:"var(--loki-green-muted, rgba(34, 197, 94, 0.15))",label:"Bootstrap"},complete:{color:"var(--loki-blue, #3b82f6)",bg:"var(--loki-blue-muted, rgba(59, 130, 246, 0.15))",label:"Completed"},stopped:{color:"var(--loki-yellow, #eab308)",bg:"var(--loki-yellow-muted, rgba(234, 179, 8, 0.15))",label:"Stopped"},paused:{color:"var(--loki-yellow, #eab308)",bg:"var(--loki-yellow-muted, rgba(234, 179, 8, 0.15))",label:"Paused"},unknown:{color:"var(--loki-text-muted, #939084)",bg:"var(--loki-bg-tertiary, #ECEAE3)",label:"Unknown"}};function ii(d,e,t){let i=d;if(i==null&&e){let l=new Date(e).getTime();i=(t?new Date(t).getTime():Date.now())-l}if(i==null||i<0)return"--";if(i<1e3)return`${i}ms`;let a=Math.floor(i/1e3);if(a<60)return`${a}s`;let s=Math.floor(a/60),r=a%60;if(s<60)return`${s}m ${r}s`;let o=Math.floor(s/60),n=s%60;return`${o}h ${n}m`}function ai(d){if(!d)return"--";try{return new Date(d).toLocaleString([],{month:"short",day:"numeric",hour:"2-digit",minute:"2-digit"})}catch{return String(d)}}var ke=class extends v{static get observedAttributes(){return["api-url","project-id","theme"]}constructor(){super(),this._loading=!1,this._error=null,this._api=null,this._runs=[],this._pollInterval=null,this._lastDataHash=null,this._emptyReason=null,this._source=null,this._freshPayload=null,this._changedAtMs=null}_freshness(){let e=t=>{if(!t)return!1;if(t.current===!0)return!0;let i=String(t.status||"").toLowerCase();return i==="running"||i==="in_progress"||i==="active"};return yt({payload:this._freshPayload,receivedAtMs:this._changedAtMs,staleAfterS:this._runs.some(e)?void 0:1/0})}_renderFreshness(){let e=this.shadowRoot&&this.shadowRoot.getElementById("freshness");if(!e)return;let t=this._freshness();e.className=`freshness ${t.known?t.isStale?"is-stale":"":"is-unknown"}`,e.dataset.source=t.source,e.dataset.stale=t.isStale===null?"unknown":String(t.isStale),e.textContent=Ze(t)}get projectId(){let e=this.getAttribute("project-id");return e?parseInt(e,10):null}set projectId(e){e!=null?this.setAttribute("project-id",String(e)):this.removeAttribute("project-id")}connectedCallback(){super.connectedCallback(),this._setupApi(),this._loadData(),this._startPolling()}disconnectedCallback(){super.disconnectedCallback(),this._stopPolling()}attributeChangedCallback(e,t,i){t!==i&&(e==="api-url"&&this._api&&(this._api=h({baseUrl:i}),this._loadData()),e==="project-id"&&this._loadData(),e==="theme"&&this._applyTheme())}_setupApi(){let e=this.getAttribute("api-url")||window.location.origin;this._api=h({baseUrl:e})}_startPolling(){this._poll=b({loadFn:()=>this._loadData(),intervalMs:5e3,element:this,immediate:!1})}_stopPolling(){this._poll&&(this._poll.stop(),this._poll=null)}async _loadData(){let e=this._api;try{let t=this.projectId,i=t!=null?`?project_id=${t}`:"",a=await e._get(`/api/v2/runs${i}`);if(e!==this._api)return;let s=a?.runs||a||[],r=Array.isArray(s)?s:[];this._freshPayload=a&&typeof a=="object"&&!Array.isArray(a)&&"freshness_s"in a?a:{freshness_s:_t(r)};let o=JSON.stringify(s),n=o!==this._lastDataHash||!!this._error;n&&(this._lastDataHash=o,this._runs=r,this._changedAtMs=Date.now()),this._emptyReason=a&&!Array.isArray(a)&&a.reason||null,this._source=a&&!Array.isArray(a)&&a.source||null,this._error=null,this._loading=!1,n?this.render():this._renderFreshness();return}catch(t){if(e!==this._api)return;this._error||(this._error=`Failed to load runs: ${t.message}`)}finally{this._loading=!1}this.render()}async _cancelRun(e){try{await this._api._post(`/api/v2/runs/${e}/cancel`),await this._loadData()}catch(t){this._error=`Cancel failed: ${t.message}`,this.render()}}async _replayRun(e){try{await this._api._post(`/api/v2/runs/${e}/replay`),await this._loadData()}catch(t){this._error=`Replay failed: ${t.message}`,this.render()}}_escapeHtml(e){return e?String(e).replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,"""):""}_getStyles(){return`
|
|
11993
11993
|
:host {
|
|
11994
11994
|
display: block;
|
|
11995
11995
|
}
|
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
# Loop harness audit, and one proposed measurement slice
|
|
2
|
+
|
|
3
|
+
Audit only. No runtime behaviour is changed by this document, and the proposal
|
|
4
|
+
at the end is read-only by construction.
|
|
5
|
+
|
|
6
|
+
Every figure here was measured against the working tree at `4d8625bb`
|
|
7
|
+
(v9.12.5) on 2026-08-04. Commands are included so each can be re-run rather
|
|
8
|
+
than trusted.
|
|
9
|
+
|
|
10
|
+
## What already exists
|
|
11
|
+
|
|
12
|
+
### Structured traces
|
|
13
|
+
|
|
14
|
+
`.loki/events.jsonl` is the trace surface, written by `emit_event_json`
|
|
15
|
+
(`autonomy/run.sh:2435`) with a UTC timestamp and arbitrary `key=value` pairs.
|
|
16
|
+
**26 distinct event types** are emitted:
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
agent_prompt budget_exceeded budget_warning
|
|
20
|
+
capability_degraded code_review_complete code_review_council_complete
|
|
21
|
+
code_review_*_oversized code_review_start dashboard_crash
|
|
22
|
+
gate_stuck iteration_complete iteration_start
|
|
23
|
+
managed_agents_fallback managed_review_council_ok phase_change
|
|
24
|
+
policy_denied provider_failover provider_recovery
|
|
25
|
+
review_verification_failed session_end session_start
|
|
26
|
+
stage_complete task_completion_claim watchdog_alert
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
grep -ohE 'emit_event_json "[a-z_]+"' autonomy/run.sh | sort -u
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Verifier surface
|
|
34
|
+
|
|
35
|
+
Gate functions in `autonomy/run.sh`:
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
_evidence_gate_and_surface _invariant_gate_and_surface
|
|
39
|
+
_semantic_gate_and_surface _loki_supervised_completion_gates_pass
|
|
40
|
+
gate_failure_disposition build_gate_escalation_context
|
|
41
|
+
run_doc_quality_gate run_magic_debate_gate
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Plus the 3-reviewer blind council, whose completion is traced by
|
|
45
|
+
`code_review_complete` and `code_review_council_complete`.
|
|
46
|
+
|
|
47
|
+
## The gap, stated precisely
|
|
48
|
+
|
|
49
|
+
**The verifiers run, but they do not record what the directive asks for.**
|
|
50
|
+
|
|
51
|
+
`code_review_complete` carries exactly three fields:
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
review_id=<id> source=managed iteration=<n>
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`review_verification_failed` carries:
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
reason=<slug> iteration=<n> implementation_retry=<bool>
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Neither carries any of:
|
|
64
|
+
|
|
65
|
+
| Field the directive asks for | Emitted today |
|
|
66
|
+
|---|---|
|
|
67
|
+
| eligibility (did this verifier apply?) | no |
|
|
68
|
+
| deterministic criterion | no |
|
|
69
|
+
| retry cap / timeout cap | no |
|
|
70
|
+
| latency | no |
|
|
71
|
+
| tokens | no |
|
|
72
|
+
| cash cost | no |
|
|
73
|
+
| verdict | partial (failure reason only) |
|
|
74
|
+
| changed the terminal outcome? | no |
|
|
75
|
+
| false-positive review | no |
|
|
76
|
+
| rollback switch | no |
|
|
77
|
+
|
|
78
|
+
The `_evidence_gate_and_surface`, `_invariant_gate_and_surface` and
|
|
79
|
+
`_semantic_gate_and_surface` functions emit **nothing structured at all** --
|
|
80
|
+
grepping their bodies for `emit|json|cost|latency|duration|verdict` returns no
|
|
81
|
+
matches.
|
|
82
|
+
|
|
83
|
+
So a loop-harness manifest cannot be *derived* from today's traces. It would
|
|
84
|
+
have to be *fabricated*, which is the failure mode this codebase treats as
|
|
85
|
+
worse than an absent measurement.
|
|
86
|
+
|
|
87
|
+
### Why that matters more than it sounds
|
|
88
|
+
|
|
89
|
+
This session produced a concrete example of the cost. A gate false positive
|
|
90
|
+
(mock-integrity firing on `require.resolve` + `spawnSync`) made first-pass
|
|
91
|
+
completion impossible for every npm user, and it was invisible until someone
|
|
92
|
+
ran the real thing. With per-verifier records carrying `verdict`,
|
|
93
|
+
`changed_terminal_outcome` and `false_positive_reviewed`, that class shows up
|
|
94
|
+
as a measurement rather than a field report.
|
|
95
|
+
|
|
96
|
+
## Other audit axes, briefly
|
|
97
|
+
|
|
98
|
+
**Memory / skills / prompts.** The main-loop prompt is assembled in memory and
|
|
99
|
+
never persisted (`build_prompt`, `autonomy/run.sh:8987`), so prompt-version
|
|
100
|
+
attribution is not currently possible. Review prompts *are* persisted
|
|
101
|
+
(`run.sh:14700`). Any prompt-versioning proposal has to start by making the
|
|
102
|
+
main prompt observable, and that is a runtime change -- out of scope here.
|
|
103
|
+
|
|
104
|
+
**Context compression.** The prompt splits at `[CACHE_BREAKPOINT]` into a
|
|
105
|
+
cache-stable prefix and a volatile tail. That is a real, already-shipped
|
|
106
|
+
compression discipline. It is not currently measured per-run.
|
|
107
|
+
|
|
108
|
+
**Model routing by quality/latency/cost.** `get_rarv_tier()` maps iteration to
|
|
109
|
+
model tier. The mapping is deterministic and readable; what is absent is any
|
|
110
|
+
*recorded* per-call association between the tier chosen and the outcome it
|
|
111
|
+
produced, which is what a routing evaluation would need.
|
|
112
|
+
|
|
113
|
+
## Proposal: `loop-harness-v1`, read-only, one slice
|
|
114
|
+
|
|
115
|
+
**Do not add a loop. Do not change runtime architecture.** The single coherent
|
|
116
|
+
reversible slice is a **report over traces that already exist**, plus the
|
|
117
|
+
smallest instrumentation that makes the report non-vacuous.
|
|
118
|
+
|
|
119
|
+
### Phase A -- report only, zero runtime change
|
|
120
|
+
|
|
121
|
+
`tools/loop-harness-report.py`, a read-only reader in the shape of the existing
|
|
122
|
+
`api_*` modules:
|
|
123
|
+
|
|
124
|
+
- reads `.loki/events.jsonl`
|
|
125
|
+
- emits one row per verifier invocation it can actually observe
|
|
126
|
+
- **every field it cannot derive reads UNKNOWN, never a default**
|
|
127
|
+
- carries the standard envelope: `source`, `freshness_s`, `reason`
|
|
128
|
+
- exits 3 (nothing to check) on a workspace with no verifier events
|
|
129
|
+
|
|
130
|
+
This is honest on day one: most columns read UNKNOWN, and the report says so.
|
|
131
|
+
That is the correct starting point -- it makes the gap visible and measurable
|
|
132
|
+
rather than asserting a completeness that does not exist.
|
|
133
|
+
|
|
134
|
+
### Phase B -- instrumentation, only where A proves it is needed
|
|
135
|
+
|
|
136
|
+
Add the missing fields to the three gate functions and the council completion
|
|
137
|
+
event. Each addition is one `emit_event_json` call with named fields, and each
|
|
138
|
+
is independently revertable.
|
|
139
|
+
|
|
140
|
+
Phase B is **not** proposed for adoption yet: it changes runtime behaviour, and
|
|
141
|
+
the directive says keep runtime unchanged unless an evidenced deterministic
|
|
142
|
+
requirement justifies it. Phase A produces that evidence.
|
|
143
|
+
|
|
144
|
+
### What would make this worth automating
|
|
145
|
+
|
|
146
|
+
The directive's bar is the right one: automate only when offline replay and
|
|
147
|
+
online outcomes prove quality-adjusted lift exceeds latency and cost. Phase A
|
|
148
|
+
cannot clear that bar and does not try -- it is the measurement that would let
|
|
149
|
+
a later proposal clear it.
|
|
150
|
+
|
|
151
|
+
## Rollback
|
|
152
|
+
|
|
153
|
+
Phase A adds one file under `tools/` and touches no runtime path. Rollback is
|
|
154
|
+
deleting it. Nothing in `.loki/` is written, so there is no state to unwind.
|
|
155
|
+
|
|
156
|
+
## The four surfaces, measured
|
|
157
|
+
|
|
158
|
+
| Surface | State | Evidence |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| Composable core-agent loop | present, modular | `run_autonomous()` + `get_rarv_tier()` + `build_prompt()` are separable functions |
|
|
161
|
+
| Bounded verification loop | present, **unmeasured** | 3 gate fns emit 0 structured records; `code_review_complete` carries 3 fields |
|
|
162
|
+
| Real-system event-driven loop | present, **partial contract** | `autonomy/trigger-server.py`: auth 7, timeout 12, idempotency 4, retry 2 -- but dedupe 0, dead-letter 0, backpressure 0 |
|
|
163
|
+
| Self-improvement / hill-climbing | present, **not wired to traces** | `LOKI_AUTO_LEARNINGS` appears 0 times in `run.sh`; the TS route has it (`counter_evidence.ts`, `episode_bridge.ts`) |
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
for p in idempot dedupe dead.letter backpressure retry timeout auth; do
|
|
167
|
+
printf '%-16s %s\n' "$p" "$(grep -ciE "$p" autonomy/trigger-server.py)"
|
|
168
|
+
done
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### The three exact gaps
|
|
172
|
+
|
|
173
|
+
1. **Verifier records carry no cost, latency, criterion, or effect.** This is
|
|
174
|
+
the blocker for every downstream ask -- a marginal-lift comparison, a
|
|
175
|
+
promotion rule, and a canary decision all need per-verifier cost and
|
|
176
|
+
outcome, and none is emitted.
|
|
177
|
+
|
|
178
|
+
2. **The trigger contract is three properties short.** Auth, timeout,
|
|
179
|
+
idempotency and bounded retry exist. Dedupe, dead-letter state and
|
|
180
|
+
backpressure do not. An idempotent trigger without dedupe still processes a
|
|
181
|
+
duplicate delivery; without dead-letter state a poisoned message retries to
|
|
182
|
+
its cap and vanishes.
|
|
183
|
+
|
|
184
|
+
3. **The learnings loop is route-asymmetric.** `LOKI_AUTO_LEARNINGS` is
|
|
185
|
+
documented as default-on in the Bun runner and is absent from `run.sh`, so
|
|
186
|
+
the bash route contributes nothing to hill-climbing. Any trace-driven
|
|
187
|
+
improvement claim measured on one route does not transfer to the other.
|
|
188
|
+
|
|
189
|
+
## Why architecture stays unchanged
|
|
190
|
+
|
|
191
|
+
Every downstream ask in the directive -- matched online cohorts, marginal-lift
|
|
192
|
+
per verifier, a promotion rule, a canary window -- is **downstream of
|
|
193
|
+
measurement that does not exist**. Building a manifest, a cohort comparison or
|
|
194
|
+
an automation rule on top of absent instrumentation would produce numbers with
|
|
195
|
+
no referent.
|
|
196
|
+
|
|
197
|
+
The cheapest surface that changes this is Phase A: a read-only reader that
|
|
198
|
+
reports what IS recorded and names what is not. It is implemented and tested
|
|
199
|
+
(`tools/loop-harness-report.py`, 8 assertions, both fabrication modes
|
|
200
|
+
mutation-tested). Against this repo's own trace it reads 776 records, finds no
|
|
201
|
+
verifier events, and exits 3 with a reason rather than printing an empty table
|
|
202
|
+
that reads as a clean run.
|
|
203
|
+
|
|
204
|
+
**The smallest reversible next step** is adopting Phase A and running it over
|
|
205
|
+
a real build's trace. That yields the first honest per-verifier row set, and
|
|
206
|
+
its UNKNOWN columns are the evidenced requirement that would justify Phase B
|
|
207
|
+
instrumentation -- which is a runtime change and is deliberately not proposed
|
|
208
|
+
until that evidence exists.
|
|
209
|
+
|
|
210
|
+
## Correction: the six named axes, measured
|
|
211
|
+
|
|
212
|
+
The first pass treated routing as unmeasured. That was wrong, and the correction
|
|
213
|
+
matters because it changes which work is worth doing.
|
|
214
|
+
|
|
215
|
+
| Axis | State | Evidence |
|
|
216
|
+
|---|---|---|
|
|
217
|
+
| Memory / retrieval | wired into the loop | 13 references in `run.sh` |
|
|
218
|
+
| Tool descriptions | present | 13 in `mcp/server.py` |
|
|
219
|
+
| Context compression | shipped, unmeasured per-run | `[CACHE_BREAKPOINT]` prefix split |
|
|
220
|
+
| Prompts | **not attributable** | main prompt in memory only (`run.sh:8987`); review prompts persisted (`:14700`) |
|
|
221
|
+
| **Quality/latency/cost routing** | **fully recorded** | see below |
|
|
222
|
+
| Verifier records | **absent** | unchanged from the first pass |
|
|
223
|
+
|
|
224
|
+
### Routing is already instrumented
|
|
225
|
+
|
|
226
|
+
`.loki/metrics/efficiency/` records carry:
|
|
227
|
+
|
|
228
|
+
```
|
|
229
|
+
model provider phase iteration status timestamp
|
|
230
|
+
cost_usd duration_ms
|
|
231
|
+
input_tokens output_tokens cache_read_tokens cache_creation_tokens
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
`LOKI_CURRENT_MODEL` holds the EXACT dispatched `--model` value, exported after
|
|
235
|
+
every mutation (opus-pin, `LOKI_MAX_TIER` clamp, mid-flight override), so the
|
|
236
|
+
recorded model is the one actually used. `run.sh:7834` documents the bug that
|
|
237
|
+
made this necessary: hardcoding the development-tier default mislabeled every
|
|
238
|
+
non-development iteration and "made the model-equivalence bench unfalsifiable."
|
|
239
|
+
|
|
240
|
+
`record_is_measured()` (`autonomy/lib/efficiency_cost.py:81`) is the single
|
|
241
|
+
definition of measured, and its docstring records why a second copy is
|
|
242
|
+
forbidden: "the four surfaces that once rendered an unmeasured run as $0.00
|
|
243
|
+
each had their own idea of what counted as measured."
|
|
244
|
+
|
|
245
|
+
`dashboard/api_runs.py` already reads these (22 references).
|
|
246
|
+
|
|
247
|
+
**So a quality/latency/cost routing evaluation is possible today** -- model,
|
|
248
|
+
cost, latency and tokens are all recorded per iteration with an honesty
|
|
249
|
+
predicate. What is missing is not instrumentation but a *comparison*: no
|
|
250
|
+
baseline pins a model choice to an outcome.
|
|
251
|
+
|
|
252
|
+
### The caveat, and why it is smaller than it looked
|
|
253
|
+
|
|
254
|
+
Efficiency records are WIPED at run start (`run.sh:6212`), which is why a
|
|
255
|
+
historical run reports `cost_usd: None` and why this workspace holds zero
|
|
256
|
+
records.
|
|
257
|
+
|
|
258
|
+
But receipts survive, and they DO carry the model. `proof-generator.py:627`
|
|
259
|
+
resolves it through four sources -- an observed value, `LOKI_CURRENT_MODEL`,
|
|
260
|
+
`LOKI_SESSION_MODEL`, `SESSION_MODEL` -- then the execution policy's
|
|
261
|
+
`sdk_id`/`alias`, and only then returns the string `"unavailable"`. It never
|
|
262
|
+
guesses.
|
|
263
|
+
|
|
264
|
+
Verified end to end: generating a receipt with `LOKI_CURRENT_MODEL=
|
|
265
|
+
claude-sonnet-5` yields `provider: {"name": "claude", "model":
|
|
266
|
+
"claude-sonnet-5"}`. The nine archived receipts read `"model": "unavailable"`
|
|
267
|
+
with `cost_usd: None` because they predate the efficiency wiring, not because
|
|
268
|
+
the mechanism is missing.
|
|
269
|
+
|
|
270
|
+
**So the cross-run corpus for a routing evaluation is the receipt archive**,
|
|
271
|
+
which retains model, provider, cost, tokens and wall clock per run. No new
|
|
272
|
+
instrumentation is required.
|
|
273
|
+
|
|
274
|
+
### What this changes about the proposal
|
|
275
|
+
|
|
276
|
+
Verifier records remain the real gap, unchanged. But routing does NOT need
|
|
277
|
+
Phase B instrumentation -- it needs an evaluation over data that already
|
|
278
|
+
exists. That is a cheaper and better-evidenced next step than instrumenting
|
|
279
|
+
the gates, and it is squarely inside the directive's "improve routing before
|
|
280
|
+
touching architecture."
|
|
281
|
+
|
|
282
|
+
## The routing evaluation: blocked on corpus, not on code
|
|
283
|
+
|
|
284
|
+
Having established that receipts retain the model, the obvious next step is the
|
|
285
|
+
evaluation itself. It cannot be built yet, and the reason is worth recording
|
|
286
|
+
precisely.
|
|
287
|
+
|
|
288
|
+
Measured across the whole archive:
|
|
289
|
+
|
|
290
|
+
```
|
|
291
|
+
receipts: 9
|
|
292
|
+
models: {"unavailable": 9}
|
|
293
|
+
with measured cost: 0
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
Every receipt reads `"model": "unavailable"` and `cost_usd: None`. They were
|
|
297
|
+
written Jul 26 and Jul 31; the efficiency wiring that populates both landed
|
|
298
|
+
later. The mechanism is proven to work -- generating a receipt with
|
|
299
|
+
`LOKI_CURRENT_MODEL` set captures the exact model -- but no archived run
|
|
300
|
+
exercised it.
|
|
301
|
+
|
|
302
|
+
**So a routing evaluation today would have zero rows to compare.** Building it
|
|
303
|
+
now produces a reader with nothing to read, and any number it reported would
|
|
304
|
+
be derived from a single degenerate cohort.
|
|
305
|
+
|
|
306
|
+
That is a corpus problem, not a code problem, and the fix is not more code: it
|
|
307
|
+
is running builds and letting the archive fill. Each real `loki start` from
|
|
308
|
+
here produces a receipt carrying model, provider, cost, tokens and wall clock.
|
|
309
|
+
The evaluation becomes worth writing once the archive holds more than one
|
|
310
|
+
distinct model.
|
|
311
|
+
|
|
312
|
+
### What this means for sequencing
|
|
313
|
+
|
|
314
|
+
The directive asks for verifier lift to exceed latency and cost on baseline
|
|
315
|
+
plus ambitious E2E plus online outcomes. That bar cannot be cleared from a
|
|
316
|
+
corpus of nine degenerate rows, and no amount of tooling changes it.
|
|
317
|
+
|
|
318
|
+
The honest ordering is therefore:
|
|
319
|
+
|
|
320
|
+
1. accumulate receipts from real runs (no code required)
|
|
321
|
+
2. write the evaluation once two or more distinct models appear
|
|
322
|
+
3. only then consider verifier instrumentation, which is the one gap where
|
|
323
|
+
the data genuinely does not exist at any volume
|
|
324
|
+
|
|
325
|
+
Steps 1 and 2 need no runtime change. Step 3 remains unproposed.
|
|
326
|
+
|
|
327
|
+
## Step 1 executed: the corpus has its first real row
|
|
328
|
+
|
|
329
|
+
Rather than leave "accumulate receipts" as advice, one real `loki start` was
|
|
330
|
+
run against a minimal spec. It completed exit 0, built working code (its own
|
|
331
|
+
6 tests pass), and wrote a receipt carrying exactly what a routing evaluation
|
|
332
|
+
needs:
|
|
333
|
+
|
|
334
|
+
```
|
|
335
|
+
provider {"name": "claude", "model": "sonnet"} <- not "unavailable"
|
|
336
|
+
cost_usd 1.3828 input/output tokens 48 / 7290
|
|
337
|
+
iterations 1 succeeded, 0 failed
|
|
338
|
+
duration_ms 128000 wall_clock_sec 576
|
|
339
|
+
base_sha ef1efe909750 head_sha 5a727616da91
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
Contrast with the nine archived receipts, all `"model": "unavailable"` and
|
|
343
|
+
`cost_usd: None`. The mechanism was never broken; those runs simply predate
|
|
344
|
+
it.
|
|
345
|
+
|
|
346
|
+
`iterations.attribution` is worth noting for any lift measurement: it splits
|
|
347
|
+
cost into `progress` and `rework`, and states its own basis -- "rework counts
|
|
348
|
+
FAILED iterations only; a completed iteration forced to repeat by a gate is
|
|
349
|
+
counted as progress, so rework is a floor". That is a self-describing lower
|
|
350
|
+
bound rather than an unqualified number.
|
|
351
|
+
|
|
352
|
+
### The verdict reads FAILED, correctly
|
|
353
|
+
|
|
354
|
+
`receipts_report` returns FAILED with `measured: True` and the real cost. The
|
|
355
|
+
reason is diff drift: 11 files / +494 recorded, more now. The cause was
|
|
356
|
+
verified by timestamp -- only `run.log` (still being appended) and
|
|
357
|
+
`.pytest_cache/` from the verification run itself changed after signing.
|
|
358
|
+
|
|
359
|
+
That is the verifier working. A receipt signed at time T and inspected at
|
|
360
|
+
T+delta, with files touched in between, SHOULD fail. The lesson for a future
|
|
361
|
+
evaluation harness: read receipts without running anything inside the
|
|
362
|
+
workspace, or the act of measuring invalidates what is measured.
|
|
363
|
+
|
|
364
|
+
### Where the corpus stands
|
|
365
|
+
|
|
366
|
+
Two distinct models are needed before a comparison means anything. The archive
|
|
367
|
+
now holds one real row (`sonnet`) plus nine degenerate ones. The evaluation
|
|
368
|
+
remains unwritten, and that is still the honest position -- but step 1 is no
|
|
369
|
+
longer hypothetical.
|
|
370
|
+
|
|
371
|
+
## Step 2: two models, and the first real comparison
|
|
372
|
+
|
|
373
|
+
A second real run, pinned with `LOKI_SESSION_MODEL=opus`, gives the corpus its
|
|
374
|
+
second distinct model. Both runs used the same shape of task (a small pure
|
|
375
|
+
helper plus its test) and both succeeded in one iteration.
|
|
376
|
+
|
|
377
|
+
| | sonnet | opus |
|
|
378
|
+
|---|---|---|
|
|
379
|
+
| cost_usd | 1.3828 | **0.6823** |
|
|
380
|
+
| output tokens | 7290 | **4446** |
|
|
381
|
+
| wall_clock_sec | 576 | **472** |
|
|
382
|
+
| progress duration_ms | 128000 | **71000** |
|
|
383
|
+
| iterations | 1 succeeded | 1 succeeded |
|
|
384
|
+
|
|
385
|
+
**This is two data points, not a finding.** Two runs of two different specs
|
|
386
|
+
cannot separate model effect from task effect, and the directive's bar --
|
|
387
|
+
lift exceeding latency and cost across baseline, ambitious E2E, and online
|
|
388
|
+
cohorts -- is nowhere near cleared. Recorded because it is the first
|
|
389
|
+
comparison the corpus has ever supported, and because the shape is now
|
|
390
|
+
proven: the fields needed for a routing evaluation arrive populated and
|
|
391
|
+
measured on every real run.
|
|
392
|
+
|
|
393
|
+
## A real defect the corpus work surfaced
|
|
394
|
+
|
|
395
|
+
Both receipts read FAILED on diff drift. The first time, the cause was my own
|
|
396
|
+
verification run writing `.pytest_cache/`. The second time I deliberately
|
|
397
|
+
touched nothing, and it STILL failed. The only file newer than the receipt:
|
|
398
|
+
|
|
399
|
+
```
|
|
400
|
+
run.log
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
**A run whose stdout is redirected into its own workspace invalidates its own
|
|
404
|
+
receipt.** The log keeps growing after the receipt is signed, so the recorded
|
|
405
|
+
diff no longer matches the tree. Any user who runs
|
|
406
|
+
`loki start ./prd.md > run.log` inside the workspace gets a receipt that
|
|
407
|
+
cannot verify, through no fault of their own.
|
|
408
|
+
|
|
409
|
+
### Scoping that claim honestly
|
|
410
|
+
|
|
411
|
+
The first write-up called this a usability trap users would hit. Checking
|
|
412
|
+
rather than assuming shrinks it:
|
|
413
|
+
|
|
414
|
+
- the documented invocation is plain `loki start prd.md` (README:333, 342,
|
|
415
|
+
360) with NO redirection, so an interactive user never creates `run.log` in
|
|
416
|
+
the workspace
|
|
417
|
+
- the runtime has no concept of its own log path -- grepping `run.sh` and the
|
|
418
|
+
CLI for `LOKI_RUN_LOG`, `RUN_LOG=` or a log-path helper returns nothing
|
|
419
|
+
- `> run.log` appears nowhere in the docs; it was MY invocation choice for a
|
|
420
|
+
backgrounded run
|
|
421
|
+
|
|
422
|
+
So this is not a shipped defect users are hitting. It is a real constraint on
|
|
423
|
+
anyone who captures stdout inside the workspace -- CI harnesses, scripted
|
|
424
|
+
runs, and future evaluation tooling -- and that includes the routing
|
|
425
|
+
evaluation this audit is building toward.
|
|
426
|
+
|
|
427
|
+
`workspace_diff.py:28` already has an `_excluded()` predicate (currently
|
|
428
|
+
`.loki/` only), so excluding a known log path would be a one-line change. It
|
|
429
|
+
is NOT proposed, for a specific reason: excluding by the literal name
|
|
430
|
+
`run.log` would silently drop a user's own file of that name from the receipt,
|
|
431
|
+
which is a worse failure than the one being fixed. A correct fix needs the
|
|
432
|
+
runtime to know its own log path, and that is a runtime change with no
|
|
433
|
+
evidenced user demand behind it.
|
|
434
|
+
|
|
435
|
+
The durable output is the constraint, not a patch: **an evaluation harness
|
|
436
|
+
must not write inside the workspace it measures**, including redirecting the
|
|
437
|
+
run's own stdout there. Recorded so it is not rediscovered a third time.
|
|
438
|
+
|
|
439
|
+
## Correction: two of the three trigger gaps were my grep, not the code
|
|
440
|
+
|
|
441
|
+
The audit claimed `autonomy/trigger-server.py` was three properties short:
|
|
442
|
+
dedupe 0, dead-letter 0, backpressure 0. Two of those were false negatives from
|
|
443
|
+
searching for the wrong words.
|
|
444
|
+
|
|
445
|
+
**Dedupe exists** and is well built. `Dispatcher.seen_delivery()` keeps recent
|
|
446
|
+
GitHub delivery ids in a lock-guarded bounded `OrderedDict`. It deliberately
|
|
447
|
+
does NOT refresh recency on a duplicate, and the code says why: "a flood of one
|
|
448
|
+
valid (authenticated) duplicate id could keep it pinned and evict up to
|
|
449
|
+
dedup_max genuinely-recent ids, letting real redeliveries slip through."
|
|
450
|
+
|
|
451
|
+
Verified by executing the method directly:
|
|
452
|
+
|
|
453
|
+
```
|
|
454
|
+
first delivery abc -> False (new)
|
|
455
|
+
repeat delivery abc -> True (deduped)
|
|
456
|
+
absent header "" -> False (never deduplicated, falls through)
|
|
457
|
+
after eviction -> False (bounded, evicts oldest first)
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
**Backpressure exists.** The dispatcher holds a bounded `queue.Queue`, catches
|
|
461
|
+
`queue.Full`, and sheds load with 503 -- the module docstring states it at line
|
|
462
|
+
12. My earlier check used a malformed `-E` alternation and returned 0 for terms
|
|
463
|
+
plainly present in the file (`queue_size` alone appears 7 times).
|
|
464
|
+
|
|
465
|
+
**Dead-letter state is the one that is genuinely absent.** No DLQ, no failed-job
|
|
466
|
+
retention: a job that exhausts its retries is dropped.
|
|
467
|
+
|
|
468
|
+
### The lesson, which is the same one twice
|
|
469
|
+
|
|
470
|
+
`phases` was reported missing from `loki proof --help` by a grep that could not
|
|
471
|
+
see it; dedupe was reported absent by a grep looking for the wrong noun. Both
|
|
472
|
+
times the code was fine and the measurement was broken.
|
|
473
|
+
|
|
474
|
+
An absence found by grep is a hypothesis, not a finding. It has to be confirmed
|
|
475
|
+
by reading the code or executing it -- exactly the standard this audit applies
|
|
476
|
+
to the runtime, now applied to the audit itself.
|
|
477
|
+
|
|
478
|
+
### Revised trigger contract state
|
|
479
|
+
|
|
480
|
+
| Property | State |
|
|
481
|
+
|---|---|
|
|
482
|
+
| authentication | present |
|
|
483
|
+
| timeout | present |
|
|
484
|
+
| bounded retry | present |
|
|
485
|
+
| idempotency | present |
|
|
486
|
+
| **dedupe** | **present** (verified by execution) |
|
|
487
|
+
| **backpressure** | **present** (bounded queue, 503 shed) |
|
|
488
|
+
| dead-letter state | absent |
|
|
489
|
+
|
|
490
|
+
One property short, not three. A trigger-to-run-to-receipt path is therefore
|
|
491
|
+
much closer to complete than the audit claimed, and the remaining gap is
|
|
492
|
+
narrow: a job that exhausts retries vanishes without a record.
|
|
493
|
+
|
|
494
|
+
## Third correction: the last "gap" is smaller still
|
|
495
|
+
|
|
496
|
+
Applying the new standard to my own remaining claim -- dead-letter state absent
|
|
497
|
+
-- turned up a near-miss worth recording, because the mistake is instructive.
|
|
498
|
+
|
|
499
|
+
Reading `_worker` (trigger-server.py:425) shows `dispatch_event` wrapped in
|
|
500
|
+
`try/finally` with NO `except`. That looks like a serious defect: one handler
|
|
501
|
+
exception would kill the worker thread, and with `DEFAULT_WORKERS = 4`, four
|
|
502
|
+
failures would silently drain all capacity while the server kept returning 200.
|
|
503
|
+
|
|
504
|
+
I reproduced exactly that behaviour and was ready to report it. **The
|
|
505
|
+
reproduction was wrong**: it used a stub that raised, not the real
|
|
506
|
+
`dispatch_event`.
|
|
507
|
+
|
|
508
|
+
The real function catches everything (`trigger-server.py:356`):
|
|
509
|
+
|
|
510
|
+
```python
|
|
511
|
+
except Exception as e: # defensive: never let a worker die on bad input
|
|
512
|
+
logging.exception("Handler for %s raised: %s", event_type, e)
|
|
513
|
+
summary, status = None, "error"
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
Verified against the actual code: `dispatch_event` returns `(None, "error")`
|
|
517
|
+
without raising, logs the traceback, and the daemon thread count is unchanged
|
|
518
|
+
(1 before, 1 after). The worker survives, and the guard is placed at the callee
|
|
519
|
+
precisely so the bare `try/finally` in the loop is safe.
|
|
520
|
+
|
|
521
|
+
### What that leaves
|
|
522
|
+
|
|
523
|
+
A failed job IS recorded -- `logging.exception` plus `log_event(..., status)`.
|
|
524
|
+
So "dead-letter state absent" overstates it too. What is genuinely missing is a
|
|
525
|
+
*queryable* record: the failure lands in logs, not in a structure something
|
|
526
|
+
could retry from or report on.
|
|
527
|
+
|
|
528
|
+
That is a real but narrow gap, and it is not worth a runtime change on this
|
|
529
|
+
evidence.
|
|
530
|
+
|
|
531
|
+
### Score on my own audit
|
|
532
|
+
|
|
533
|
+
Of the three trigger gaps originally claimed:
|
|
534
|
+
|
|
535
|
+
- dedupe -- **present**, found by grepping the wrong noun
|
|
536
|
+
- backpressure -- **present**, found by a malformed regex
|
|
537
|
+
- dead-letter -- **overstated**; failures are logged, just not queryable
|
|
538
|
+
|
|
539
|
+
And one defect I nearly reported was an artifact of testing my own mock.
|
|
540
|
+
|
|
541
|
+
Four measurement errors in one audit section. The corrective standard, now
|
|
542
|
+
demonstrated three times: **an absence is a hypothesis until the real code is
|
|
543
|
+
read or executed** -- and a reproduction must exercise the real function, not
|
|
544
|
+
a stand-in shaped like it.
|
|
545
|
+
|
|
546
|
+
## Clean checkpoint: 5605deca, all 19 jobs green
|
|
547
|
+
|
|
548
|
+
Recorded per the gate discipline: the trigger-contract correction reached
|
|
549
|
+
`Tests@5605deca: completed / success`, 19 of 19 jobs, zero failures. No run was
|
|
550
|
+
superseded to get there.
|
|
551
|
+
|
|
552
|
+
### Where the four surfaces actually stand, after all corrections
|
|
553
|
+
|
|
554
|
+
| Surface | State | Confidence |
|
|
555
|
+
|---|---|---|
|
|
556
|
+
| Composable core-agent | modular, separable functions | read |
|
|
557
|
+
| Bounded verification | verifiers run; records carry no cost/latency/criterion/effect | read + grep |
|
|
558
|
+
| Event-driven trigger | auth, timeout, retry, idempotency, dedupe, backpressure all present; failures logged but not queryable | **executed** |
|
|
559
|
+
| Self-improvement | `LOKI_AUTO_LEARNINGS` on the TS route only, absent from `run.sh` and the CLI | grep, unverified |
|
|
560
|
+
|
|
561
|
+
The confidence column matters more than the state column. Everything I checked
|
|
562
|
+
by execution survived scrutiny; three of the four things I checked by grep did
|
|
563
|
+
not.
|
|
564
|
+
|
|
565
|
+
### The smallest next slice, and why it is not code
|
|
566
|
+
|
|
567
|
+
The directive's bar for any verifier change is a matched ablation showing lift
|
|
568
|
+
above p95 latency and cost, on the same task, across seeds. Nothing in this
|
|
569
|
+
repo can currently produce that number:
|
|
570
|
+
|
|
571
|
+
- verifier records carry no cost or latency at all, so the denominator is
|
|
572
|
+
missing
|
|
573
|
+
- the routing corpus holds two real rows from two DIFFERENT specs, which is
|
|
574
|
+
explicitly excluded ("do not infer this from different tasks or one seed")
|
|
575
|
+
|
|
576
|
+
So the smallest honest next slice is **the same spec run twice under two
|
|
577
|
+
models**, which is the minimum a matched ablation admits. That is one command
|
|
578
|
+
and no code. Until it exists, any harness or eval built on top would be
|
|
579
|
+
measuring a corpus that cannot support the claim.
|
|
580
|
+
|
|
581
|
+
The one surface still unverified by execution is self-improvement: the
|
|
582
|
+
`LOKI_AUTO_LEARNINGS` asymmetry was found by grep, and grep has been wrong
|
|
583
|
+
three times in this audit. It should be confirmed by running before it is
|
|
584
|
+
treated as a finding.
|
|
585
|
+
|
|
586
|
+
## The matched pair, executed
|
|
587
|
+
|
|
588
|
+
The previous entry said the smallest honest next step was the same spec run
|
|
589
|
+
twice under two models. That has now been done: identical `prd.md` (byte-for-byte,
|
|
590
|
+
`diff -q` verified), identical iteration cap, run SEQUENTIALLY so CPU contention
|
|
591
|
+
could not confound latency, logs written OUTSIDE each workspace so the run could
|
|
592
|
+
not invalidate its own receipt.
|
|
593
|
+
|
|
594
|
+
| | sonnet | opus |
|
|
595
|
+
|---|---|---|
|
|
596
|
+
| cost_usd | 0.7177 | **0.6257** |
|
|
597
|
+
| output tokens | 3658 | 3547 |
|
|
598
|
+
| wall_clock_sec | 458 | **406** |
|
|
599
|
+
| progress duration_ms | 65000 | 67000 |
|
|
600
|
+
| iterations | 1 of 1 succeeded | 1 of 1 succeeded |
|
|
601
|
+
| produced code | 1 test passing | 1 test passing |
|
|
602
|
+
|
|
603
|
+
Both arms solved the task. On this pair, opus was 12.8% cheaper and 11% faster
|
|
604
|
+
in wall clock, while taking marginally longer in measured progress duration --
|
|
605
|
+
the wall/duration split suggests the difference sits in orchestration overhead,
|
|
606
|
+
not model latency.
|
|
607
|
+
|
|
608
|
+
### What this does and does not license
|
|
609
|
+
|
|
610
|
+
It **does** establish that the corpus can now produce a matched comparison: same
|
|
611
|
+
task, same budget, controlled ordering, both outcomes verified by running the
|
|
612
|
+
produced tests rather than trusting the receipt.
|
|
613
|
+
|
|
614
|
+
It **does not** clear the directive's bar. That requires lift above p95 latency
|
|
615
|
+
and cost across seeds, and this is n=1 per arm. A single pair cannot separate
|
|
616
|
+
model effect from run-to-run variance, and the two prior unmatched runs
|
|
617
|
+
(sonnet $1.3828, opus $0.6823) differ from these by more than the arms differ
|
|
618
|
+
from each other -- which is itself the evidence that one pair proves nothing
|
|
619
|
+
about the population.
|
|
620
|
+
|
|
621
|
+
The honest reading: the *method* is now demonstrated end to end. The *finding*
|
|
622
|
+
needs repetition, and repetition is provider spend, which is a founder call.
|
|
623
|
+
|
|
624
|
+
### Method notes worth keeping
|
|
625
|
+
|
|
626
|
+
Three procedural constraints were learned by getting them wrong first:
|
|
627
|
+
|
|
628
|
+
1. log outside the workspace, or the run invalidates its own receipt
|
|
629
|
+
2. run arms sequentially, or contention confounds the latency column
|
|
630
|
+
3. verify the produced artifact by executing it -- a receipt records what a run
|
|
631
|
+
claimed to do, not whether the code works
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var u_=Object.create;var{getPrototypeOf:p_,defineProperty:eK,getOwnPropertyNames:d_}=Object;var c_=Object.prototype.hasOwnProperty;function l_(Z){return this[Z]}var i_,a_,s_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?i_??=new WeakMap:a_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?u_(p_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of d_(Z))if(!c_.call(K,$))eK(K,$,{get:l_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var n_=(Z)=>Z;function o_(Z,X){this[Z]=n_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:o_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as r_}from"url";import{existsSync as UQ}from"fs";import{homedir as t_}from"os";function e_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(t_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(r_(import.meta.url));i0=e_()});import{readFileSync as Zf}from"fs";import{resolve as Xf,dirname as Qf}from"path";import{fileURLToPath as Yf}from"url";function h3(){if(h5!==null)return h5;let Z="9.12.
|
|
2
|
+
var u_=Object.create;var{getPrototypeOf:p_,defineProperty:eK,getOwnPropertyNames:d_}=Object;var c_=Object.prototype.hasOwnProperty;function l_(Z){return this[Z]}var i_,a_,s_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?i_??=new WeakMap:a_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?u_(p_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of d_(Z))if(!c_.call(K,$))eK(K,$,{get:l_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var n_=(Z)=>Z;function o_(Z,X){this[Z]=n_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:o_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as r_}from"url";import{existsSync as UQ}from"fs";import{homedir as t_}from"os";function e_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(t_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(r_(import.meta.url));i0=e_()});import{readFileSync as Zf}from"fs";import{resolve as Xf,dirname as Qf}from"path";import{fileURLToPath as Yf}from"url";function h3(){if(h5!==null)return h5;let Z="9.12.6";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Qf(Yf(import.meta.url)),Q=X$(X);h5=Zf(Xf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var bO={};l0(bO,{runOrThrow:()=>Mf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>wf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>yO});async function NQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Mf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Tf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Tf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function wf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return Cf?"":Z}var Cf,L0,F8,p0,KV0,a0,W8,Q9,v;var S6=p(()=>{Cf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),KV0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),W8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as _f}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(_f(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>$h});import{existsSync as Y9,readFileSync as g3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as tf}from"path";import{homedir as ef}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*LQ/X);if(J>LQ)J=LQ;let z=LQ-J,K=F8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${W8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function Xh(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1232,4 +1232,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1232
1232
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (v_(),h_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1233
1233
|
`),process.stderr.write(g_),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var pW0=await uW0(Bun.argv.slice(2));process.exit(pW0);
|
|
1234
1234
|
|
|
1235
|
-
//# debugId=
|
|
1235
|
+
//# debugId=71D1F11FDC1381C964756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "9.12.
|
|
4
|
+
"version": "9.12.6",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "9.12.
|
|
5
|
+
"version": "9.12.6",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""loop-harness-v1: what the verifiers in a run actually recorded.
|
|
3
|
+
|
|
4
|
+
READ-ONLY. Opens .loki/events.jsonl and nothing else. Writes nothing, spawns
|
|
5
|
+
nothing, and touches no runtime path. Rollback is deleting this file.
|
|
6
|
+
|
|
7
|
+
WHY THIS REPORTS MOSTLY UNKNOWN, AND WHY THAT IS THE POINT. A loop-harness
|
|
8
|
+
manifest wants, per verifier invocation: eligibility, the deterministic
|
|
9
|
+
criterion, retry and timeout caps, latency, tokens, cash cost, the verdict,
|
|
10
|
+
whether it changed the terminal outcome, false-positive review, and a rollback
|
|
11
|
+
switch.
|
|
12
|
+
|
|
13
|
+
Measured at v9.12.5, the runtime emits almost none of that:
|
|
14
|
+
|
|
15
|
+
code_review_complete review_id, source, iteration
|
|
16
|
+
review_verification_failed reason, iteration, implementation_retry
|
|
17
|
+
_evidence_gate_and_surface nothing structured
|
|
18
|
+
_invariant_gate_and_surface nothing structured
|
|
19
|
+
_semantic_gate_and_surface nothing structured
|
|
20
|
+
|
|
21
|
+
So this reader CANNOT derive most columns. It reports them UNKNOWN rather than
|
|
22
|
+
defaulting them, because a manifest that fills a cost column with 0.0 or an
|
|
23
|
+
eligibility column with "yes" is asserting a measurement nobody took -- and a
|
|
24
|
+
fabricated manifest about verification is worse than no manifest at all.
|
|
25
|
+
|
|
26
|
+
The UNKNOWNs are the deliverable. They say exactly which instrumentation is
|
|
27
|
+
missing, so a later decision to add it is evidenced rather than assumed.
|
|
28
|
+
|
|
29
|
+
Exit codes follow this repo's convention:
|
|
30
|
+
0 verifier records were found and reported
|
|
31
|
+
2 the trace could not be read
|
|
32
|
+
3 nothing to check -- no verifier events in this workspace
|
|
33
|
+
64 usage error
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import argparse
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
import sys
|
|
40
|
+
|
|
41
|
+
sys.dont_write_bytecode = True
|
|
42
|
+
|
|
43
|
+
UNKNOWN = "UNKNOWN"
|
|
44
|
+
|
|
45
|
+
# Event types that represent a verifier doing something. Derived from
|
|
46
|
+
# autonomy/run.sh's emit_event_json call sites, not invented here.
|
|
47
|
+
_VERIFIER_EVENTS = {
|
|
48
|
+
"code_review_start",
|
|
49
|
+
"code_review_complete",
|
|
50
|
+
"code_review_council_complete",
|
|
51
|
+
"review_verification_failed",
|
|
52
|
+
"managed_review_council_ok",
|
|
53
|
+
"gate_stuck",
|
|
54
|
+
"policy_denied",
|
|
55
|
+
"task_completion_claim",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
# Fields a loop-harness manifest wants, and where each one comes from today.
|
|
59
|
+
# "" means no emitter records it, so the column reads UNKNOWN for every row.
|
|
60
|
+
_FIELD_SOURCES = {
|
|
61
|
+
"verifier": "event type",
|
|
62
|
+
"iteration": "iteration",
|
|
63
|
+
"verdict": "partial: only the failure path records a reason",
|
|
64
|
+
"eligible": "",
|
|
65
|
+
"criterion": "",
|
|
66
|
+
"retry_cap": "",
|
|
67
|
+
"timeout_cap": "",
|
|
68
|
+
"latency_ms": "",
|
|
69
|
+
"tokens": "",
|
|
70
|
+
"cost_usd": "",
|
|
71
|
+
"changed_terminal_outcome": "",
|
|
72
|
+
"false_positive_reviewed": "",
|
|
73
|
+
"rollback_switch": "",
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _Parser(argparse.ArgumentParser):
|
|
78
|
+
"""Usage errors exit 64, not argparse's default 2.
|
|
79
|
+
|
|
80
|
+
In this repo 2 means "could NOT be checked" -- a real answer about the
|
|
81
|
+
subject. A mistyped flag is not that.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def error(self, message):
|
|
85
|
+
self.print_usage(sys.stderr)
|
|
86
|
+
sys.stderr.write("%s: error: %s\n" % (self.prog, message))
|
|
87
|
+
raise SystemExit(64)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _events(path):
|
|
91
|
+
"""Every well-formed record, oldest first. A torn line is skipped.
|
|
92
|
+
|
|
93
|
+
events.jsonl is appended to by concurrent shell writers, so a partial
|
|
94
|
+
final line is normal operation and must not blank out the history behind
|
|
95
|
+
it.
|
|
96
|
+
"""
|
|
97
|
+
out = []
|
|
98
|
+
try:
|
|
99
|
+
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
100
|
+
for line in fh:
|
|
101
|
+
line = line.strip()
|
|
102
|
+
if not line:
|
|
103
|
+
continue
|
|
104
|
+
try:
|
|
105
|
+
rec = json.loads(line)
|
|
106
|
+
except (ValueError, json.JSONDecodeError):
|
|
107
|
+
continue
|
|
108
|
+
if isinstance(rec, dict):
|
|
109
|
+
out.append(rec)
|
|
110
|
+
except OSError:
|
|
111
|
+
return None
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _row(rec):
|
|
116
|
+
"""One verifier invocation, with every underivable field UNKNOWN."""
|
|
117
|
+
data = rec.get("data") if isinstance(rec.get("data"), dict) else {}
|
|
118
|
+
etype = rec.get("type") or UNKNOWN
|
|
119
|
+
|
|
120
|
+
verdict = UNKNOWN
|
|
121
|
+
if etype in ("code_review_complete", "managed_review_council_ok"):
|
|
122
|
+
# Completion is not a verdict: these fire when the council FINISHED,
|
|
123
|
+
# not when it approved. Recording "pass" here would invent an outcome.
|
|
124
|
+
verdict = UNKNOWN
|
|
125
|
+
elif etype == "review_verification_failed":
|
|
126
|
+
verdict = data.get("reason") or "failed"
|
|
127
|
+
elif etype == "policy_denied":
|
|
128
|
+
verdict = "denied"
|
|
129
|
+
|
|
130
|
+
row = {
|
|
131
|
+
"verifier": etype,
|
|
132
|
+
"timestamp": rec.get("timestamp") or UNKNOWN,
|
|
133
|
+
"iteration": data.get("iteration", UNKNOWN),
|
|
134
|
+
"verdict": verdict,
|
|
135
|
+
}
|
|
136
|
+
for field, source in _FIELD_SOURCES.items():
|
|
137
|
+
if field in row:
|
|
138
|
+
continue
|
|
139
|
+
row[field] = UNKNOWN if not source else data.get(field, UNKNOWN)
|
|
140
|
+
return row
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def report(loki_dir):
|
|
144
|
+
"""The manifest envelope. Empty results always carry a reason."""
|
|
145
|
+
path = os.path.join(loki_dir, "events.jsonl")
|
|
146
|
+
env = {
|
|
147
|
+
"report": "loop-harness-v1",
|
|
148
|
+
"source": [path],
|
|
149
|
+
"rows": [],
|
|
150
|
+
"count": 0,
|
|
151
|
+
"unmeasured_fields": sorted(f for f, s in _FIELD_SOURCES.items()
|
|
152
|
+
if not s),
|
|
153
|
+
"reason": None,
|
|
154
|
+
}
|
|
155
|
+
if not os.path.isdir(loki_dir):
|
|
156
|
+
env["reason"] = "no .loki directory at %s" % loki_dir
|
|
157
|
+
return env, 3
|
|
158
|
+
if not os.path.isfile(path):
|
|
159
|
+
env["reason"] = ("%s does not exist, so no verifier ever recorded "
|
|
160
|
+
"anything here" % path)
|
|
161
|
+
return env, 3
|
|
162
|
+
|
|
163
|
+
records = _events(path)
|
|
164
|
+
if records is None:
|
|
165
|
+
env["reason"] = "%s could not be read" % path
|
|
166
|
+
return env, 2
|
|
167
|
+
|
|
168
|
+
rows = [_row(r) for r in records if r.get("type") in _VERIFIER_EVENTS]
|
|
169
|
+
if not rows:
|
|
170
|
+
# NOT an empty table. "no verifier events" and "the verifiers all
|
|
171
|
+
# passed" are different claims, and only one of them is supported.
|
|
172
|
+
env["reason"] = (
|
|
173
|
+
"no verifier events among %d records in this trace (the run "
|
|
174
|
+
"predates verifier tracing, or no verifier ran)" % len(records))
|
|
175
|
+
return env, 3
|
|
176
|
+
|
|
177
|
+
env["rows"] = rows
|
|
178
|
+
env["count"] = len(rows)
|
|
179
|
+
return env, 0
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def main(argv=None):
|
|
183
|
+
parser = _Parser(description="loop-harness-v1 verifier manifest (read-only)")
|
|
184
|
+
parser.add_argument("--loki-dir", default=os.environ.get("LOKI_DIR")
|
|
185
|
+
or os.path.join(os.getcwd(), ".loki"))
|
|
186
|
+
parser.add_argument("--json", action="store_true")
|
|
187
|
+
args = parser.parse_args(argv)
|
|
188
|
+
|
|
189
|
+
env, code = report(args.loki_dir)
|
|
190
|
+
|
|
191
|
+
if args.json:
|
|
192
|
+
print(json.dumps(env, indent=2))
|
|
193
|
+
return code
|
|
194
|
+
|
|
195
|
+
print("loop-harness-v1 source: %s" % ", ".join(env["source"]))
|
|
196
|
+
if not env["rows"]:
|
|
197
|
+
print(" nothing to check: %s" % env["reason"])
|
|
198
|
+
else:
|
|
199
|
+
print(" %-30s %-10s %s" % ("VERIFIER", "ITERATION", "VERDICT"))
|
|
200
|
+
for r in env["rows"]:
|
|
201
|
+
print(" %-30s %-10s %s"
|
|
202
|
+
% (r["verifier"], r["iteration"], r["verdict"]))
|
|
203
|
+
print(" %d verifier records" % env["count"])
|
|
204
|
+
print("")
|
|
205
|
+
print(" NOT RECORDED BY THE RUNTIME (every row reads UNKNOWN):")
|
|
206
|
+
for f in env["unmeasured_fields"]:
|
|
207
|
+
print(" - %s" % f)
|
|
208
|
+
print(" These are the fields a loop-harness manifest needs and the")
|
|
209
|
+
print(" runtime does not emit. They are reported UNKNOWN rather than")
|
|
210
|
+
print(" defaulted, because a fabricated verification manifest is worse")
|
|
211
|
+
print(" than an absent one.")
|
|
212
|
+
return code
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
if __name__ == "__main__":
|
|
216
|
+
raise SystemExit(main())
|