loki-mode 8.49.0 → 8.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/lib/codex-usage.py +174 -0
- package/autonomy/run.sh +76 -1
- package/dashboard/__init__.py +1 -1
- package/docs/WANG-PRINCIPLES-PLAN.md +179 -0
- package/loki-ts/data/model-pricing.json +53 -35
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v8.
|
|
6
|
+
# Loki Mode v8.51.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -469,4 +469,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
469
469
|
|
|
470
470
|
---
|
|
471
471
|
|
|
472
|
-
**v8.
|
|
472
|
+
**v8.51.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
8.
|
|
1
|
+
8.51.0
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Recover token usage for a codex iteration from its session rollout.
|
|
3
|
+
|
|
4
|
+
W1 in docs/WANG-PRINCIPLES-PLAN.md. Wang states the whole thesis conditionally:
|
|
5
|
+
"the right agentic loop AND the right eval or metric for the agents to
|
|
6
|
+
optimize". We had the loop and not the metric.
|
|
7
|
+
|
|
8
|
+
THE PROBLEM, measured on a real FireLater run: every efficiency record showed
|
|
9
|
+
input_tokens=0, output_tokens=0, cost_usd=0. Not just cost -- we were recording
|
|
10
|
+
NOTHING. `_read_iteration_cost` looks for `.loki/metrics/result-cost-<n>.json`
|
|
11
|
+
or `.loki/context/tracking.json`, and on codex neither exists, so cost silently
|
|
12
|
+
resolved to 0. A zero is a claim the iteration was free.
|
|
13
|
+
|
|
14
|
+
That made cost-per-resolved-issue unmeasurable, which is the axis with a
|
|
15
|
+
MEASURED 20x industry spread (~$14/pass on Sonnet 4.5; an open harness lands
|
|
16
|
+
tasks at ~1/20th Devin's cost). It is the number a buyer compares first.
|
|
17
|
+
|
|
18
|
+
WHY THE ROLLOUT AND NOT `--json`. codex emits usage on its `turn.completed`
|
|
19
|
+
event, but only under `codex exec --json`. The runner pipes codex stdout
|
|
20
|
+
through `tee` into the log files it parses for completion signals, so switching
|
|
21
|
+
the main dispatch to JSONL would change the format every one of those readers
|
|
22
|
+
depends on. codex ALSO persists a session rollout to
|
|
23
|
+
~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl containing `total_token_usage`.
|
|
24
|
+
Reading that is a side channel: zero risk to the dispatch pipeline.
|
|
25
|
+
|
|
26
|
+
FAIL-SAFE: every failure path prints nothing and exits non-zero, so the caller
|
|
27
|
+
records "unknown" rather than a fabricated 0 -- the distinction this exists for.
|
|
28
|
+
|
|
29
|
+
Usage:
|
|
30
|
+
codex-usage.py <since-epoch> [sessions-dir]
|
|
31
|
+
Prints one line: input_tokens output_tokens cached_tokens cache_write_tokens
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import os
|
|
37
|
+
import sys
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
import json
|
|
42
|
+
except Exception: # pragma: no cover
|
|
43
|
+
sys.exit(1)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _find_usage(obj):
|
|
47
|
+
"""Depth-first search for a total_token_usage dict."""
|
|
48
|
+
if isinstance(obj, dict):
|
|
49
|
+
u = obj.get("total_token_usage")
|
|
50
|
+
if isinstance(u, dict):
|
|
51
|
+
return u
|
|
52
|
+
for v in obj.values():
|
|
53
|
+
found = _find_usage(v)
|
|
54
|
+
if found:
|
|
55
|
+
return found
|
|
56
|
+
elif isinstance(obj, list):
|
|
57
|
+
for v in obj:
|
|
58
|
+
found = _find_usage(v)
|
|
59
|
+
if found:
|
|
60
|
+
return found
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def main() -> int:
|
|
65
|
+
if len(sys.argv) < 2:
|
|
66
|
+
return 1
|
|
67
|
+
try:
|
|
68
|
+
since = float(sys.argv[1])
|
|
69
|
+
except ValueError:
|
|
70
|
+
return 1
|
|
71
|
+
|
|
72
|
+
root = Path(sys.argv[2]) if len(sys.argv) > 2 else Path.home() / ".codex" / "sessions"
|
|
73
|
+
if not root.is_dir():
|
|
74
|
+
return 1
|
|
75
|
+
|
|
76
|
+
# Only rollouts written during THIS iteration. Without the mtime bound a
|
|
77
|
+
# stale session from an earlier run would be attributed to this one, which
|
|
78
|
+
# is worse than reporting nothing: it would look like real data.
|
|
79
|
+
candidates = []
|
|
80
|
+
for p in root.rglob("rollout-*.jsonl"):
|
|
81
|
+
try:
|
|
82
|
+
if p.stat().st_mtime >= since:
|
|
83
|
+
candidates.append(p)
|
|
84
|
+
except OSError:
|
|
85
|
+
continue
|
|
86
|
+
if not candidates:
|
|
87
|
+
return 1
|
|
88
|
+
|
|
89
|
+
# The rollout carries CUMULATIVE totals, so the last usage record in the
|
|
90
|
+
# newest matching file is this iteration's total. Summing across files would
|
|
91
|
+
# double-count a resumed session.
|
|
92
|
+
candidates.sort(key=lambda p: p.stat().st_mtime)
|
|
93
|
+
usage = None
|
|
94
|
+
for p in reversed(candidates):
|
|
95
|
+
last = None
|
|
96
|
+
try:
|
|
97
|
+
with p.open(encoding="utf-8", errors="replace") as fh:
|
|
98
|
+
for line in fh:
|
|
99
|
+
line = line.strip()
|
|
100
|
+
if not line:
|
|
101
|
+
continue
|
|
102
|
+
try:
|
|
103
|
+
found = _find_usage(json.loads(line))
|
|
104
|
+
except Exception:
|
|
105
|
+
continue
|
|
106
|
+
if found:
|
|
107
|
+
last = found
|
|
108
|
+
except OSError:
|
|
109
|
+
continue
|
|
110
|
+
if last:
|
|
111
|
+
usage = last
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
if not usage:
|
|
115
|
+
return 1
|
|
116
|
+
|
|
117
|
+
def _n(key):
|
|
118
|
+
v = usage.get(key, 0)
|
|
119
|
+
return int(v) if isinstance(v, (int, float)) and v >= 0 else 0
|
|
120
|
+
|
|
121
|
+
# codex reports input_tokens INCLUSIVE of cached. The pricing tiers charge
|
|
122
|
+
# cached reads at a lower rate, so the uncached remainder is what bills at
|
|
123
|
+
# full input price; emitting the inclusive figure would overstate cost.
|
|
124
|
+
total_in = _n("input_tokens")
|
|
125
|
+
cached = _n("cached_input_tokens")
|
|
126
|
+
uncached = max(total_in - cached, 0)
|
|
127
|
+
|
|
128
|
+
out = _n("output_tokens")
|
|
129
|
+
cwrite = _n("cache_write_input_tokens")
|
|
130
|
+
|
|
131
|
+
# Cost, if the model is priced. codex reports tokens but never dollars, so
|
|
132
|
+
# without this the whole chain still lands on cost_usd=0 -- tokens recovered
|
|
133
|
+
# and the number that matters still missing.
|
|
134
|
+
#
|
|
135
|
+
# A model we cannot price prints an EMPTY cost field, never 0: "unknown" and
|
|
136
|
+
# "free" are different claims and only one of them is honest.
|
|
137
|
+
cost = ""
|
|
138
|
+
model = os.environ.get("LOKI_CODEX_RESOLVED_MODEL", "").strip()
|
|
139
|
+
if model:
|
|
140
|
+
here = Path(__file__).resolve().parent
|
|
141
|
+
for cand in (here / ".." / ".." / "loki-ts" / "data" / "model-pricing.json",):
|
|
142
|
+
try:
|
|
143
|
+
table = json.loads(cand.read_text(encoding="utf-8")).get("pricing", {})
|
|
144
|
+
except Exception:
|
|
145
|
+
continue
|
|
146
|
+
p = table.get(model)
|
|
147
|
+
if not p:
|
|
148
|
+
# Tier names (small/medium/high) and suffixed variants
|
|
149
|
+
# (gpt-5.6-sol-high) both reach here; match the longest prefix.
|
|
150
|
+
for k in sorted(table, key=len, reverse=True):
|
|
151
|
+
if model.startswith(k):
|
|
152
|
+
p = table[k]
|
|
153
|
+
break
|
|
154
|
+
if p:
|
|
155
|
+
try:
|
|
156
|
+
cost = "%.6f" % (
|
|
157
|
+
uncached / 1_000_000 * float(p.get("input", 0))
|
|
158
|
+
+ out / 1_000_000 * float(p.get("output", 0))
|
|
159
|
+
+ cached / 1_000_000 * float(p.get("cache_read", p.get("input", 0)))
|
|
160
|
+
+ cwrite / 1_000_000 * float(p.get("cache_write", p.get("input", 0)))
|
|
161
|
+
)
|
|
162
|
+
except Exception:
|
|
163
|
+
cost = ""
|
|
164
|
+
break
|
|
165
|
+
|
|
166
|
+
print(uncached, out, cached, cwrite, cost)
|
|
167
|
+
return 0
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
if __name__ == "__main__":
|
|
171
|
+
try:
|
|
172
|
+
sys.exit(main())
|
|
173
|
+
except Exception:
|
|
174
|
+
sys.exit(1)
|
package/autonomy/run.sh
CHANGED
|
@@ -10646,7 +10646,21 @@ LOKI_STUCK_JSON
|
|
|
10646
10646
|
)" || return 1
|
|
10647
10647
|
;;
|
|
10648
10648
|
*)
|
|
10649
|
-
|
|
10649
|
+
# SKIP the header. These files open with a static banner --
|
|
10650
|
+
# "# Test mutation findings (HIGH blocks this iteration)" -- which is
|
|
10651
|
+
# byte-identical on every run. Comparing it meant EVERY repeated
|
|
10652
|
+
# failure looked "stuck", including a run making real progress
|
|
10653
|
+
# through different findings each iteration. That is the one
|
|
10654
|
+
# direction this valve must never fail in, and a real FireLater run
|
|
10655
|
+
# is what exposed it: gate-stuck-mutation_integrity.last had
|
|
10656
|
+
# recorded the banner, not a cause.
|
|
10657
|
+
#
|
|
10658
|
+
# Take the first line that is neither blank nor a comment, and strip
|
|
10659
|
+
# ANSI colour (the detectors emit it, and the same finding rendered
|
|
10660
|
+
# with and without colour would otherwise compare unequal).
|
|
10661
|
+
cur="$(grep -vE '^[[:space:]]*(#|$)' "$reason_file" 2>/dev/null \
|
|
10662
|
+
| head -1 \
|
|
10663
|
+
| sed 's/\x1b\[[0-9;]*m//g')" || return 1
|
|
10650
10664
|
;;
|
|
10651
10665
|
esac
|
|
10652
10666
|
[ -n "$cur" ] || return 1
|
|
@@ -22523,6 +22537,11 @@ if __name__ == "__main__":
|
|
|
22523
22537
|
local -a _loki_codex_pipe_status=()
|
|
22524
22538
|
LOKI_CODEX_REASONING_EFFORT="$_loki_codex_effort" \
|
|
22525
22539
|
CODEX_MODEL_REASONING_EFFORT="$_loki_codex_effort" \
|
|
22540
|
+
# Stamp BEFORE the call: the usage reader bounds its rollout
|
|
22541
|
+
# search by mtime, so a stale session from an earlier iteration
|
|
22542
|
+
# cannot be attributed to this one. Attributing the wrong
|
|
22543
|
+
# session is worse than reporting nothing -- it looks like data.
|
|
22544
|
+
_loki_codex_usage_since="$(date +%s 2>/dev/null || echo 0)"
|
|
22526
22545
|
LOKI_DEADLINE_IDLE_TIMEOUT="${LOKI_PROVIDER_IDLE_TIMEOUT:-0}" \
|
|
22527
22546
|
_loki_with_deadline "${LOKI_PROVIDER_CALL_TIMEOUT:-0}" \
|
|
22528
22547
|
codex exec --sandbox workspace-write --skip-git-repo-check \
|
|
@@ -22531,6 +22550,49 @@ if __name__ == "__main__":
|
|
|
22531
22550
|
exit_code="$(_loki_provider_pipeline_exit_code \
|
|
22532
22551
|
"${_loki_codex_pipe_status[0]:-125}" \
|
|
22533
22552
|
"${_loki_codex_pipe_status[1]:-125}" 0)"
|
|
22553
|
+
# W1: recover token usage from the codex session rollout.
|
|
22554
|
+
#
|
|
22555
|
+
# Measured on a real FireLater run: EVERY efficiency record had
|
|
22556
|
+
# input_tokens=0, output_tokens=0, cost_usd=0. Not just cost --
|
|
22557
|
+
# we recorded nothing, because _read_iteration_cost looks for a
|
|
22558
|
+
# result-cost file or context tracker and codex writes neither.
|
|
22559
|
+
# A zero is a claim that the iteration was free.
|
|
22560
|
+
#
|
|
22561
|
+
# codex reports usage only under `codex exec --json`, and the
|
|
22562
|
+
# dispatch above pipes stdout through tee into logs the runner
|
|
22563
|
+
# parses for completion signals -- switching to JSONL would
|
|
22564
|
+
# change the format every one of those readers depends on. The
|
|
22565
|
+
# session rollout carries the same total_token_usage, so this
|
|
22566
|
+
# reads it as a side channel with zero risk to the pipeline.
|
|
22567
|
+
#
|
|
22568
|
+
# Best-effort by construction: on any failure the helper prints
|
|
22569
|
+
# nothing and exits non-zero, and no result-cost file is
|
|
22570
|
+
# written, so cost stays UNKNOWN rather than a fabricated 0.
|
|
22571
|
+
if [ -n "${_loki_codex_usage_since:-}" ] \
|
|
22572
|
+
&& [ -f "${SCRIPT_DIR}/lib/codex-usage.py" ]; then
|
|
22573
|
+
_cx_usage="$(LOKI_CODEX_RESOLVED_MODEL="${LOKI_CURRENT_MODEL:-${PROVIDER_MODEL_DEVELOPMENT:-}}" \
|
|
22574
|
+
python3 "${SCRIPT_DIR}/lib/codex-usage.py" \
|
|
22575
|
+
"$_loki_codex_usage_since" 2>/dev/null)" || _cx_usage=""
|
|
22576
|
+
if [ -n "$_cx_usage" ]; then
|
|
22577
|
+
set -- $_cx_usage
|
|
22578
|
+
mkdir -p "${TARGET_DIR:-.}/.loki/metrics" 2>/dev/null || true
|
|
22579
|
+
# total_cost_usd is emitted ONLY when the model was
|
|
22580
|
+
# priced. Omitting the key leaves cost UNKNOWN; writing
|
|
22581
|
+
# 0 would claim the iteration was free.
|
|
22582
|
+
if [ -n "${5:-}" ]; then
|
|
22583
|
+
printf '{"input_tokens":%s,"output_tokens":%s,"cache_read_tokens":%s,"cache_creation_tokens":%s,"total_cost_usd":%s}\n' \
|
|
22584
|
+
"${1:-0}" "${2:-0}" "${3:-0}" "${4:-0}" "$5" \
|
|
22585
|
+
> "${TARGET_DIR:-.}/.loki/metrics/result-cost-${ITERATION_COUNT}.json" 2>/dev/null || true
|
|
22586
|
+
else
|
|
22587
|
+
printf '{"input_tokens":%s,"output_tokens":%s,"cache_read_tokens":%s,"cache_creation_tokens":%s}\n' \
|
|
22588
|
+
"${1:-0}" "${2:-0}" "${3:-0}" "${4:-0}" \
|
|
22589
|
+
> "${TARGET_DIR:-.}/.loki/metrics/result-cost-${ITERATION_COUNT}.json" 2>/dev/null || true
|
|
22590
|
+
fi
|
|
22591
|
+
log_info "Codex usage: ${1:-0} in (+${3:-0} cached), ${2:-0} out, cost=${5:-unknown}"
|
|
22592
|
+
else
|
|
22593
|
+
log_warn "Codex token usage unavailable for iteration ${ITERATION_COUNT}; cost will read UNKNOWN, not zero."
|
|
22594
|
+
fi
|
|
22595
|
+
fi
|
|
22534
22596
|
;;
|
|
22535
22597
|
|
|
22536
22598
|
cline)
|
|
@@ -22932,6 +22994,19 @@ if __name__ == "__main__":
|
|
|
22932
22994
|
mk_count=$(track_gate_failure "mock_integrity")
|
|
22933
22995
|
gate_failures="${gate_failures}mock_integrity,"
|
|
22934
22996
|
log_warn "Mock integrity gate FAILED ($mk_count consecutive) - CRITICAL/HIGH mock problems"
|
|
22997
|
+
# F0, third gate. Measured on the v8.49.0 FireLater run:
|
|
22998
|
+
# mock_integrity failed 3 times -- MORE than any other
|
|
22999
|
+
# gate -- and was not wired to the stuck check, so an
|
|
23000
|
+
# unfixable mock problem could grind indefinitely.
|
|
23001
|
+
if _loki_gate_stuck "mock_integrity" \
|
|
23002
|
+
"${TARGET_DIR:-.}/.loki/quality/mock-findings.txt" "$mk_count"; then
|
|
23003
|
+
log_error "Mock integrity has failed $mk_count times for the SAME reason. Another iteration would reach the same verdict. Stopping instead of grinding."
|
|
23004
|
+
emit_event_json "gate_stuck" \
|
|
23005
|
+
"gate=mock_integrity" \
|
|
23006
|
+
"consecutive=$mk_count" 2>/dev/null || true
|
|
23007
|
+
save_state "${retry:-0}" "gate_stuck_mock_integrity" 20 2>/dev/null || true
|
|
23008
|
+
return 20
|
|
23009
|
+
fi
|
|
22935
23010
|
;;
|
|
22936
23011
|
*)
|
|
22937
23012
|
_stg_ok=not_run
|
package/dashboard/__init__.py
CHANGED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# Wang's Five Principles: measured audit and plan
|
|
2
|
+
|
|
3
|
+
Founder ask: make Wang's five agentic-system principles the heart of the Loki
|
|
4
|
+
engine, grounded in researched speed / quality / cost data.
|
|
5
|
+
|
|
6
|
+
**This was asked for and not delivered.** A day was spent on gates, packaging
|
|
7
|
+
and test-detection instead. This is the plan, written against measurement
|
|
8
|
+
rather than intention.
|
|
9
|
+
|
|
10
|
+
The five principles, as Wang states them:
|
|
11
|
+
|
|
12
|
+
1. **Systems Thinking** -- never goes out of style
|
|
13
|
+
2. **Speed** -- "one of the most critical things"
|
|
14
|
+
3. **Reliability** -- "extremely important"
|
|
15
|
+
4. **Extensibility** -- scale to complex multi-agent setups
|
|
16
|
+
5. **Feedback Loops + Evals/Metrics** -- *"if you can develop the right agentic
|
|
17
|
+
loop and have the right eval or metric for the agents to optimize, a swarm of
|
|
18
|
+
agents can accomplish more than a team of a hundred engineers"*
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 0. The research finding that reframes everything
|
|
23
|
+
|
|
24
|
+
> **"The same model in a different harness routinely drops 10-15 points --
|
|
25
|
+
> pick the harness, not just the model."** (2026 SWE-bench harness comparisons)
|
|
26
|
+
|
|
27
|
+
That is our entire thesis, stated by the market. We cannot train a frontier
|
|
28
|
+
model. **We do not have to.** The harness is worth 10-15 points, and the harness
|
|
29
|
+
is the thing we build.
|
|
30
|
+
|
|
31
|
+
Supporting 2026 data:
|
|
32
|
+
|
|
33
|
+
| finding | number |
|
|
34
|
+
|---|---|
|
|
35
|
+
| Harness delta on an identical model | **10-15 points** |
|
|
36
|
+
| Top of SWE-bench leaderboard | 75-80% (GPT-5.6 Sol 96.2%, Claude Fable 5 95.0% claimed) |
|
|
37
|
+
| Cost per SWE-bench pass, Sonnet 4.5 | **~$14** |
|
|
38
|
+
| Open harness + open weights vs Devin | **~1/20th the cost** |
|
|
39
|
+
| Agent failures traced to planning | 82% |
|
|
40
|
+
| First-iteration pass rate | 47.8% |
|
|
41
|
+
| Resolutions needing user correction | 91.49% |
|
|
42
|
+
|
|
43
|
+
The cost line matters commercially: **a 20x cost spread exists between harnesses
|
|
44
|
+
running comparable models.** Cost-per-resolved-issue is a competitive axis we
|
|
45
|
+
have never measured.
|
|
46
|
+
|
|
47
|
+
## 1. Honest audit: where we actually stand
|
|
48
|
+
|
|
49
|
+
Measured on this machine, not asserted.
|
|
50
|
+
|
|
51
|
+
| principle | state | evidence |
|
|
52
|
+
|---|---|---|
|
|
53
|
+
| **1 Systems Thinking** | STRONG | 8 quality gates, RARV-C loop, council, Evidence Receipt, dual-route parity enforced by test |
|
|
54
|
+
| **2 Speed** | MEASURED, UNOPTIMISED | agent call = **980s = 96%** of iteration; all gates together = 44s |
|
|
55
|
+
| **3 Reliability** | STRONG, newly so | 73 mutation-proven trust invariants; four gates were shipping broken until v8.38.0 |
|
|
56
|
+
| **4 Extensibility** | STRONG | 4 providers, 41 agent types, MCP (34 tools), plugin marketplace |
|
|
57
|
+
| **5 Feedback Loops / Evals** | **BROKEN** | `cost_usd == 0` on **3 of 5** efficiency records; no eval score for our own harness |
|
|
58
|
+
|
|
59
|
+
**Principle 5 is the gap, and it is the one Wang weights highest.** He states the
|
|
60
|
+
whole thesis conditionally: *"if you can develop the right agentic loop AND have
|
|
61
|
+
the right eval or metric for the agents to optimize."* We have the loop. We do
|
|
62
|
+
not have the metric.
|
|
63
|
+
|
|
64
|
+
## 2. What "broken" means concretely
|
|
65
|
+
|
|
66
|
+
**Cost is not measured.** The efficiency schema has every field --
|
|
67
|
+
`cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`,
|
|
68
|
+
`cache_creation_tokens`, `model`, `duration_ms` -- and `cost_usd` is **0** on 3
|
|
69
|
+
of 5 records. We cannot answer "what did this issue cost?", which is:
|
|
70
|
+
|
|
71
|
+
- the axis with a measured **20x industry spread**
|
|
72
|
+
- the thing a buyer compares first
|
|
73
|
+
- unmeasurable *after* the fact, because token counts are per-call
|
|
74
|
+
|
|
75
|
+
**Our harness has no eval score.** `benchmarks/` contains a SWE-bench-lite
|
|
76
|
+
dataset and a results directory. There is no recorded score for the current
|
|
77
|
+
harness. So when the research says a harness is worth 10-15 points, **we cannot
|
|
78
|
+
say which side of that we are on.**
|
|
79
|
+
|
|
80
|
+
This is not a small omission. It is the difference between "we believe our
|
|
81
|
+
harness is good" and "our harness scores X, and here is the receipt."
|
|
82
|
+
|
|
83
|
+
## 3. The plan
|
|
84
|
+
|
|
85
|
+
Ordered by Wang's own weighting: the eval/metric loop first, because he makes
|
|
86
|
+
everything else conditional on it.
|
|
87
|
+
|
|
88
|
+
### W1 -- Make cost real (unblocks principle 5)
|
|
89
|
+
|
|
90
|
+
`cost_usd == 0` on most records while every input field is present. Fix the
|
|
91
|
+
computation, not the schema.
|
|
92
|
+
|
|
93
|
+
- Compute from `input/output/cache_read/cache_creation` x the model's price.
|
|
94
|
+
All four token counts are already recorded; the pricing table already exists
|
|
95
|
+
(`loki-ts/data/model-pricing.json`, cache tiers included).
|
|
96
|
+
- Emit **cost per iteration** and **cost per resolved issue** -- the second is
|
|
97
|
+
the one with the 20x spread.
|
|
98
|
+
- Guard direction: a missing price must record **unknown**, never 0. A zero is a
|
|
99
|
+
claim that the iteration was free, and it is the claim currently being made.
|
|
100
|
+
|
|
101
|
+
### W2 -- Score our own harness (completes principle 5)
|
|
102
|
+
|
|
103
|
+
`benchmarks/datasets/swebench-lite.json` exists and is unused for this.
|
|
104
|
+
|
|
105
|
+
- Run the harness against SWE-bench-lite and record the score with the
|
|
106
|
+
Evidence Receipt attached.
|
|
107
|
+
- Report **score AND cost-per-resolved-issue** together. Score alone is what
|
|
108
|
+
everyone publishes; the pair is what nobody does, and it is exactly our
|
|
109
|
+
Evidence Receipt moat applied to ourselves.
|
|
110
|
+
- Re-run per release: this becomes the regression signal for harness quality,
|
|
111
|
+
which today has none.
|
|
112
|
+
|
|
113
|
+
### W3 -- Attack the 980s (principle 2, speed)
|
|
114
|
+
|
|
115
|
+
Measured: the agent call is **96%** of an iteration. Gates are 44s. Every speed
|
|
116
|
+
knob shipped so far (v8.33.0-v8.35.0) targets the 4%.
|
|
117
|
+
|
|
118
|
+
- The lever is prompt size and cache discipline, not gate ordering. The
|
|
119
|
+
`[CACHE_BREAKPOINT]` split exists; cache reads price at 0.1x input.
|
|
120
|
+
- W1 is the precondition: without real cost numbers we cannot tell whether a
|
|
121
|
+
prompt change helped or hurt.
|
|
122
|
+
- **Do not ship more gate-latency work until this is measured.** It is
|
|
123
|
+
optimising 4% while 96% is unexamined.
|
|
124
|
+
|
|
125
|
+
### W4 -- Close the loop the metrics feed (principle 5, second half)
|
|
126
|
+
|
|
127
|
+
Wang's phrasing is *"the right agentic loop AND the right metric."* We have
|
|
128
|
+
`LOKI_INJECT_FINDINGS` (findings to next iteration) and `LOKI_AUTO_LEARNINGS`.
|
|
129
|
+
What we lack is the metric flowing back:
|
|
130
|
+
|
|
131
|
+
- Feed cost and iteration count into the run's own decisions, not just the
|
|
132
|
+
receipt. An agent that knows it is on iteration 3 of a budget behaves
|
|
133
|
+
differently from one that does not.
|
|
134
|
+
- This is the piece that turns telemetry into a *feedback loop* rather than a
|
|
135
|
+
report.
|
|
136
|
+
|
|
137
|
+
### W5 -- Systems thinking: stop shipping half-wired features (principle 1)
|
|
138
|
+
|
|
139
|
+
Not new work; a standing rule earned the hard way. This session found the same
|
|
140
|
+
shape repeatedly: a fact recorded but never rendered, a gate that never ran, a
|
|
141
|
+
detector never packaged, a valve that fired one iteration too late.
|
|
142
|
+
|
|
143
|
+
- Every new capability ships **recorded, rendered, acted on, and
|
|
144
|
+
mutation-proven** -- or it is not shipped.
|
|
145
|
+
- Already enforced by the 73-invariant detector. The rule is written down so it
|
|
146
|
+
survives.
|
|
147
|
+
|
|
148
|
+
## 4. What we do NOT do
|
|
149
|
+
|
|
150
|
+
- **No model training.** Cursor (custom MoE, ~250 tok/s) and Cognition
|
|
151
|
+
(SWE-1.6, ~950 tok/s) bought speed that way. Not reachable. The 10-15 point
|
|
152
|
+
harness delta is, and it is ours.
|
|
153
|
+
- **No more gate-latency work before W3 is measured.** 44s of gates against a
|
|
154
|
+
980s agent call.
|
|
155
|
+
- **No published score without its cost.** Score alone is the industry norm and
|
|
156
|
+
it is the half that flatters.
|
|
157
|
+
|
|
158
|
+
## 5. Acceptance
|
|
159
|
+
|
|
160
|
+
- `cost_usd` is non-zero and correct on **every** efficiency record, or
|
|
161
|
+
explicitly `unknown`
|
|
162
|
+
- a recorded SWE-bench-lite score for our harness, **with cost per resolved
|
|
163
|
+
issue**, regenerated per release
|
|
164
|
+
- a measured before/after on the 980s agent call
|
|
165
|
+
|
|
166
|
+
## 6. Why this is the edge
|
|
167
|
+
|
|
168
|
+
Wang's claim is conditional on the metric. The market's finding is that the
|
|
169
|
+
harness is worth 10-15 points. Our moat is the Evidence Receipt -- proof that a
|
|
170
|
+
result is real.
|
|
171
|
+
|
|
172
|
+
**Turn the receipt on ourselves**: publish a harness score *and* its cost, both
|
|
173
|
+
provable. Nobody in the category does this. It is the same trust argument that
|
|
174
|
+
sells the product, applied to the product's own claims.
|
|
175
|
+
|
|
176
|
+
Sources: 2026 SWE-bench harness comparisons (morphllm, codesota, awesomeagents),
|
|
177
|
+
agent-evaluation framework surveys 2026, Wang interviews (YC Startup Library,
|
|
178
|
+
CSIS), SlopCodeBench (arXiv 2603.24755), developer-agent misalignment study
|
|
179
|
+
(arXiv 2605.29442).
|
|
@@ -1,38 +1,56 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
2
|
+
"$schema_version": 1,
|
|
3
|
+
"_comment": "Rolling pricing table consumed by loki-ts/src/runner/budget.ts. Update this file when Anthropic / OpenAI / others publish new prices; no code change required. Pricing is USD per 1 million tokens. Aliases (opus/sonnet/haiku) point to the latest model of that family per providers/model_catalog.json.",
|
|
4
|
+
"_updated": "2026-06-30",
|
|
5
|
+
"_source": "https://www.anthropic.com/pricing + provider docs. sonnet=claude-sonnet-5 (list $3/$15; intro $2/$10 through 2026-08-31). opus=claude-opus-5. codex=gpt-5.3-codex standard tier. Cache tiers: read 0.1x input, write 1.25x input (Anthropic + OpenAI published multipliers).",
|
|
6
|
+
"pricing": {
|
|
7
|
+
"fable": {
|
|
8
|
+
"input": 10.0,
|
|
9
|
+
"output": 50.0,
|
|
10
|
+
"cache_read": 1.0,
|
|
11
|
+
"cache_write": 12.5
|
|
12
|
+
},
|
|
13
|
+
"opus": {
|
|
14
|
+
"input": 5.0,
|
|
15
|
+
"output": 25.0,
|
|
16
|
+
"cache_read": 0.5,
|
|
17
|
+
"cache_write": 6.25
|
|
18
|
+
},
|
|
19
|
+
"sonnet": {
|
|
20
|
+
"input": 3.0,
|
|
21
|
+
"output": 15.0,
|
|
22
|
+
"cache_read": 0.3,
|
|
23
|
+
"cache_write": 3.75
|
|
24
|
+
},
|
|
25
|
+
"haiku": {
|
|
26
|
+
"input": 1.0,
|
|
27
|
+
"output": 5.0,
|
|
28
|
+
"cache_read": 0.1,
|
|
29
|
+
"cache_write": 1.25
|
|
30
|
+
},
|
|
31
|
+
"gpt-5.3-codex": {
|
|
32
|
+
"input": 1.75,
|
|
33
|
+
"output": 14.0,
|
|
34
|
+
"cache_read": 0.175,
|
|
35
|
+
"cache_write": 2.1875
|
|
36
|
+
},
|
|
37
|
+
"gpt-5.6-sol": {
|
|
38
|
+
"input": 1.75,
|
|
39
|
+
"output": 14.0,
|
|
40
|
+
"cache_read": 0.175,
|
|
41
|
+
"cache_write": 2.1875
|
|
42
|
+
},
|
|
43
|
+
"gpt-5.6-terra": {
|
|
44
|
+
"input": 0.6,
|
|
45
|
+
"output": 4.8,
|
|
46
|
+
"cache_read": 0.06,
|
|
47
|
+
"cache_write": 0.75
|
|
48
|
+
},
|
|
49
|
+
"gpt-5.6-luna": {
|
|
50
|
+
"input": 0.15,
|
|
51
|
+
"output": 1.2,
|
|
52
|
+
"cache_read": 0.015,
|
|
53
|
+
"cache_write": 0.1875
|
|
54
|
+
}
|
|
36
55
|
}
|
|
37
|
-
}
|
|
38
56
|
}
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var h_=Object.create;var{getPrototypeOf:v_,defineProperty:rK,getOwnPropertyNames:g_}=Object;var m_=Object.prototype.hasOwnProperty;function u_(Z){return this[Z]}var p_,d_,c_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?p_??=new WeakMap:d_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?h_(v_(Z)):{};let K=X||!Z||!Z.__esModule?rK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of g_(Z))if(!m_.call(K,$))rK(K,$,{get:u_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var qQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var l_=(Z)=>Z;function i_(Z,X){this[Z]=l_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)rK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:i_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>P4,findRepoRootForVersion:()=>eK,REPO_ROOT:()=>r0});import{resolve as n7,dirname as tK}from"path";import{fileURLToPath as a_}from"url";import{existsSync as GQ}from"fs";import{homedir as s_}from"os";function n_(){let Z=RO;for(let X=0;X<6;X++){if(GQ(n7(Z,"VERSION"))&&GQ(n7(Z,"autonomy/run.sh")))return Z;let Q=tK(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function eK(Z){let X=Z;for(let Q=0;Q<6;Q++){if(GQ(n7(X,"VERSION"))&&GQ(n7(X,"autonomy/run.sh")))return X;let Y=tK(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function P4(){return n7(s_(),".loki")}var RO,r0;var G8=p(()=>{RO=tK(a_(import.meta.url));r0=n_()});import{readFileSync as o_}from"fs";import{resolve as r_,dirname as t_}from"path";import{fileURLToPath as e_}from"url";function f3(){if(h5!==null)return h5;let Z="8.
|
|
2
|
+
var h_=Object.create;var{getPrototypeOf:v_,defineProperty:rK,getOwnPropertyNames:g_}=Object;var m_=Object.prototype.hasOwnProperty;function u_(Z){return this[Z]}var p_,d_,c_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?p_??=new WeakMap:d_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?h_(v_(Z)):{};let K=X||!Z||!Z.__esModule?rK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of g_(Z))if(!m_.call(K,$))rK(K,$,{get:u_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var qQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var l_=(Z)=>Z;function i_(Z,X){this[Z]=l_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)rK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:i_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>P4,findRepoRootForVersion:()=>eK,REPO_ROOT:()=>r0});import{resolve as n7,dirname as tK}from"path";import{fileURLToPath as a_}from"url";import{existsSync as GQ}from"fs";import{homedir as s_}from"os";function n_(){let Z=RO;for(let X=0;X<6;X++){if(GQ(n7(Z,"VERSION"))&&GQ(n7(Z,"autonomy/run.sh")))return Z;let Q=tK(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function eK(Z){let X=Z;for(let Q=0;Q<6;Q++){if(GQ(n7(X,"VERSION"))&&GQ(n7(X,"autonomy/run.sh")))return X;let Y=tK(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function P4(){return n7(s_(),".loki")}var RO,r0;var G8=p(()=>{RO=tK(a_(import.meta.url));r0=n_()});import{readFileSync as o_}from"fs";import{resolve as r_,dirname as t_}from"path";import{fileURLToPath as e_}from"url";function f3(){if(h5!==null)return h5;let Z="8.51.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=t_(e_(import.meta.url)),Q=eK(X);h5=o_(r_(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var HQ=p(()=>{G8()});var bO={};l0(bO,{runOrThrow:()=>Of,run:()=>E0,readStreamCapped:()=>UQ,commandVersion:()=>Af,commandExists:()=>X9,ShellError:()=>Z$,MAX_STDOUT_BYTES:()=>yO});async function UQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([UQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Of(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Z$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Lf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Lf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Af(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Z$;var x9=p(()=>{Z$=class Z$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return jf?"":Z}var jf,L0,k8,p0,ZV0,i0,H8,Q9,v;var S6=p(()=>{jf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),k8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),ZV0=o7("\x1B[0;34m"),i0=o7("\x1B[0;36m"),H8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as xf}from"fs";async function E7(){if(k4!==void 0)return k4;let Z="/opt/homebrew/bin/python3.12";if(xf(Z))return k4=Z,Z;let X=await X9("python3.12");if(X)return k4=X,X;let Q=await X9("python3");return k4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var k4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>Yh});import{existsSync as Y9,readFileSync as v3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as sf}from"path";import{homedir as nf}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*NQ/X);if(J>NQ)J=NQ;let z=NQ-J,K=k8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${H8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function rf(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1227,4 +1227,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1227
1227
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (b_(),y_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1228
1228
|
`),process.stderr.write(__),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var fW0=await _W0(Bun.argv.slice(2));process.exit(fW0);
|
|
1229
1229
|
|
|
1230
|
-
//# debugId=
|
|
1230
|
+
//# debugId=A91F4A3644413E8F64756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "8.
|
|
4
|
+
"version": "8.51.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "8.
|
|
5
|
+
"version": "8.51.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|