loki-mode 9.0.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/estimate-run.py +14 -2
- package/tools/receipt-find.py +329 -0
- package/tools/token-guard.py +218 -0
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v9.
|
|
6
|
+
# Loki Mode v9.2.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -469,4 +469,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
469
469
|
|
|
470
470
|
---
|
|
471
471
|
|
|
472
|
-
**v9.
|
|
472
|
+
**v9.2.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
9.
|
|
1
|
+
9.2.0
|
package/dashboard/__init__.py
CHANGED
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="9.
|
|
2
|
+
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="9.2.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Xf(Qf(import.meta.url)),Q=X$(X);h5=e_(Zf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var bO={};l0(bO,{runOrThrow:()=>jf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Tf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>yO});async function NQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function jf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Mf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Mf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Tf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return wf?"":Z}var wf,L0,F8,p0,zV0,a0,W8,Q9,v;var S6=p(()=>{wf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),zV0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),W8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as bf}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(bf(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>Kh});import{existsSync as Y9,readFileSync as g3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as rf}from"path";import{homedir as tf}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*LQ/X);if(J>LQ)J=LQ;let z=LQ-J,K=F8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${W8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function Zh(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1232,4 +1232,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1232
1232
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (h_(),f_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1233
1233
|
`),process.stderr.write(v_),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var uW0=await mW0(Bun.argv.slice(2));process.exit(uW0);
|
|
1234
1234
|
|
|
1235
|
-
//# debugId=
|
|
1235
|
+
//# debugId=F7F47FC30C33C44064756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "9.
|
|
4
|
+
"version": "9.2.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "9.
|
|
5
|
+
"version": "9.2.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
package/tools/estimate-run.py
CHANGED
|
@@ -332,13 +332,25 @@ def main(argv=None):
|
|
|
332
332
|
ap.add_argument("--json", action="store_true")
|
|
333
333
|
args = ap.parse_args(argv)
|
|
334
334
|
|
|
335
|
+
# A workspace that does not EXIST is not the same fact as one with no cost
|
|
336
|
+
# history, and exit 0 collapsed them. preflight.sh consumes this tool, so a
|
|
337
|
+
# mistyped or unmounted path silently became "no basis to project from"
|
|
338
|
+
# instead of an error. Same defect fixed in model-advisor.py in v8.96.0;
|
|
339
|
+
# tests/test_tool_exit_contract.py now catches the class rather than the
|
|
340
|
+
# instance.
|
|
341
|
+
if not os.path.isdir(args.workspace):
|
|
342
|
+
sys.stderr.write(
|
|
343
|
+
"cannot estimate: workspace does not exist: %s\n" % args.workspace)
|
|
344
|
+
return 66
|
|
345
|
+
|
|
335
346
|
est = estimate(args.workspace, args.iterations)
|
|
336
347
|
if args.json:
|
|
337
348
|
print(json.dumps(est, indent=2))
|
|
338
349
|
else:
|
|
339
350
|
print(render(est))
|
|
340
|
-
# Exit 0
|
|
341
|
-
# failure. Callers read
|
|
351
|
+
# Exit 0 for a REAL workspace with no basis: that is a successful, honest
|
|
352
|
+
# answer rather than a tool failure, and the output says so. Callers read
|
|
353
|
+
# has_basis.
|
|
342
354
|
return 0
|
|
343
355
|
|
|
344
356
|
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Find the receipts an auditor actually needs, out of a workspace full of them.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. receipt-bundle.py verifies EVERY receipt under a workspace and
|
|
5
|
+
rolls them into one verdict. That is the right artifact to hand a compliance
|
|
6
|
+
reviewer, and the wrong one to work from. An auditor with three hundred runs is
|
|
7
|
+
not asking "is the archive sound"; they are asking "which run cost $40", "which
|
|
8
|
+
ones did not verify", "what has happened since the incident on the 12th".
|
|
9
|
+
Nothing queried receipts. This does, and the whole design problem is that a
|
|
10
|
+
filter is a claim about what it did NOT return.
|
|
11
|
+
|
|
12
|
+
THE RULES. Each is a specific way a result set can claim more than it earned.
|
|
13
|
+
|
|
14
|
+
1. AN UNMEASURED COST MATCHES NO NUMERIC FILTER, IN EITHER DIRECTION. A receipt
|
|
15
|
+
that never recorded cost is not "under $5" and it is not "over $5" -- the
|
|
16
|
+
comparison is undefined, and answering it either way invents a measurement.
|
|
17
|
+
Treating unmeasured as 0.0 is the exact lie the cost-honesty line exists to
|
|
18
|
+
prevent (v8.51.0-v8.54.0 fixed it on four surfaces); it would make every
|
|
19
|
+
unmeasured run answer "cheapest in the archive". So unmeasured receipts are
|
|
20
|
+
EXCLUDED from a cost filter and the exclusion is COUNTED and REPORTED. An
|
|
21
|
+
auditor who asked for runs over $5 must be told the filter could not
|
|
22
|
+
consider N of them, or they will read the result as exhaustive.
|
|
23
|
+
|
|
24
|
+
The predicate is record_is_measured() in autonomy/lib/efficiency_cost.py,
|
|
25
|
+
reached through receipt-diff.py's measured_cost(), which already maps the
|
|
26
|
+
receipt's `cost.usd` onto the per-iteration `cost_usd` key it expects. A
|
|
27
|
+
second copy of that predicate is how the honesty rule drifts, so there is
|
|
28
|
+
not one here.
|
|
29
|
+
|
|
30
|
+
Measured is necessary but NOT sufficient for a cost comparison:
|
|
31
|
+
record_is_measured is true when ANY of five fields is non-zero, so a receipt
|
|
32
|
+
with tokens but a null `usd` is honestly measured and still has no dollar
|
|
33
|
+
figure. Both conditions are required -- see _usd().
|
|
34
|
+
|
|
35
|
+
2. ZERO MATCHES IS A RESULT; ZERO RECEIPTS IS NOT THE SAME FACT. "The filter
|
|
36
|
+
ran and nothing qualified" and "there was nothing to search" look identical
|
|
37
|
+
in a list of zero lines, and they mean opposite things: one is a clean
|
|
38
|
+
answer, the other is a broken invocation pointed at the wrong directory.
|
|
39
|
+
They get different summaries and different exit codes.
|
|
40
|
+
|
|
41
|
+
3. A MALFORMED RECEIPT IS COUNTED AND NAMED, NEVER SILENTLY SKIPPED. A search
|
|
42
|
+
that drops what it cannot read reports a tidier archive than exists, and
|
|
43
|
+
does it invisibly -- the reader sees only files that parsed and cannot tell
|
|
44
|
+
whether the rest were irrelevant or unreadable. Same rule as the bundle's:
|
|
45
|
+
absent is not clean.
|
|
46
|
+
|
|
47
|
+
4. A BAD --since IS REJECTED, NOT GUESSED. An unparseable date that silently
|
|
48
|
+
matches everything (or nothing) returns a result set with no relation to the
|
|
49
|
+
question asked, and looks exactly like a correct one.
|
|
50
|
+
|
|
51
|
+
5. FILTERS ARE AND, AND THE OUTPUT SAYS WHICH WERE APPLIED. A result set with
|
|
52
|
+
no visible filter line can be pasted anywhere and read as "all receipts".
|
|
53
|
+
|
|
54
|
+
Usage:
|
|
55
|
+
tools/receipt-find.py [workspace] [--min-usd N] [--max-usd N]
|
|
56
|
+
[--failed-only] [--since YYYY-MM-DD] [--json]
|
|
57
|
+
|
|
58
|
+
Exit codes:
|
|
59
|
+
0 at least one receipt matched
|
|
60
|
+
1 receipts were searched, none matched (a valid result, not an error)
|
|
61
|
+
3 no receipts found at all -- nothing was searched
|
|
62
|
+
2 usage error (argparse; e.g. an unparseable --since)
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
import argparse
|
|
66
|
+
import datetime
|
|
67
|
+
import importlib.util
|
|
68
|
+
import json
|
|
69
|
+
import pathlib
|
|
70
|
+
import sys
|
|
71
|
+
|
|
72
|
+
# A stale .pyc can mask a mutation and turn a real probe into a false
|
|
73
|
+
# "MUTATION SURVIVED", since invalidation is mtime+size and a restore is
|
|
74
|
+
# byte-identical. Set before the loader below runs.
|
|
75
|
+
sys.dont_write_bytecode = True
|
|
76
|
+
|
|
77
|
+
_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _load(name, path):
|
|
81
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
82
|
+
mod = importlib.util.module_from_spec(spec)
|
|
83
|
+
spec.loader.exec_module(mod)
|
|
84
|
+
return mod
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# measured_cost() reuses record_is_measured() AND maps cost.usd -> cost_usd.
|
|
88
|
+
# Both halves of rule 1 already live there; this file must not restate either.
|
|
89
|
+
_rd = _load("receipt_diff", _ROOT / "tools" / "receipt-diff.py")
|
|
90
|
+
measured_cost = _rd.measured_cost
|
|
91
|
+
|
|
92
|
+
# The headline a receipt carries when the run did not verify. "VERIFIED WITH
|
|
93
|
+
# GAPS" is deliberately NOT failed -- it verified, with recorded gaps -- and a
|
|
94
|
+
# receipt with no headline at all is not silently treated as fine: it is
|
|
95
|
+
# UNKNOWN, reported in its own bucket, and never matched by --failed-only,
|
|
96
|
+
# because "we cannot tell" must not read as "it passed".
|
|
97
|
+
FAILED_HEADLINE = "NOT VERIFIED"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _usd(proof):
|
|
101
|
+
"""The receipt's cost in dollars, or None when there is no such number.
|
|
102
|
+
|
|
103
|
+
None means the comparison is UNDEFINED, and every caller must skip rather
|
|
104
|
+
than substitute a value. Returning 0.0 here would make an unmeasured run
|
|
105
|
+
the cheapest in the archive and match every --max-usd -- rule 1, and the
|
|
106
|
+
single place it is decided.
|
|
107
|
+
"""
|
|
108
|
+
rec = measured_cost(proof)
|
|
109
|
+
if rec is None:
|
|
110
|
+
return None
|
|
111
|
+
return rec.get("cost_usd")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _headline(proof):
|
|
115
|
+
honesty = proof.get("honesty")
|
|
116
|
+
if not isinstance(honesty, dict):
|
|
117
|
+
return None
|
|
118
|
+
h = honesty.get("headline")
|
|
119
|
+
return h if isinstance(h, str) and h else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _date(proof):
|
|
123
|
+
"""The receipt's UTC date as YYYY-MM-DD, or None.
|
|
124
|
+
|
|
125
|
+
Sliced, not parsed. generated_at ends in "Z", which datetime.fromisoformat
|
|
126
|
+
rejects before Python 3.11, and this tool must behave identically on both
|
|
127
|
+
interpreters. ISO dates compare correctly as strings, so a slice is both
|
|
128
|
+
the portable answer and the smaller one.
|
|
129
|
+
"""
|
|
130
|
+
v = proof.get("generated_at")
|
|
131
|
+
if not isinstance(v, str) or len(v) < 10:
|
|
132
|
+
return None
|
|
133
|
+
d = v[:10]
|
|
134
|
+
try:
|
|
135
|
+
datetime.date.fromisoformat(d)
|
|
136
|
+
except ValueError:
|
|
137
|
+
return None
|
|
138
|
+
return d
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def find_receipts(workspace):
|
|
142
|
+
"""Every proof.json under the workspace, sorted for a stable report.
|
|
143
|
+
|
|
144
|
+
ponytail: same rglob as receipt-bundle.find_receipts, so a receipt archived
|
|
145
|
+
outside .loki/proofs/ is still findable. Deliberately not imported --
|
|
146
|
+
receipt-bundle loads proof-verify and runs git per receipt, and this tool
|
|
147
|
+
reads JSON only.
|
|
148
|
+
"""
|
|
149
|
+
root = pathlib.Path(workspace)
|
|
150
|
+
if not root.is_dir():
|
|
151
|
+
return []
|
|
152
|
+
return sorted(p for p in root.rglob("proof.json") if p.is_file())
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def search(workspace, min_usd=None, max_usd=None, failed_only=False,
|
|
156
|
+
since=None):
|
|
157
|
+
"""Filter receipts under `workspace`. Pure: no writes, no network.
|
|
158
|
+
|
|
159
|
+
Filters AND together (rule 5). A receipt excluded by the cost filter for
|
|
160
|
+
being unmeasured is counted in `excluded_unmeasured` -- but only when a
|
|
161
|
+
cost filter was actually applied, since without one nothing excluded it and
|
|
162
|
+
reporting an exclusion would imply a filter it never faced.
|
|
163
|
+
"""
|
|
164
|
+
cost_filtered = min_usd is not None or max_usd is not None
|
|
165
|
+
paths = find_receipts(workspace)
|
|
166
|
+
|
|
167
|
+
matches = []
|
|
168
|
+
malformed = []
|
|
169
|
+
excluded_unmeasured = 0
|
|
170
|
+
|
|
171
|
+
for path in paths:
|
|
172
|
+
try:
|
|
173
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
174
|
+
proof = json.load(f)
|
|
175
|
+
if not isinstance(proof, dict):
|
|
176
|
+
raise ValueError("receipt is not a JSON object")
|
|
177
|
+
except Exception as exc:
|
|
178
|
+
# Rule 3: counted and named, with the reason. Never skipped.
|
|
179
|
+
malformed.append({"path": str(path), "reason": str(exc)})
|
|
180
|
+
continue
|
|
181
|
+
|
|
182
|
+
why = []
|
|
183
|
+
|
|
184
|
+
if cost_filtered:
|
|
185
|
+
usd = _usd(proof)
|
|
186
|
+
if usd is None:
|
|
187
|
+
# Rule 1: undefined in BOTH directions. Not below, not above.
|
|
188
|
+
excluded_unmeasured += 1
|
|
189
|
+
continue
|
|
190
|
+
if min_usd is not None and usd < min_usd:
|
|
191
|
+
continue
|
|
192
|
+
if max_usd is not None and usd > max_usd:
|
|
193
|
+
continue
|
|
194
|
+
why.append("cost.usd=%s" % usd)
|
|
195
|
+
|
|
196
|
+
headline = _headline(proof)
|
|
197
|
+
if failed_only:
|
|
198
|
+
if headline != FAILED_HEADLINE:
|
|
199
|
+
continue
|
|
200
|
+
why.append("honesty.headline=%s" % headline)
|
|
201
|
+
|
|
202
|
+
date = _date(proof)
|
|
203
|
+
if since is not None:
|
|
204
|
+
if date is None or date < since:
|
|
205
|
+
continue
|
|
206
|
+
why.append("generated_at=%s" % date)
|
|
207
|
+
|
|
208
|
+
matches.append({
|
|
209
|
+
"path": str(path),
|
|
210
|
+
"matched": why,
|
|
211
|
+
"cost_usd": _usd(proof),
|
|
212
|
+
"headline": headline,
|
|
213
|
+
"date": date,
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
return {
|
|
217
|
+
"report": "loki-receipt-find/v1",
|
|
218
|
+
"workspace": str(workspace),
|
|
219
|
+
"filters": _filters(min_usd, max_usd, failed_only, since),
|
|
220
|
+
"matches": matches,
|
|
221
|
+
"match_count": len(matches),
|
|
222
|
+
"scanned": len(paths),
|
|
223
|
+
"malformed": malformed,
|
|
224
|
+
"malformed_count": len(malformed),
|
|
225
|
+
"excluded_unmeasured": excluded_unmeasured,
|
|
226
|
+
"summary": _summary(len(paths), len(matches), len(malformed),
|
|
227
|
+
excluded_unmeasured,
|
|
228
|
+
_filters(min_usd, max_usd, failed_only, since)),
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _filters(min_usd, max_usd, failed_only, since):
|
|
233
|
+
"""The applied filters, in the order they are applied.
|
|
234
|
+
|
|
235
|
+
`is not None`, never truthiness: --min-usd 0 is a real filter, and reading
|
|
236
|
+
it as "no filter" would make the output line state something untrue.
|
|
237
|
+
"""
|
|
238
|
+
f = []
|
|
239
|
+
if min_usd is not None:
|
|
240
|
+
f.append("min-usd=%s" % min_usd)
|
|
241
|
+
if max_usd is not None:
|
|
242
|
+
f.append("max-usd=%s" % max_usd)
|
|
243
|
+
if failed_only:
|
|
244
|
+
f.append("failed-only")
|
|
245
|
+
if since is not None:
|
|
246
|
+
f.append("since=%s" % since)
|
|
247
|
+
return f
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _summary(scanned, matched, malformed, excluded, filters):
|
|
251
|
+
applied = ", ".join(filters) if filters else "none (no filter applied)"
|
|
252
|
+
if scanned == 0:
|
|
253
|
+
return ("NO RECEIPTS -- no proof.json found under this workspace, so "
|
|
254
|
+
"nothing was searched. This is not the same as zero matches.")
|
|
255
|
+
if matched == 0:
|
|
256
|
+
# "found", not "searched": a malformed receipt is FOUND and counted,
|
|
257
|
+
# and the malformed line below says it was NOT searched. Saying
|
|
258
|
+
# "searched" here would contradict it in the same sentence.
|
|
259
|
+
head = ("NO MATCHES -- %d receipt(s) found, none matched. The "
|
|
260
|
+
"filter ran; the archive is not empty." % scanned)
|
|
261
|
+
else:
|
|
262
|
+
head = "%d of %d receipt(s) matched." % (matched, scanned)
|
|
263
|
+
head += " Filters applied: %s." % applied
|
|
264
|
+
if excluded:
|
|
265
|
+
head += (" %d receipt(s) EXCLUDED from the cost filter: cost was never "
|
|
266
|
+
"measured, so it is neither above nor below the threshold."
|
|
267
|
+
% excluded)
|
|
268
|
+
if malformed:
|
|
269
|
+
head += (" %d receipt(s) could not be read and were NOT searched."
|
|
270
|
+
% malformed)
|
|
271
|
+
return head
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _render(report):
|
|
275
|
+
lines = []
|
|
276
|
+
for m in report["matches"]:
|
|
277
|
+
lines.append("%s [%s]" % (m["path"], ", ".join(m["matched"]) or "-"))
|
|
278
|
+
for bad in report["malformed"]:
|
|
279
|
+
lines.append("MALFORMED %s (%s)" % (bad["path"], bad["reason"]))
|
|
280
|
+
lines.append("")
|
|
281
|
+
lines.append(report["summary"])
|
|
282
|
+
return "\n".join(lines)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _since(value):
|
|
286
|
+
"""Rule 4: an unparseable --since is rejected, never guessed."""
|
|
287
|
+
try:
|
|
288
|
+
return datetime.date.fromisoformat(value).isoformat()
|
|
289
|
+
except ValueError:
|
|
290
|
+
raise argparse.ArgumentTypeError(
|
|
291
|
+
"not a YYYY-MM-DD date: %r. A date that cannot be parsed would "
|
|
292
|
+
"silently match everything or nothing." % value)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def main(argv=None):
|
|
296
|
+
ap = argparse.ArgumentParser(
|
|
297
|
+
description="Find receipts under a workspace by measurable criteria.")
|
|
298
|
+
ap.add_argument("workspace", nargs="?", default=".",
|
|
299
|
+
help="workspace to search for receipts (default: .)")
|
|
300
|
+
ap.add_argument("--min-usd", type=float, default=None,
|
|
301
|
+
help="only receipts whose measured cost is >= N")
|
|
302
|
+
ap.add_argument("--max-usd", type=float, default=None,
|
|
303
|
+
help="only receipts whose measured cost is <= N")
|
|
304
|
+
ap.add_argument("--failed-only", action="store_true",
|
|
305
|
+
help='only receipts whose headline is "%s"'
|
|
306
|
+
% FAILED_HEADLINE)
|
|
307
|
+
ap.add_argument("--since", type=_since, default=None,
|
|
308
|
+
metavar="YYYY-MM-DD",
|
|
309
|
+
help="only receipts generated on or after this UTC date")
|
|
310
|
+
ap.add_argument("--json", action="store_true",
|
|
311
|
+
help="emit the full report as JSON")
|
|
312
|
+
args = ap.parse_args(argv)
|
|
313
|
+
|
|
314
|
+
report = search(args.workspace, min_usd=args.min_usd,
|
|
315
|
+
max_usd=args.max_usd, failed_only=args.failed_only,
|
|
316
|
+
since=args.since)
|
|
317
|
+
|
|
318
|
+
if args.json:
|
|
319
|
+
print(json.dumps(report, indent=2))
|
|
320
|
+
else:
|
|
321
|
+
print(_render(report))
|
|
322
|
+
|
|
323
|
+
if report["scanned"] == 0:
|
|
324
|
+
return 3
|
|
325
|
+
return 0 if report["match_count"] else 1
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
if __name__ == "__main__":
|
|
329
|
+
sys.exit(main())
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fail a build when a run's TOKEN usage regressed. A token gate for CI.
|
|
3
|
+
|
|
4
|
+
tools/cost-guard.py gates on dollars. Dollars are a derived unit: they depend
|
|
5
|
+
on a per-provider price table that changes under you, so a fixed dollar
|
|
6
|
+
ceiling silently LOOSENS the moment a cheaper model is swapped in and tightens
|
|
7
|
+
on a dearer one, without the work having changed at all. Tokens are the
|
|
8
|
+
provider-independent unit of work actually done. Nothing gated on them.
|
|
9
|
+
|
|
10
|
+
THE RULE THAT MAKES IT A GATE RATHER THAN DECORATION:
|
|
11
|
+
|
|
12
|
+
UNMEASURED IS NOT WITHIN BUDGET. Exit 2, never 0.
|
|
13
|
+
|
|
14
|
+
A gate's exit code is a merge decision, and "we did not measure" carries zero
|
|
15
|
+
information about budget compliance. Exit 0 is reachable from exactly ONE
|
|
16
|
+
place in this file (the end of evaluate()), and only downstream of a real
|
|
17
|
+
measurement. No policy, no records, unmeasured tokens: all 2.
|
|
18
|
+
|
|
19
|
+
WHICH FIELDS COUNT AS MEASURED, and why it is not cost-guard's five.
|
|
20
|
+
Whether a number is measured is record_is_measured() in
|
|
21
|
+
autonomy/lib/efficiency_cost.py, imported and never restated. But that
|
|
22
|
+
predicate is satisfied by cost OR tokens, and this gate is about tokens, so it
|
|
23
|
+
is fed the FOUR TOKEN FIELDS ONLY. Feeding it cost_usd as well would be a hole
|
|
24
|
+
in this specific gate: a record carrying a real cost and no usage (the exact
|
|
25
|
+
shape codex wrote before v8.51.0) makes collect_efficiency return
|
|
26
|
+
available=True with integer-zero token counts, and this gate would report
|
|
27
|
+
"WITHIN BUDGET: 0 output tokens" for a run whose tokens were never recorded.
|
|
28
|
+
Verified, not theoretical. Choosing which fields to ask about is not a second
|
|
29
|
+
copy of the rule; the rule itself stays in one place.
|
|
30
|
+
|
|
31
|
+
CACHED READS ARE NOT FRESH INPUT, and conflating them is the defect to avoid.
|
|
32
|
+
This repo measured a single call at 10,651,759 cache-read tokens against
|
|
33
|
+
34,729 output tokens: a ratio near 300 to 1. A "total" dominated by cache
|
|
34
|
+
reads is a CONTEXT signal, and it moves when the prompt prefix or the cache
|
|
35
|
+
hit rate changes, not when the agent does more work. So:
|
|
36
|
+
|
|
37
|
+
--max-output-tokens the WORK signal. Output tokens alone: what the model
|
|
38
|
+
actually generated. Reach for this by default.
|
|
39
|
+
--max-total-tokens the CONTEXT+work signal. Sums the four fields named in
|
|
40
|
+
TOTAL_FIELDS, cache reads included, which is what makes
|
|
41
|
+
it big. The exact field list is PRINTED with the
|
|
42
|
+
verdict, on pass, on fail and in JSON, because a total
|
|
43
|
+
whose definition is not stated is not evidence.
|
|
44
|
+
|
|
45
|
+
Both may be given; each is judged on its own number and either can breach.
|
|
46
|
+
|
|
47
|
+
A genuinely measured ZERO under budget PASSES. Zero is falsy, so every guard
|
|
48
|
+
here is an explicit `is None` check -- on the measurement AND on the policy
|
|
49
|
+
arguments, since `--max-output-tokens 0` is a legitimate (harsh) policy and a
|
|
50
|
+
falsy check would read it as no policy at all.
|
|
51
|
+
|
|
52
|
+
Usage:
|
|
53
|
+
tools/token-guard.py [workspace] --max-output-tokens 200000
|
|
54
|
+
tools/token-guard.py [workspace] --max-total-tokens 5000000
|
|
55
|
+
tools/token-guard.py [workspace] --max-output-tokens 200000 --json
|
|
56
|
+
|
|
57
|
+
Exit: 0 within budget, 1 over budget, 2 cannot evaluate.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
import argparse
|
|
61
|
+
import json
|
|
62
|
+
import os
|
|
63
|
+
import sys
|
|
64
|
+
|
|
65
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
66
|
+
sys.path.insert(0, os.path.join(os.path.dirname(_HERE), "autonomy", "lib"))
|
|
67
|
+
|
|
68
|
+
from efficiency_cost import collect_efficiency, record_is_measured # noqa: E402
|
|
69
|
+
|
|
70
|
+
OK, OVER, CANNOT = 0, 1, 2
|
|
71
|
+
|
|
72
|
+
# The four fields --max-total-tokens sums, and the SAME tuple the printed
|
|
73
|
+
# definition is built from. One constant so the number and its stated meaning
|
|
74
|
+
# cannot drift apart: a total that sums something other than what it claims is
|
|
75
|
+
# the dishonesty this flag is most exposed to.
|
|
76
|
+
TOTAL_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens",
|
|
77
|
+
"cache_creation_tokens")
|
|
78
|
+
|
|
79
|
+
# Measured-ness for a TOKEN gate. cost_usd is deliberately absent; see the
|
|
80
|
+
# module docstring.
|
|
81
|
+
MEASURED_FIELDS = TOTAL_FIELDS
|
|
82
|
+
|
|
83
|
+
TOTAL_DEFINITION = "total = " + " + ".join(TOTAL_FIELDS)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _num(v):
|
|
87
|
+
"""A number as itself; None, "", or a bool as None.
|
|
88
|
+
|
|
89
|
+
The bool exclusion is defence in depth ONLY, and does not reach the CLI
|
|
90
|
+
path: collect_efficiency() runs _to_int() over every field first, so a
|
|
91
|
+
JSON `true` is already an int 1 by the time it arrives here. Stated
|
|
92
|
+
because an untrue claim of bool-safety is worse than a known gap; the fix
|
|
93
|
+
belongs in _to_int(), upstream. This guard still holds for a cost block
|
|
94
|
+
handed to measured_tokens() directly.
|
|
95
|
+
"""
|
|
96
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
97
|
+
return None
|
|
98
|
+
return v
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def measured_tokens(cost):
|
|
102
|
+
"""{field: value} for the four token fields, or None when unmeasured.
|
|
103
|
+
|
|
104
|
+
None means the run's tokens were never observed. It is NOT zero, and the
|
|
105
|
+
caller must not treat it as zero.
|
|
106
|
+
"""
|
|
107
|
+
if not isinstance(cost, dict):
|
|
108
|
+
return None
|
|
109
|
+
rec = {field: _num(cost.get(field)) for field in MEASURED_FIELDS}
|
|
110
|
+
if not record_is_measured(rec):
|
|
111
|
+
return None
|
|
112
|
+
# Measured, so a None from a partially-written record is a real absence in
|
|
113
|
+
# a real measurement; count it as 0 for summing rather than poisoning the
|
|
114
|
+
# arithmetic.
|
|
115
|
+
return {field: (0 if rec[field] is None else rec[field])
|
|
116
|
+
for field in MEASURED_FIELDS}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _loki_dir(workspace):
|
|
120
|
+
"""Accept either a workspace root or a .loki dir; collect_ wants .loki."""
|
|
121
|
+
if os.path.basename(os.path.normpath(workspace)) == ".loki":
|
|
122
|
+
return workspace
|
|
123
|
+
return os.path.join(workspace, ".loki")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _cannot(why):
|
|
127
|
+
return {"status": "cannot_evaluate", "exit_code": CANNOT, "why": why,
|
|
128
|
+
"output_tokens": None, "total_tokens": None, "tokens": None,
|
|
129
|
+
"total_definition": TOTAL_DEFINITION, "reason": None}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def evaluate(workspace, max_output_tokens=None, max_total_tokens=None):
|
|
133
|
+
"""Decide the gate. Returns a dict carrying exit_code and the numbers.
|
|
134
|
+
|
|
135
|
+
Every early return is CANNOT (2). The single exit-0 return is at the very
|
|
136
|
+
bottom, downstream of a measurement.
|
|
137
|
+
"""
|
|
138
|
+
# `is None`, not falsy: --max-output-tokens 0 is a real policy.
|
|
139
|
+
if max_output_tokens is None and max_total_tokens is None:
|
|
140
|
+
return _cannot(
|
|
141
|
+
"no token policy given: pass --max-output-tokens and/or "
|
|
142
|
+
"--max-total-tokens. A gate with no policy checks nothing, and "
|
|
143
|
+
"must not report a pass for it.")
|
|
144
|
+
|
|
145
|
+
cost, _model = collect_efficiency(_loki_dir(workspace))
|
|
146
|
+
tokens = measured_tokens(cost)
|
|
147
|
+
if tokens is None:
|
|
148
|
+
# THE POINT OF THIS FILE. Absence is not compliance.
|
|
149
|
+
return _cannot(
|
|
150
|
+
"tokens are UNMEASURED for %s -- no efficiency record carried an "
|
|
151
|
+
"observed token count. Unmeasured is not within budget: this gate "
|
|
152
|
+
"cannot say whether the run complied, so it reports no verdict "
|
|
153
|
+
"rather than a green one." % workspace)
|
|
154
|
+
|
|
155
|
+
output = tokens["output_tokens"]
|
|
156
|
+
total = sum(tokens[field] for field in TOTAL_FIELDS)
|
|
157
|
+
|
|
158
|
+
breaches = []
|
|
159
|
+
# Tokens are integers, so a strict `>` is exact. No float tolerance: there
|
|
160
|
+
# is no binary-representation slop to absorb, and a tolerance here would
|
|
161
|
+
# silently widen the ceiling the operator asked for.
|
|
162
|
+
if max_output_tokens is not None and output > max_output_tokens:
|
|
163
|
+
breaches.append(
|
|
164
|
+
"output tokens %d exceed the ceiling %d by %d (%.1f%% over)"
|
|
165
|
+
% (output, max_output_tokens, output - max_output_tokens,
|
|
166
|
+
((output - max_output_tokens) / max_output_tokens * 100.0)
|
|
167
|
+
if max_output_tokens else 0.0))
|
|
168
|
+
|
|
169
|
+
if max_total_tokens is not None and total > max_total_tokens:
|
|
170
|
+
breaches.append(
|
|
171
|
+
"total tokens %d exceed the ceiling %d by %d (%.1f%% over) [%s]"
|
|
172
|
+
% (total, max_total_tokens, total - max_total_tokens,
|
|
173
|
+
((total - max_total_tokens) / max_total_tokens * 100.0)
|
|
174
|
+
if max_total_tokens else 0.0, TOTAL_DEFINITION))
|
|
175
|
+
|
|
176
|
+
verdict = {"output_tokens": output, "total_tokens": total,
|
|
177
|
+
"tokens": dict(tokens), "total_definition": TOTAL_DEFINITION,
|
|
178
|
+
"why": None}
|
|
179
|
+
if breaches:
|
|
180
|
+
verdict.update({"status": "over_budget", "exit_code": OVER,
|
|
181
|
+
"reason": "; ".join(breaches)})
|
|
182
|
+
return verdict
|
|
183
|
+
verdict.update({"status": "within_budget", "exit_code": OK,
|
|
184
|
+
"reason": None})
|
|
185
|
+
return verdict
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def render(d):
|
|
189
|
+
if d["status"] == "cannot_evaluate":
|
|
190
|
+
return "CANNOT EVALUATE: %s" % d["why"]
|
|
191
|
+
if d["status"] == "over_budget":
|
|
192
|
+
return "OVER BUDGET: %s" % d["reason"]
|
|
193
|
+
return ("WITHIN BUDGET: measured %d output tokens, %d total tokens [%s]"
|
|
194
|
+
% (d["output_tokens"], d["total_tokens"], d["total_definition"]))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def main(argv=None):
|
|
198
|
+
ap = argparse.ArgumentParser(
|
|
199
|
+
description="Fail CI when a run's token usage regressed past a "
|
|
200
|
+
"budget policy.")
|
|
201
|
+
ap.add_argument("workspace", nargs="?", default=".",
|
|
202
|
+
help="workspace root (or its .loki dir); default .")
|
|
203
|
+
ap.add_argument("--max-output-tokens", type=int,
|
|
204
|
+
help="ceiling on output tokens alone (the work signal)")
|
|
205
|
+
ap.add_argument("--max-total-tokens", type=int,
|
|
206
|
+
help="ceiling on %s (cache reads dominate this)"
|
|
207
|
+
% TOTAL_DEFINITION)
|
|
208
|
+
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
209
|
+
help="emit the verdict as JSON")
|
|
210
|
+
args = ap.parse_args(argv)
|
|
211
|
+
|
|
212
|
+
d = evaluate(args.workspace, args.max_output_tokens, args.max_total_tokens)
|
|
213
|
+
print(json.dumps(d, indent=2) if args.as_json else render(d))
|
|
214
|
+
return d["exit_code"]
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
if __name__ == "__main__":
|
|
218
|
+
sys.exit(main())
|