loki-mode 8.99.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/baseline-pin.py +251 -0
- package/tools/gate-init.py +292 -0
- package/tools/receipt-find.py +329 -0
- package/tools/token-guard.py +218 -0
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode
|
|
6
|
+
# Loki Mode v9.1.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -469,4 +469,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
469
469
|
|
|
470
470
|
---
|
|
471
471
|
|
|
472
|
-
**
|
|
472
|
+
**v9.1.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
9.1.0
|
package/dashboard/__init__.py
CHANGED
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="
|
|
2
|
+
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="9.1.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Xf(Qf(import.meta.url)),Q=X$(X);h5=e_(Zf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var bO={};l0(bO,{runOrThrow:()=>jf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Tf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>yO});async function NQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function jf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Mf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Mf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Tf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return wf?"":Z}var wf,L0,F8,p0,zV0,a0,W8,Q9,v;var S6=p(()=>{wf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),zV0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),W8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as bf}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(bf(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>Kh});import{existsSync as Y9,readFileSync as g3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as rf}from"path";import{homedir as tf}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*LQ/X);if(J>LQ)J=LQ;let z=LQ-J,K=F8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${W8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function Zh(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1232,4 +1232,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1232
1232
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (h_(),f_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1233
1233
|
`),process.stderr.write(v_),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var uW0=await mW0(Bun.argv.slice(2));process.exit(uW0);
|
|
1234
1234
|
|
|
1235
|
-
//# debugId=
|
|
1235
|
+
//# debugId=0282E3F3DE141D2464756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "9.1.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "9.1.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Pin one run as THE cost baseline, then resolve it later.
|
|
3
|
+
|
|
4
|
+
cost-guard.py compares against a baseline receipt, but somebody has to hand-pick
|
|
5
|
+
that file and keep it fresh. There is no way to say "the current main is the
|
|
6
|
+
reference" and have every later PR compare against it. This is that:
|
|
7
|
+
|
|
8
|
+
tools/baseline-pin.py set <workspace> # pin the newest receipt there
|
|
9
|
+
tools/baseline-pin.py show # what is pinned, and is it intact
|
|
10
|
+
tools/baseline-pin.py path # the proof path, for --baseline
|
|
11
|
+
|
|
12
|
+
tools/cost-guard.py . --max-increase-pct 10 \\
|
|
13
|
+
--baseline "$(tools/baseline-pin.py path)"
|
|
14
|
+
|
|
15
|
+
WHAT MAKES A PIN WORTH TRUSTING, and why each one is a refusal rather than a
|
|
16
|
+
warning:
|
|
17
|
+
|
|
18
|
+
1. NEVER PIN AN UNVERIFIABLE OR UNMEASURED RUN. A baseline nobody can verify
|
|
19
|
+
makes every later comparison meaningless -- the percentage is computed
|
|
20
|
+
against a number that may have been typed in. A baseline with no measured
|
|
21
|
+
cost makes every percentage undefined. Both refuse, with DIFFERENT messages,
|
|
22
|
+
because the operator's next move differs: one means re-run the build, the
|
|
23
|
+
other means fix instrumentation. Integrity is checked FIRST: a receipt that
|
|
24
|
+
fails it has no trustworthy cost to report, so "no measured cost" would be
|
|
25
|
+
the wrong diagnosis to hand back.
|
|
26
|
+
|
|
27
|
+
Neither predicate is restated here. Integrity is verify_integrity() from
|
|
28
|
+
autonomy/lib/proof-verify.py; measurement is measured_usd() from
|
|
29
|
+
cost-guard.py, which already maps the receipt's cost.usd onto the
|
|
30
|
+
record_is_measured() key name. A second copy of either is how the rule
|
|
31
|
+
drifts.
|
|
32
|
+
|
|
33
|
+
2. A PIN RECORDS WHAT IT PINNED, not just a path. The receipt's hash, the cost
|
|
34
|
+
observed at pin time, and when. A path alone silently follows a file that
|
|
35
|
+
can be edited afterwards, so "the baseline" would mean whatever that file
|
|
36
|
+
says today.
|
|
37
|
+
|
|
38
|
+
The hash is over the RAW FILE BYTES, deliberately not the canonical
|
|
39
|
+
verification-stripped digest the receipt hashes itself with. That digest is
|
|
40
|
+
blind to any edit confined to the verification block -- swap the whole block
|
|
41
|
+
and it is unchanged. For pinning, the question is "did this file change",
|
|
42
|
+
and a reordered or re-signed file IS a changed file.
|
|
43
|
+
|
|
44
|
+
3. show DETECTS DRIFT SINCE PINNING and says so loudly, and `path` REFUSES to
|
|
45
|
+
emit a drifted baseline. A stale baseline that quietly drifts is worse than
|
|
46
|
+
none: composed into --baseline "$(...)" it poisons the comparison while
|
|
47
|
+
still looking like a green gate.
|
|
48
|
+
|
|
49
|
+
Exit: 0 fine, 1 refused / drifted / no pin. Never an empty success.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
import argparse
|
|
53
|
+
import glob
|
|
54
|
+
import hashlib
|
|
55
|
+
import importlib.util
|
|
56
|
+
import json
|
|
57
|
+
import os
|
|
58
|
+
import sys
|
|
59
|
+
import time
|
|
60
|
+
|
|
61
|
+
sys.dont_write_bytecode = True
|
|
62
|
+
|
|
63
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
64
|
+
_LIB = os.path.join(os.path.dirname(_HERE), "autonomy", "lib")
|
|
65
|
+
sys.path.insert(0, _LIB)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _load(name, path):
|
|
69
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
70
|
+
mod = importlib.util.module_from_spec(spec)
|
|
71
|
+
spec.loader.exec_module(mod)
|
|
72
|
+
return mod
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# Hyphenated filenames, so both need the loader dance rather than an import.
|
|
76
|
+
_pv = _load("proof_verify", os.path.join(_LIB, "proof-verify.py"))
|
|
77
|
+
_cg = _load("cost_guard", os.path.join(_HERE, "cost-guard.py"))
|
|
78
|
+
|
|
79
|
+
OK, REFUSED = 0, 1
|
|
80
|
+
DEFAULT_FILE = os.path.join(".loki", "baseline.json")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _file_sha256(path):
|
|
84
|
+
with open(path, "rb") as handle:
|
|
85
|
+
return hashlib.sha256(handle.read()).hexdigest()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def newest_receipt(workspace):
|
|
89
|
+
"""The most recent proof.json under a workspace, or None.
|
|
90
|
+
|
|
91
|
+
Same convention as ci-gate.py: accept a workspace root or its .loki dir.
|
|
92
|
+
"""
|
|
93
|
+
root = os.path.normpath(workspace)
|
|
94
|
+
if os.path.basename(root) == ".loki":
|
|
95
|
+
root = os.path.dirname(root) or "."
|
|
96
|
+
found = glob.glob(os.path.join(root, ".loki", "proofs", "*", "proof.json"))
|
|
97
|
+
return max(found, key=os.path.getmtime) if found else None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Refused(Exception):
|
|
101
|
+
"""A refusal carrying the reason the operator needs to act on."""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def build_pin(workspace):
|
|
105
|
+
"""The pin record for a workspace, or raise Refused explaining why not."""
|
|
106
|
+
proof_path = newest_receipt(workspace)
|
|
107
|
+
if proof_path is None:
|
|
108
|
+
raise Refused(
|
|
109
|
+
"no receipt under %s/.loki/proofs/*/proof.json, so there is no run "
|
|
110
|
+
"to pin. Run a build first." % os.path.normpath(workspace))
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
proof = _pv._load_proof(proof_path)
|
|
114
|
+
except Exception as exc:
|
|
115
|
+
raise Refused("receipt %s could not be read: %s" % (proof_path, exc))
|
|
116
|
+
|
|
117
|
+
verdict = _pv.verify_integrity(proof)
|
|
118
|
+
if not verdict["ok"]:
|
|
119
|
+
raise Refused(
|
|
120
|
+
"receipt %s FAILED integrity verification, so it must not become a "
|
|
121
|
+
"baseline: every later comparison would be against a number nobody "
|
|
122
|
+
"can verify.\n %s"
|
|
123
|
+
% (proof_path,
|
|
124
|
+
"\n ".join(verdict["reasons"] or [verdict["reason"]])))
|
|
125
|
+
|
|
126
|
+
usd = _cg.measured_usd(proof.get("cost"))
|
|
127
|
+
if usd is None:
|
|
128
|
+
raise Refused(
|
|
129
|
+
"receipt %s records NO MEASURED COST, so it must not become a "
|
|
130
|
+
"baseline: a percentage increase against an unmeasured number is "
|
|
131
|
+
"undefined. Unmeasured is not $0.00. Fix the cost instrumentation "
|
|
132
|
+
"for that run, then pin it." % proof_path)
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
"pin": "loki-cost-baseline/v1",
|
|
136
|
+
"proof_path": os.path.abspath(proof_path),
|
|
137
|
+
"receipt_sha256": _file_sha256(proof_path),
|
|
138
|
+
"cost_usd": usd,
|
|
139
|
+
"pinned_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def read_pin(pin_file):
|
|
144
|
+
"""The pin record, or raise Refused. Never an empty success."""
|
|
145
|
+
if not os.path.exists(pin_file):
|
|
146
|
+
raise Refused(
|
|
147
|
+
"no baseline pinned (%s does not exist). Pin one with: "
|
|
148
|
+
"tools/baseline-pin.py set <workspace>" % pin_file)
|
|
149
|
+
try:
|
|
150
|
+
with open(pin_file, "r", encoding="utf-8") as handle:
|
|
151
|
+
rec = json.load(handle)
|
|
152
|
+
except Exception as exc:
|
|
153
|
+
raise Refused("pin file %s is unreadable: %s" % (pin_file, exc))
|
|
154
|
+
if not isinstance(rec, dict) or not rec.get("proof_path"):
|
|
155
|
+
raise Refused("pin file %s records no proof_path" % pin_file)
|
|
156
|
+
return rec
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def check_pin(rec):
|
|
160
|
+
"""Re-hash the pinned receipt. Returns (state, detail).
|
|
161
|
+
|
|
162
|
+
state is "intact", "changed", or "missing".
|
|
163
|
+
"""
|
|
164
|
+
path = rec["proof_path"]
|
|
165
|
+
if not os.path.exists(path):
|
|
166
|
+
return "missing", "the pinned receipt no longer exists at %s" % path
|
|
167
|
+
now = _file_sha256(path)
|
|
168
|
+
if now != rec.get("receipt_sha256"):
|
|
169
|
+
return "changed", (
|
|
170
|
+
"the pinned receipt CHANGED since it was pinned: %s\n pinned "
|
|
171
|
+
"sha256 %s\n current sha256 %s\n This baseline no longer "
|
|
172
|
+
"describes the run it was pinned from. Re-pin it deliberately, or "
|
|
173
|
+
"every comparison against it is against an unknown."
|
|
174
|
+
% (path, rec.get("receipt_sha256"), now))
|
|
175
|
+
return "intact", "receipt unchanged since pinning"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def main(argv):
|
|
179
|
+
ap = argparse.ArgumentParser(
|
|
180
|
+
description="Pin a run as the cost baseline, then resolve it later.")
|
|
181
|
+
sub = ap.add_subparsers(dest="cmd")
|
|
182
|
+
|
|
183
|
+
p_set = sub.add_parser("set", help="pin the newest receipt in a workspace")
|
|
184
|
+
p_set.add_argument("workspace")
|
|
185
|
+
p_set.add_argument("--file", default=DEFAULT_FILE)
|
|
186
|
+
|
|
187
|
+
p_show = sub.add_parser("show", help="what is pinned, and is it intact")
|
|
188
|
+
p_show.add_argument("--file", default=DEFAULT_FILE)
|
|
189
|
+
p_show.add_argument("--json", action="store_true")
|
|
190
|
+
|
|
191
|
+
p_path = sub.add_parser("path", help="the pinned proof path, for --baseline")
|
|
192
|
+
p_path.add_argument("--file", default=DEFAULT_FILE)
|
|
193
|
+
|
|
194
|
+
args = ap.parse_args(argv)
|
|
195
|
+
if not args.cmd:
|
|
196
|
+
ap.print_help(sys.stderr)
|
|
197
|
+
return REFUSED
|
|
198
|
+
|
|
199
|
+
if args.cmd == "set":
|
|
200
|
+
try:
|
|
201
|
+
rec = build_pin(args.workspace)
|
|
202
|
+
except Refused as exc:
|
|
203
|
+
sys.stderr.write("REFUSED TO PIN: %s\n" % exc)
|
|
204
|
+
return REFUSED
|
|
205
|
+
parent = os.path.dirname(os.path.abspath(args.file))
|
|
206
|
+
if parent:
|
|
207
|
+
os.makedirs(parent, exist_ok=True)
|
|
208
|
+
with open(args.file, "w", encoding="utf-8") as handle:
|
|
209
|
+
json.dump(rec, handle, indent=2, sort_keys=True)
|
|
210
|
+
handle.write("\n")
|
|
211
|
+
sys.stderr.write(
|
|
212
|
+
"PINNED %s\n cost $%.4f sha256 %s\n written to %s\n"
|
|
213
|
+
% (rec["proof_path"], rec["cost_usd"], rec["receipt_sha256"],
|
|
214
|
+
args.file))
|
|
215
|
+
return OK
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
rec = read_pin(args.file)
|
|
219
|
+
except Refused as exc:
|
|
220
|
+
sys.stderr.write("NO BASELINE: %s\n" % exc)
|
|
221
|
+
return REFUSED
|
|
222
|
+
|
|
223
|
+
state, detail = check_pin(rec)
|
|
224
|
+
|
|
225
|
+
if args.cmd == "path":
|
|
226
|
+
if state != "intact":
|
|
227
|
+
sys.stderr.write("REFUSING TO EMIT BASELINE: %s\n" % detail)
|
|
228
|
+
return REFUSED
|
|
229
|
+
# ONLY the path on stdout, so this composes into --baseline "$(...)".
|
|
230
|
+
sys.stdout.write("%s\n" % rec["proof_path"])
|
|
231
|
+
return OK
|
|
232
|
+
|
|
233
|
+
if getattr(args, "json", False):
|
|
234
|
+
out = dict(rec)
|
|
235
|
+
out["state"] = state
|
|
236
|
+
out["detail"] = detail
|
|
237
|
+
sys.stdout.write(json.dumps(out, indent=2, sort_keys=True) + "\n")
|
|
238
|
+
else:
|
|
239
|
+
sys.stdout.write(
|
|
240
|
+
"baseline: %s\n pinned at %s\n cost at pin $%.4f\n"
|
|
241
|
+
% (rec["proof_path"], rec.get("pinned_at", "unknown"),
|
|
242
|
+
rec.get("cost_usd") or 0.0))
|
|
243
|
+
if state != "intact":
|
|
244
|
+
sys.stderr.write("BASELINE %s: %s\n" % (state.upper(), detail))
|
|
245
|
+
return REFUSED
|
|
246
|
+
sys.stderr.write("baseline INTACT: %s\n" % detail)
|
|
247
|
+
return OK
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
if __name__ == "__main__":
|
|
251
|
+
sys.exit(main(sys.argv[1:]))
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Generate the policy file and CI snippet that turn the cost gate on.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. The chain already works: policy-load.py validates a policy
|
|
5
|
+
file, ci-gate.py enforces it, gate-report.py renders the verdict where an
|
|
6
|
+
engineer will see it. Adopting it does not. A team has to hand-write a JSON
|
|
7
|
+
file against a schema documented only in a validator's source, work out that
|
|
8
|
+
`--as-args` is the flag that joins the first tool to the second, and compose
|
|
9
|
+
three commands in the right order. That is several required steps before any
|
|
10
|
+
value, which is the highest-severity adoption defect this project recognises:
|
|
11
|
+
a gate nobody turns on enforces exactly as much as no gate at all.
|
|
12
|
+
|
|
13
|
+
So: one command that writes the file and prints the snippet.
|
|
14
|
+
|
|
15
|
+
python3 tools/gate-init.py . --out .loki-policy.json --print-workflow
|
|
16
|
+
|
|
17
|
+
THE RULE THIS FILE EXISTS TO HOLD, and the reason it is not a template dump:
|
|
18
|
+
|
|
19
|
+
NEVER INVENT A CEILING.
|
|
20
|
+
|
|
21
|
+
A generator's whole appeal is that it fills in the blanks, and a cost ceiling
|
|
22
|
+
is the one blank it must refuse to fill from nothing. A plausible-looking
|
|
23
|
+
$5.00 in a generated file is worse than an empty one: the operator commits it,
|
|
24
|
+
CI goes green, and the number enforced is one nobody chose and nobody can
|
|
25
|
+
defend when it starts blocking merges. The green is the problem -- it reads as
|
|
26
|
+
"reviewed and within budget" when it means "arbitrary". This repo has paid for
|
|
27
|
+
that shape repeatedly, most recently a tarball assertion that passed on "6 or
|
|
28
|
+
more" of 6 patterns: a check reporting a pass without having checked.
|
|
29
|
+
|
|
30
|
+
So a ceiling is emitted ONLY when measured runs exist to derive it from, and
|
|
31
|
+
the derivation is stated in the output. With no measured history the `max_usd`
|
|
32
|
+
key is ABSENT and the operator is told, on stderr, that they must choose one.
|
|
33
|
+
|
|
34
|
+
WHY ABSENT AND NOT null, WHICH IS WHAT YOU WOULD REACH FOR FIRST. Verified by
|
|
35
|
+
running policy-load.py on all four candidates:
|
|
36
|
+
|
|
37
|
+
{"max_usd": null, ...} rejected: "max_usd must be a number"
|
|
38
|
+
{"_note": "...", ...} rejected: unknown policy key
|
|
39
|
+
{"require_receipt": false} rejected: "enforces nothing"
|
|
40
|
+
{"require_receipt": true} loads, exit 0
|
|
41
|
+
|
|
42
|
+
JSON has no comments and policy-load rejects unknown keys, so the "you must
|
|
43
|
+
choose a ceiling" note CANNOT live in the file. It goes to stderr and into the
|
|
44
|
+
printed snippet, where it is at least as visible and cannot break the loader.
|
|
45
|
+
The receipt requirement carries the file on its own so the emitted policy still
|
|
46
|
+
enforces something rather than tripping policy-load's own vacuity check.
|
|
47
|
+
|
|
48
|
+
WHY IT RUNS policy-load ON ITS OWN OUTPUT. A generator that emits a file the
|
|
49
|
+
loader rejects is worse than one that emits nothing: the operator has a
|
|
50
|
+
committed artifact, a passing generator, and a broken gate. Self-validation is
|
|
51
|
+
a subprocess call to the real validator, never a re-implementation of its rules
|
|
52
|
+
here -- a second copy of the schema is how the two drift apart, and the copy
|
|
53
|
+
that drifts is always the one nobody runs.
|
|
54
|
+
|
|
55
|
+
WHAT IT REFUSES TO DO. It never overwrites an existing policy without --force:
|
|
56
|
+
that file is reviewed, committed, and possibly a LOWER ceiling than this tool
|
|
57
|
+
would derive, so silently replacing it is a destructive act that reads as a
|
|
58
|
+
routine re-run. It never writes into .github/ -- that directory is live, and a
|
|
59
|
+
snippet this tool has never executed has no business landing in a real
|
|
60
|
+
workflow. The snippet is labelled as a starting point for that reason.
|
|
61
|
+
|
|
62
|
+
Exit: 0 policy written and validated; 1 refused (existing file, or the
|
|
63
|
+
generated policy failed its own validation); 2 cannot write.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
import argparse
|
|
67
|
+
import json
|
|
68
|
+
import os
|
|
69
|
+
import subprocess
|
|
70
|
+
import sys
|
|
71
|
+
|
|
72
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
73
|
+
|
|
74
|
+
OK, REFUSED, CANNOT = 0, 1, 2
|
|
75
|
+
|
|
76
|
+
DEFAULT_OUT = ".loki-policy.json"
|
|
77
|
+
|
|
78
|
+
# Headroom over observed spend. A ceiling at the median blocks half of all
|
|
79
|
+
# normal runs, which trains the team to bypass the gate -- the failure mode a
|
|
80
|
+
# too-tight budget shares with a too-loose one is that it stops being obeyed.
|
|
81
|
+
HEADROOM = 2.0
|
|
82
|
+
|
|
83
|
+
# ponytail: reuse cost-history's loader and median rather than re-reading
|
|
84
|
+
# JSONL here. Its honesty rules (unmeasured is null and excluded, corrupt lines
|
|
85
|
+
# are counted) are the ones this derivation needs, and a second reader is how
|
|
86
|
+
# they drift.
|
|
87
|
+
sys.path.insert(0, _HERE)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _load_cost_history():
|
|
91
|
+
"""cost-history.py as a module, despite the hyphen in its name."""
|
|
92
|
+
import importlib.util
|
|
93
|
+
|
|
94
|
+
sys.dont_write_bytecode = True
|
|
95
|
+
path = os.path.join(_HERE, "cost-history.py")
|
|
96
|
+
spec = importlib.util.spec_from_file_location("cost_history", path)
|
|
97
|
+
module = importlib.util.module_from_spec(spec)
|
|
98
|
+
spec.loader.exec_module(module)
|
|
99
|
+
return module
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def measured_costs(workspace, history_file=None):
|
|
103
|
+
"""Every measured USD figure in this workspace's history, oldest first.
|
|
104
|
+
|
|
105
|
+
Rows recorded as unmeasured carry usd=null and are EXCLUDED, never read as
|
|
106
|
+
0. A workspace whose every run failed to measure has no basis for a
|
|
107
|
+
ceiling, and averaging zeros into one would produce a $0.00 ceiling that
|
|
108
|
+
blocks every future run while looking like a real budget.
|
|
109
|
+
"""
|
|
110
|
+
ch = _load_cost_history()
|
|
111
|
+
path = history_file or os.path.join(workspace, ch.DEFAULT_FILE)
|
|
112
|
+
entries, corrupt = ch.load(path)
|
|
113
|
+
if entries is None:
|
|
114
|
+
return [], 0, path
|
|
115
|
+
costs = [e["usd"] for e in entries
|
|
116
|
+
if e.get("measured") and ch._num(e.get("usd")) is not None]
|
|
117
|
+
return costs, corrupt, path
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def derive(costs):
|
|
121
|
+
"""(ceiling, basis) from measured runs, or (None, why-not).
|
|
122
|
+
|
|
123
|
+
Returns None for the ceiling whenever there is nothing to derive it from.
|
|
124
|
+
That None is the product: it is what stops a number with no basis being
|
|
125
|
+
written into a file someone will commit.
|
|
126
|
+
"""
|
|
127
|
+
if not costs:
|
|
128
|
+
return None, ("no measured runs in this workspace's cost history")
|
|
129
|
+
ch = _load_cost_history()
|
|
130
|
+
med = ch.median(costs)
|
|
131
|
+
if med <= 0:
|
|
132
|
+
# Every measured run summed to zero. Present records, no spend: that is
|
|
133
|
+
# a measurement gap wearing a number's clothes, and a $0.00 ceiling
|
|
134
|
+
# derived from it would block every run that ever does real work.
|
|
135
|
+
return None, ("%d measured run(s) but their median is $%.4f; a ceiling "
|
|
136
|
+
"of zero would block every run that does real work"
|
|
137
|
+
% (len(costs), med))
|
|
138
|
+
ceiling = round(med * HEADROOM, 2)
|
|
139
|
+
basis = ("median of %d measured run(s) = $%.4f, times %.1fx headroom"
|
|
140
|
+
% (len(costs), med, HEADROOM))
|
|
141
|
+
return ceiling, basis
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def build_policy(ceiling):
|
|
145
|
+
"""The policy dict. max_usd is ABSENT, never null, when unset.
|
|
146
|
+
|
|
147
|
+
require_receipt is always true: it is the one policy that needs no measured
|
|
148
|
+
history to justify, and without it a ceiling-less policy would imply no
|
|
149
|
+
ci-gate flags at all -- which policy-load correctly rejects as a gate with
|
|
150
|
+
nothing to check.
|
|
151
|
+
"""
|
|
152
|
+
policy = {"require_receipt": True}
|
|
153
|
+
if ceiling is not None:
|
|
154
|
+
policy["max_usd"] = ceiling
|
|
155
|
+
return policy
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def validate(path):
|
|
159
|
+
"""Run the REAL loader on the generated file. Returns (ok, message)."""
|
|
160
|
+
proc = subprocess.run(
|
|
161
|
+
[sys.executable, os.path.join(_HERE, "policy-load.py"),
|
|
162
|
+
"--file", path, "--as-args"],
|
|
163
|
+
capture_output=True, text=True)
|
|
164
|
+
if proc.returncode != 0:
|
|
165
|
+
return False, (proc.stderr.strip() or
|
|
166
|
+
"policy-load exited %d with no message" % proc.returncode)
|
|
167
|
+
return True, proc.stdout.strip()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def workflow_snippet(out_path):
|
|
171
|
+
"""A starting point, NOT a tested workflow. Labelled as such in the text.
|
|
172
|
+
|
|
173
|
+
This tool has never run this YAML. Calling it verified would be the same
|
|
174
|
+
manufactured confidence the gate itself exists to prevent, one layer up.
|
|
175
|
+
"""
|
|
176
|
+
return """# STARTING POINT -- NOT TESTED. This tool generated this snippet but has
|
|
177
|
+
# never executed it. Review it, adapt the checkout/setup steps to your repo,
|
|
178
|
+
# and run it once on a branch before trusting it to block a merge.
|
|
179
|
+
#
|
|
180
|
+
# Add to a job in .github/workflows/<your-workflow>.yml:
|
|
181
|
+
|
|
182
|
+
- name: Cost gate
|
|
183
|
+
run: |
|
|
184
|
+
python3 tools/ci-gate.py . --json \\
|
|
185
|
+
$(python3 tools/policy-load.py --file {out} --as-args) \\
|
|
186
|
+
| python3 tools/gate-report.py --format github
|
|
187
|
+
""".format(out=out_path)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def generate(workspace, out_path, force=False, history_file=None):
|
|
191
|
+
"""Write the policy. Returns a verdict dict; never overwrites silently."""
|
|
192
|
+
if os.path.exists(out_path) and not force:
|
|
193
|
+
return {"status": "exists", "exit_code": REFUSED,
|
|
194
|
+
"why": ("refusing to overwrite existing policy file %s -- it is "
|
|
195
|
+
"reviewed and committed, and may hold a lower ceiling "
|
|
196
|
+
"than this tool would derive. Re-run with --force to "
|
|
197
|
+
"replace it." % out_path),
|
|
198
|
+
"policy": None, "ceiling": None, "basis": None,
|
|
199
|
+
"out": out_path}
|
|
200
|
+
|
|
201
|
+
costs, corrupt, hist_path = measured_costs(workspace, history_file)
|
|
202
|
+
ceiling, basis = derive(costs)
|
|
203
|
+
policy = build_policy(ceiling)
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
parent = os.path.dirname(os.path.abspath(out_path))
|
|
207
|
+
if parent:
|
|
208
|
+
os.makedirs(parent, exist_ok=True)
|
|
209
|
+
with open(out_path, "w", encoding="utf-8") as handle:
|
|
210
|
+
handle.write(json.dumps(policy, indent=2, sort_keys=True) + "\n")
|
|
211
|
+
except OSError as exc:
|
|
212
|
+
return {"status": "cannot_write", "exit_code": CANNOT,
|
|
213
|
+
"why": "could not write %s: %s" % (out_path, exc),
|
|
214
|
+
"policy": policy, "ceiling": ceiling, "basis": basis,
|
|
215
|
+
"out": out_path}
|
|
216
|
+
|
|
217
|
+
ok, message = validate(out_path)
|
|
218
|
+
if not ok:
|
|
219
|
+
# Generated and rejected by the real loader. Non-zero, loudly: a
|
|
220
|
+
# committed file the gate cannot load is a gate that never runs.
|
|
221
|
+
return {"status": "invalid", "exit_code": REFUSED,
|
|
222
|
+
"why": ("the generated policy failed policy-load.py: %s" % message),
|
|
223
|
+
"policy": policy, "ceiling": ceiling, "basis": basis,
|
|
224
|
+
"out": out_path}
|
|
225
|
+
|
|
226
|
+
return {"status": "written", "exit_code": OK, "why": None,
|
|
227
|
+
"policy": policy, "ceiling": ceiling, "basis": basis,
|
|
228
|
+
"out": out_path, "ci_gate_args": message,
|
|
229
|
+
"history_file": hist_path, "measured_runs": len(costs),
|
|
230
|
+
"corrupt_lines": corrupt}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def main(argv=None):
|
|
234
|
+
ap = argparse.ArgumentParser(
|
|
235
|
+
description="Generate a ci-gate policy file and the CI snippet that "
|
|
236
|
+
"enforces it.")
|
|
237
|
+
ap.add_argument("workspace", nargs="?", default=".",
|
|
238
|
+
help="workspace to derive a ceiling from; default .")
|
|
239
|
+
ap.add_argument("--out", default=DEFAULT_OUT,
|
|
240
|
+
help="policy file to write; default %s" % DEFAULT_OUT)
|
|
241
|
+
ap.add_argument("--force", action="store_true",
|
|
242
|
+
help="overwrite an existing policy file")
|
|
243
|
+
ap.add_argument("--print-workflow", action="store_true",
|
|
244
|
+
dest="print_workflow",
|
|
245
|
+
help="also print a starting-point CI snippet")
|
|
246
|
+
ap.add_argument("--history-file", default=None,
|
|
247
|
+
help="cost history JSONL; default <workspace>/.loki/cost-history.jsonl")
|
|
248
|
+
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
249
|
+
help="emit the verdict as JSON")
|
|
250
|
+
args = ap.parse_args(argv)
|
|
251
|
+
|
|
252
|
+
verdict = generate(args.workspace, args.out, force=args.force,
|
|
253
|
+
history_file=args.history_file)
|
|
254
|
+
|
|
255
|
+
if args.as_json:
|
|
256
|
+
print(json.dumps(verdict, indent=2, sort_keys=True))
|
|
257
|
+
elif verdict["exit_code"] != OK:
|
|
258
|
+
print("gate-init: %s" % verdict["why"], file=sys.stderr)
|
|
259
|
+
else:
|
|
260
|
+
print("wrote %s" % verdict["out"])
|
|
261
|
+
print(json.dumps(verdict["policy"], indent=2, sort_keys=True))
|
|
262
|
+
if verdict["ceiling"] is not None:
|
|
263
|
+
print("\nmax_usd = %s, derived from: %s"
|
|
264
|
+
% (verdict["ceiling"], verdict["basis"]))
|
|
265
|
+
print("\nvalidated by policy-load.py; ci-gate args: %s"
|
|
266
|
+
% verdict.get("ci_gate_args", ""))
|
|
267
|
+
|
|
268
|
+
# OUTSIDE the format branch, deliberately. Under --json this note used to
|
|
269
|
+
# vanish entirely: the operator got "ceiling": null and a basis string that
|
|
270
|
+
# reads as a diagnostic field rather than an instruction, and a policy
|
|
271
|
+
# enforcing less than it appears to. A machine-readable format is a reason
|
|
272
|
+
# to render the warning differently, never a reason to drop it -- that is
|
|
273
|
+
# the same "looks configured, enforces nothing" shape the whole chain
|
|
274
|
+
# exists to prevent. stderr keeps stdout valid JSON for a pipe.
|
|
275
|
+
if verdict["exit_code"] == OK and verdict["ceiling"] is None:
|
|
276
|
+
print("\nNO CEILING WAS SET: %s." % verdict["basis"], file=sys.stderr)
|
|
277
|
+
print("The max_usd key is ABSENT rather than guessed. You must "
|
|
278
|
+
"choose a ceiling and add it, for example:\n"
|
|
279
|
+
' "max_usd": 5.00\n'
|
|
280
|
+
"Record some runs first (python3 tools/cost-history.py "
|
|
281
|
+
"record <workspace>) and re-run with --force to derive one "
|
|
282
|
+
"from measured spend.", file=sys.stderr)
|
|
283
|
+
|
|
284
|
+
if args.print_workflow and verdict["exit_code"] == OK:
|
|
285
|
+
print()
|
|
286
|
+
print(workflow_snippet(verdict["out"]))
|
|
287
|
+
|
|
288
|
+
return verdict["exit_code"]
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
if __name__ == "__main__":
|
|
292
|
+
sys.exit(main())
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Find the receipts an auditor actually needs, out of a workspace full of them.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. receipt-bundle.py verifies EVERY receipt under a workspace and
|
|
5
|
+
rolls them into one verdict. That is the right artifact to hand a compliance
|
|
6
|
+
reviewer, and the wrong one to work from. An auditor with three hundred runs is
|
|
7
|
+
not asking "is the archive sound"; they are asking "which run cost $40", "which
|
|
8
|
+
ones did not verify", "what has happened since the incident on the 12th".
|
|
9
|
+
Nothing queried receipts. This does, and the whole design problem is that a
|
|
10
|
+
filter is a claim about what it did NOT return.
|
|
11
|
+
|
|
12
|
+
THE RULES. Each is a specific way a result set can claim more than it earned.
|
|
13
|
+
|
|
14
|
+
1. AN UNMEASURED COST MATCHES NO NUMERIC FILTER, IN EITHER DIRECTION. A receipt
|
|
15
|
+
that never recorded cost is not "under $5" and it is not "over $5" -- the
|
|
16
|
+
comparison is undefined, and answering it either way invents a measurement.
|
|
17
|
+
Treating unmeasured as 0.0 is the exact lie the cost-honesty line exists to
|
|
18
|
+
prevent (v8.51.0-v8.54.0 fixed it on four surfaces); it would make every
|
|
19
|
+
unmeasured run answer "cheapest in the archive". So unmeasured receipts are
|
|
20
|
+
EXCLUDED from a cost filter and the exclusion is COUNTED and REPORTED. An
|
|
21
|
+
auditor who asked for runs over $5 must be told the filter could not
|
|
22
|
+
consider N of them, or they will read the result as exhaustive.
|
|
23
|
+
|
|
24
|
+
The predicate is record_is_measured() in autonomy/lib/efficiency_cost.py,
|
|
25
|
+
reached through receipt-diff.py's measured_cost(), which already maps the
|
|
26
|
+
receipt's `cost.usd` onto the per-iteration `cost_usd` key it expects. A
|
|
27
|
+
second copy of that predicate is how the honesty rule drifts, so there is
|
|
28
|
+
not one here.
|
|
29
|
+
|
|
30
|
+
Measured is necessary but NOT sufficient for a cost comparison:
|
|
31
|
+
record_is_measured is true when ANY of five fields is non-zero, so a receipt
|
|
32
|
+
with tokens but a null `usd` is honestly measured and still has no dollar
|
|
33
|
+
figure. Both conditions are required -- see _usd().
|
|
34
|
+
|
|
35
|
+
2. ZERO MATCHES IS A RESULT; ZERO RECEIPTS IS NOT THE SAME FACT. "The filter
|
|
36
|
+
ran and nothing qualified" and "there was nothing to search" look identical
|
|
37
|
+
in a list of zero lines, and they mean opposite things: one is a clean
|
|
38
|
+
answer, the other is a broken invocation pointed at the wrong directory.
|
|
39
|
+
They get different summaries and different exit codes.
|
|
40
|
+
|
|
41
|
+
3. A MALFORMED RECEIPT IS COUNTED AND NAMED, NEVER SILENTLY SKIPPED. A search
|
|
42
|
+
that drops what it cannot read reports a tidier archive than exists, and
|
|
43
|
+
does it invisibly -- the reader sees only files that parsed and cannot tell
|
|
44
|
+
whether the rest were irrelevant or unreadable. Same rule as the bundle's:
|
|
45
|
+
absent is not clean.
|
|
46
|
+
|
|
47
|
+
4. A BAD --since IS REJECTED, NOT GUESSED. An unparseable date that silently
|
|
48
|
+
matches everything (or nothing) returns a result set with no relation to the
|
|
49
|
+
question asked, and looks exactly like a correct one.
|
|
50
|
+
|
|
51
|
+
5. FILTERS ARE AND, AND THE OUTPUT SAYS WHICH WERE APPLIED. A result set with
|
|
52
|
+
no visible filter line can be pasted anywhere and read as "all receipts".
|
|
53
|
+
|
|
54
|
+
Usage:
|
|
55
|
+
tools/receipt-find.py [workspace] [--min-usd N] [--max-usd N]
|
|
56
|
+
[--failed-only] [--since YYYY-MM-DD] [--json]
|
|
57
|
+
|
|
58
|
+
Exit codes:
|
|
59
|
+
0 at least one receipt matched
|
|
60
|
+
1 receipts were searched, none matched (a valid result, not an error)
|
|
61
|
+
3 no receipts found at all -- nothing was searched
|
|
62
|
+
2 usage error (argparse; e.g. an unparseable --since)
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
import argparse
|
|
66
|
+
import datetime
|
|
67
|
+
import importlib.util
|
|
68
|
+
import json
|
|
69
|
+
import pathlib
|
|
70
|
+
import sys
|
|
71
|
+
|
|
72
|
+
# A stale .pyc can mask a mutation and turn a real probe into a false
|
|
73
|
+
# "MUTATION SURVIVED", since invalidation is mtime+size and a restore is
|
|
74
|
+
# byte-identical. Set before the loader below runs.
|
|
75
|
+
sys.dont_write_bytecode = True
|
|
76
|
+
|
|
77
|
+
_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _load(name, path):
|
|
81
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
82
|
+
mod = importlib.util.module_from_spec(spec)
|
|
83
|
+
spec.loader.exec_module(mod)
|
|
84
|
+
return mod
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# measured_cost() reuses record_is_measured() AND maps cost.usd -> cost_usd.
|
|
88
|
+
# Both halves of rule 1 already live there; this file must not restate either.
|
|
89
|
+
_rd = _load("receipt_diff", _ROOT / "tools" / "receipt-diff.py")
|
|
90
|
+
measured_cost = _rd.measured_cost
|
|
91
|
+
|
|
92
|
+
# The headline a receipt carries when the run did not verify. "VERIFIED WITH
|
|
93
|
+
# GAPS" is deliberately NOT failed -- it verified, with recorded gaps -- and a
|
|
94
|
+
# receipt with no headline at all is not silently treated as fine: it is
|
|
95
|
+
# UNKNOWN, reported in its own bucket, and never matched by --failed-only,
|
|
96
|
+
# because "we cannot tell" must not read as "it passed".
|
|
97
|
+
FAILED_HEADLINE = "NOT VERIFIED"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _usd(proof):
|
|
101
|
+
"""The receipt's cost in dollars, or None when there is no such number.
|
|
102
|
+
|
|
103
|
+
None means the comparison is UNDEFINED, and every caller must skip rather
|
|
104
|
+
than substitute a value. Returning 0.0 here would make an unmeasured run
|
|
105
|
+
the cheapest in the archive and match every --max-usd -- rule 1, and the
|
|
106
|
+
single place it is decided.
|
|
107
|
+
"""
|
|
108
|
+
rec = measured_cost(proof)
|
|
109
|
+
if rec is None:
|
|
110
|
+
return None
|
|
111
|
+
return rec.get("cost_usd")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _headline(proof):
|
|
115
|
+
honesty = proof.get("honesty")
|
|
116
|
+
if not isinstance(honesty, dict):
|
|
117
|
+
return None
|
|
118
|
+
h = honesty.get("headline")
|
|
119
|
+
return h if isinstance(h, str) and h else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _date(proof):
|
|
123
|
+
"""The receipt's UTC date as YYYY-MM-DD, or None.
|
|
124
|
+
|
|
125
|
+
Sliced, not parsed. generated_at ends in "Z", which datetime.fromisoformat
|
|
126
|
+
rejects before Python 3.11, and this tool must behave identically on both
|
|
127
|
+
interpreters. ISO dates compare correctly as strings, so a slice is both
|
|
128
|
+
the portable answer and the smaller one.
|
|
129
|
+
"""
|
|
130
|
+
v = proof.get("generated_at")
|
|
131
|
+
if not isinstance(v, str) or len(v) < 10:
|
|
132
|
+
return None
|
|
133
|
+
d = v[:10]
|
|
134
|
+
try:
|
|
135
|
+
datetime.date.fromisoformat(d)
|
|
136
|
+
except ValueError:
|
|
137
|
+
return None
|
|
138
|
+
return d
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def find_receipts(workspace):
|
|
142
|
+
"""Every proof.json under the workspace, sorted for a stable report.
|
|
143
|
+
|
|
144
|
+
ponytail: same rglob as receipt-bundle.find_receipts, so a receipt archived
|
|
145
|
+
outside .loki/proofs/ is still findable. Deliberately not imported --
|
|
146
|
+
receipt-bundle loads proof-verify and runs git per receipt, and this tool
|
|
147
|
+
reads JSON only.
|
|
148
|
+
"""
|
|
149
|
+
root = pathlib.Path(workspace)
|
|
150
|
+
if not root.is_dir():
|
|
151
|
+
return []
|
|
152
|
+
return sorted(p for p in root.rglob("proof.json") if p.is_file())
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def search(workspace, min_usd=None, max_usd=None, failed_only=False,
|
|
156
|
+
since=None):
|
|
157
|
+
"""Filter receipts under `workspace`. Pure: no writes, no network.
|
|
158
|
+
|
|
159
|
+
Filters AND together (rule 5). A receipt excluded by the cost filter for
|
|
160
|
+
being unmeasured is counted in `excluded_unmeasured` -- but only when a
|
|
161
|
+
cost filter was actually applied, since without one nothing excluded it and
|
|
162
|
+
reporting an exclusion would imply a filter it never faced.
|
|
163
|
+
"""
|
|
164
|
+
cost_filtered = min_usd is not None or max_usd is not None
|
|
165
|
+
paths = find_receipts(workspace)
|
|
166
|
+
|
|
167
|
+
matches = []
|
|
168
|
+
malformed = []
|
|
169
|
+
excluded_unmeasured = 0
|
|
170
|
+
|
|
171
|
+
for path in paths:
|
|
172
|
+
try:
|
|
173
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
174
|
+
proof = json.load(f)
|
|
175
|
+
if not isinstance(proof, dict):
|
|
176
|
+
raise ValueError("receipt is not a JSON object")
|
|
177
|
+
except Exception as exc:
|
|
178
|
+
# Rule 3: counted and named, with the reason. Never skipped.
|
|
179
|
+
malformed.append({"path": str(path), "reason": str(exc)})
|
|
180
|
+
continue
|
|
181
|
+
|
|
182
|
+
why = []
|
|
183
|
+
|
|
184
|
+
if cost_filtered:
|
|
185
|
+
usd = _usd(proof)
|
|
186
|
+
if usd is None:
|
|
187
|
+
# Rule 1: undefined in BOTH directions. Not below, not above.
|
|
188
|
+
excluded_unmeasured += 1
|
|
189
|
+
continue
|
|
190
|
+
if min_usd is not None and usd < min_usd:
|
|
191
|
+
continue
|
|
192
|
+
if max_usd is not None and usd > max_usd:
|
|
193
|
+
continue
|
|
194
|
+
why.append("cost.usd=%s" % usd)
|
|
195
|
+
|
|
196
|
+
headline = _headline(proof)
|
|
197
|
+
if failed_only:
|
|
198
|
+
if headline != FAILED_HEADLINE:
|
|
199
|
+
continue
|
|
200
|
+
why.append("honesty.headline=%s" % headline)
|
|
201
|
+
|
|
202
|
+
date = _date(proof)
|
|
203
|
+
if since is not None:
|
|
204
|
+
if date is None or date < since:
|
|
205
|
+
continue
|
|
206
|
+
why.append("generated_at=%s" % date)
|
|
207
|
+
|
|
208
|
+
matches.append({
|
|
209
|
+
"path": str(path),
|
|
210
|
+
"matched": why,
|
|
211
|
+
"cost_usd": _usd(proof),
|
|
212
|
+
"headline": headline,
|
|
213
|
+
"date": date,
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
return {
|
|
217
|
+
"report": "loki-receipt-find/v1",
|
|
218
|
+
"workspace": str(workspace),
|
|
219
|
+
"filters": _filters(min_usd, max_usd, failed_only, since),
|
|
220
|
+
"matches": matches,
|
|
221
|
+
"match_count": len(matches),
|
|
222
|
+
"scanned": len(paths),
|
|
223
|
+
"malformed": malformed,
|
|
224
|
+
"malformed_count": len(malformed),
|
|
225
|
+
"excluded_unmeasured": excluded_unmeasured,
|
|
226
|
+
"summary": _summary(len(paths), len(matches), len(malformed),
|
|
227
|
+
excluded_unmeasured,
|
|
228
|
+
_filters(min_usd, max_usd, failed_only, since)),
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _filters(min_usd, max_usd, failed_only, since):
|
|
233
|
+
"""The applied filters, in the order they are applied.
|
|
234
|
+
|
|
235
|
+
`is not None`, never truthiness: --min-usd 0 is a real filter, and reading
|
|
236
|
+
it as "no filter" would make the output line state something untrue.
|
|
237
|
+
"""
|
|
238
|
+
f = []
|
|
239
|
+
if min_usd is not None:
|
|
240
|
+
f.append("min-usd=%s" % min_usd)
|
|
241
|
+
if max_usd is not None:
|
|
242
|
+
f.append("max-usd=%s" % max_usd)
|
|
243
|
+
if failed_only:
|
|
244
|
+
f.append("failed-only")
|
|
245
|
+
if since is not None:
|
|
246
|
+
f.append("since=%s" % since)
|
|
247
|
+
return f
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _summary(scanned, matched, malformed, excluded, filters):
|
|
251
|
+
applied = ", ".join(filters) if filters else "none (no filter applied)"
|
|
252
|
+
if scanned == 0:
|
|
253
|
+
return ("NO RECEIPTS -- no proof.json found under this workspace, so "
|
|
254
|
+
"nothing was searched. This is not the same as zero matches.")
|
|
255
|
+
if matched == 0:
|
|
256
|
+
# "found", not "searched": a malformed receipt is FOUND and counted,
|
|
257
|
+
# and the malformed line below says it was NOT searched. Saying
|
|
258
|
+
# "searched" here would contradict it in the same sentence.
|
|
259
|
+
head = ("NO MATCHES -- %d receipt(s) found, none matched. The "
|
|
260
|
+
"filter ran; the archive is not empty." % scanned)
|
|
261
|
+
else:
|
|
262
|
+
head = "%d of %d receipt(s) matched." % (matched, scanned)
|
|
263
|
+
head += " Filters applied: %s." % applied
|
|
264
|
+
if excluded:
|
|
265
|
+
head += (" %d receipt(s) EXCLUDED from the cost filter: cost was never "
|
|
266
|
+
"measured, so it is neither above nor below the threshold."
|
|
267
|
+
% excluded)
|
|
268
|
+
if malformed:
|
|
269
|
+
head += (" %d receipt(s) could not be read and were NOT searched."
|
|
270
|
+
% malformed)
|
|
271
|
+
return head
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _render(report):
|
|
275
|
+
lines = []
|
|
276
|
+
for m in report["matches"]:
|
|
277
|
+
lines.append("%s [%s]" % (m["path"], ", ".join(m["matched"]) or "-"))
|
|
278
|
+
for bad in report["malformed"]:
|
|
279
|
+
lines.append("MALFORMED %s (%s)" % (bad["path"], bad["reason"]))
|
|
280
|
+
lines.append("")
|
|
281
|
+
lines.append(report["summary"])
|
|
282
|
+
return "\n".join(lines)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _since(value):
|
|
286
|
+
"""Rule 4: an unparseable --since is rejected, never guessed."""
|
|
287
|
+
try:
|
|
288
|
+
return datetime.date.fromisoformat(value).isoformat()
|
|
289
|
+
except ValueError:
|
|
290
|
+
raise argparse.ArgumentTypeError(
|
|
291
|
+
"not a YYYY-MM-DD date: %r. A date that cannot be parsed would "
|
|
292
|
+
"silently match everything or nothing." % value)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def main(argv=None):
|
|
296
|
+
ap = argparse.ArgumentParser(
|
|
297
|
+
description="Find receipts under a workspace by measurable criteria.")
|
|
298
|
+
ap.add_argument("workspace", nargs="?", default=".",
|
|
299
|
+
help="workspace to search for receipts (default: .)")
|
|
300
|
+
ap.add_argument("--min-usd", type=float, default=None,
|
|
301
|
+
help="only receipts whose measured cost is >= N")
|
|
302
|
+
ap.add_argument("--max-usd", type=float, default=None,
|
|
303
|
+
help="only receipts whose measured cost is <= N")
|
|
304
|
+
ap.add_argument("--failed-only", action="store_true",
|
|
305
|
+
help='only receipts whose headline is "%s"'
|
|
306
|
+
% FAILED_HEADLINE)
|
|
307
|
+
ap.add_argument("--since", type=_since, default=None,
|
|
308
|
+
metavar="YYYY-MM-DD",
|
|
309
|
+
help="only receipts generated on or after this UTC date")
|
|
310
|
+
ap.add_argument("--json", action="store_true",
|
|
311
|
+
help="emit the full report as JSON")
|
|
312
|
+
args = ap.parse_args(argv)
|
|
313
|
+
|
|
314
|
+
report = search(args.workspace, min_usd=args.min_usd,
|
|
315
|
+
max_usd=args.max_usd, failed_only=args.failed_only,
|
|
316
|
+
since=args.since)
|
|
317
|
+
|
|
318
|
+
if args.json:
|
|
319
|
+
print(json.dumps(report, indent=2))
|
|
320
|
+
else:
|
|
321
|
+
print(_render(report))
|
|
322
|
+
|
|
323
|
+
if report["scanned"] == 0:
|
|
324
|
+
return 3
|
|
325
|
+
return 0 if report["match_count"] else 1
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
if __name__ == "__main__":
|
|
329
|
+
sys.exit(main())
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fail a build when a run's TOKEN usage regressed. A token gate for CI.
|
|
3
|
+
|
|
4
|
+
tools/cost-guard.py gates on dollars. Dollars are a derived unit: they depend
|
|
5
|
+
on a per-provider price table that changes under you, so a fixed dollar
|
|
6
|
+
ceiling silently LOOSENS the moment a cheaper model is swapped in and tightens
|
|
7
|
+
on a dearer one, without the work having changed at all. Tokens are the
|
|
8
|
+
provider-independent unit of work actually done. Nothing gated on them.
|
|
9
|
+
|
|
10
|
+
THE RULE THAT MAKES IT A GATE RATHER THAN DECORATION:
|
|
11
|
+
|
|
12
|
+
UNMEASURED IS NOT WITHIN BUDGET. Exit 2, never 0.
|
|
13
|
+
|
|
14
|
+
A gate's exit code is a merge decision, and "we did not measure" carries zero
|
|
15
|
+
information about budget compliance. Exit 0 is reachable from exactly ONE
|
|
16
|
+
place in this file (the end of evaluate()), and only downstream of a real
|
|
17
|
+
measurement. No policy, no records, unmeasured tokens: all 2.
|
|
18
|
+
|
|
19
|
+
WHICH FIELDS COUNT AS MEASURED, and why it is not cost-guard's five.
|
|
20
|
+
Whether a number is measured is record_is_measured() in
|
|
21
|
+
autonomy/lib/efficiency_cost.py, imported and never restated. But that
|
|
22
|
+
predicate is satisfied by cost OR tokens, and this gate is about tokens, so it
|
|
23
|
+
is fed the FOUR TOKEN FIELDS ONLY. Feeding it cost_usd as well would be a hole
|
|
24
|
+
in this specific gate: a record carrying a real cost and no usage (the exact
|
|
25
|
+
shape codex wrote before v8.51.0) makes collect_efficiency return
|
|
26
|
+
available=True with integer-zero token counts, and this gate would report
|
|
27
|
+
"WITHIN BUDGET: 0 output tokens" for a run whose tokens were never recorded.
|
|
28
|
+
Verified, not theoretical. Choosing which fields to ask about is not a second
|
|
29
|
+
copy of the rule; the rule itself stays in one place.
|
|
30
|
+
|
|
31
|
+
CACHED READS ARE NOT FRESH INPUT, and conflating them is the defect to avoid.
|
|
32
|
+
This repo measured a single call at 10,651,759 cache-read tokens against
|
|
33
|
+
34,729 output tokens: a ratio near 300 to 1. A "total" dominated by cache
|
|
34
|
+
reads is a CONTEXT signal, and it moves when the prompt prefix or the cache
|
|
35
|
+
hit rate changes, not when the agent does more work. So:
|
|
36
|
+
|
|
37
|
+
--max-output-tokens the WORK signal. Output tokens alone: what the model
|
|
38
|
+
actually generated. Reach for this by default.
|
|
39
|
+
--max-total-tokens the CONTEXT+work signal. Sums the four fields named in
|
|
40
|
+
TOTAL_FIELDS, cache reads included, which is what makes
|
|
41
|
+
it big. The exact field list is PRINTED with the
|
|
42
|
+
verdict, on pass, on fail and in JSON, because a total
|
|
43
|
+
whose definition is not stated is not evidence.
|
|
44
|
+
|
|
45
|
+
Both may be given; each is judged on its own number and either can breach.
|
|
46
|
+
|
|
47
|
+
A genuinely measured ZERO under budget PASSES. Zero is falsy, so every guard
|
|
48
|
+
here is an explicit `is None` check -- on the measurement AND on the policy
|
|
49
|
+
arguments, since `--max-output-tokens 0` is a legitimate (harsh) policy and a
|
|
50
|
+
falsy check would read it as no policy at all.
|
|
51
|
+
|
|
52
|
+
Usage:
|
|
53
|
+
tools/token-guard.py [workspace] --max-output-tokens 200000
|
|
54
|
+
tools/token-guard.py [workspace] --max-total-tokens 5000000
|
|
55
|
+
tools/token-guard.py [workspace] --max-output-tokens 200000 --json
|
|
56
|
+
|
|
57
|
+
Exit: 0 within budget, 1 over budget, 2 cannot evaluate.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
import argparse
|
|
61
|
+
import json
|
|
62
|
+
import os
|
|
63
|
+
import sys
|
|
64
|
+
|
|
65
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
66
|
+
sys.path.insert(0, os.path.join(os.path.dirname(_HERE), "autonomy", "lib"))
|
|
67
|
+
|
|
68
|
+
from efficiency_cost import collect_efficiency, record_is_measured # noqa: E402
|
|
69
|
+
|
|
70
|
+
OK, OVER, CANNOT = 0, 1, 2
|
|
71
|
+
|
|
72
|
+
# The four fields --max-total-tokens sums, and the SAME tuple the printed
|
|
73
|
+
# definition is built from. One constant so the number and its stated meaning
|
|
74
|
+
# cannot drift apart: a total that sums something other than what it claims is
|
|
75
|
+
# the dishonesty this flag is most exposed to.
|
|
76
|
+
TOTAL_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens",
|
|
77
|
+
"cache_creation_tokens")
|
|
78
|
+
|
|
79
|
+
# Measured-ness for a TOKEN gate. cost_usd is deliberately absent; see the
|
|
80
|
+
# module docstring.
|
|
81
|
+
MEASURED_FIELDS = TOTAL_FIELDS
|
|
82
|
+
|
|
83
|
+
TOTAL_DEFINITION = "total = " + " + ".join(TOTAL_FIELDS)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _num(v):
|
|
87
|
+
"""A number as itself; None, "", or a bool as None.
|
|
88
|
+
|
|
89
|
+
The bool exclusion is defence in depth ONLY, and does not reach the CLI
|
|
90
|
+
path: collect_efficiency() runs _to_int() over every field first, so a
|
|
91
|
+
JSON `true` is already an int 1 by the time it arrives here. Stated
|
|
92
|
+
because an untrue claim of bool-safety is worse than a known gap; the fix
|
|
93
|
+
belongs in _to_int(), upstream. This guard still holds for a cost block
|
|
94
|
+
handed to measured_tokens() directly.
|
|
95
|
+
"""
|
|
96
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
97
|
+
return None
|
|
98
|
+
return v
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def measured_tokens(cost):
|
|
102
|
+
"""{field: value} for the four token fields, or None when unmeasured.
|
|
103
|
+
|
|
104
|
+
None means the run's tokens were never observed. It is NOT zero, and the
|
|
105
|
+
caller must not treat it as zero.
|
|
106
|
+
"""
|
|
107
|
+
if not isinstance(cost, dict):
|
|
108
|
+
return None
|
|
109
|
+
rec = {field: _num(cost.get(field)) for field in MEASURED_FIELDS}
|
|
110
|
+
if not record_is_measured(rec):
|
|
111
|
+
return None
|
|
112
|
+
# Measured, so a None from a partially-written record is a real absence in
|
|
113
|
+
# a real measurement; count it as 0 for summing rather than poisoning the
|
|
114
|
+
# arithmetic.
|
|
115
|
+
return {field: (0 if rec[field] is None else rec[field])
|
|
116
|
+
for field in MEASURED_FIELDS}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _loki_dir(workspace):
|
|
120
|
+
"""Accept either a workspace root or a .loki dir; collect_ wants .loki."""
|
|
121
|
+
if os.path.basename(os.path.normpath(workspace)) == ".loki":
|
|
122
|
+
return workspace
|
|
123
|
+
return os.path.join(workspace, ".loki")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _cannot(why):
|
|
127
|
+
return {"status": "cannot_evaluate", "exit_code": CANNOT, "why": why,
|
|
128
|
+
"output_tokens": None, "total_tokens": None, "tokens": None,
|
|
129
|
+
"total_definition": TOTAL_DEFINITION, "reason": None}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def evaluate(workspace, max_output_tokens=None, max_total_tokens=None):
|
|
133
|
+
"""Decide the gate. Returns a dict carrying exit_code and the numbers.
|
|
134
|
+
|
|
135
|
+
Every early return is CANNOT (2). The single exit-0 return is at the very
|
|
136
|
+
bottom, downstream of a measurement.
|
|
137
|
+
"""
|
|
138
|
+
# `is None`, not falsy: --max-output-tokens 0 is a real policy.
|
|
139
|
+
if max_output_tokens is None and max_total_tokens is None:
|
|
140
|
+
return _cannot(
|
|
141
|
+
"no token policy given: pass --max-output-tokens and/or "
|
|
142
|
+
"--max-total-tokens. A gate with no policy checks nothing, and "
|
|
143
|
+
"must not report a pass for it.")
|
|
144
|
+
|
|
145
|
+
cost, _model = collect_efficiency(_loki_dir(workspace))
|
|
146
|
+
tokens = measured_tokens(cost)
|
|
147
|
+
if tokens is None:
|
|
148
|
+
# THE POINT OF THIS FILE. Absence is not compliance.
|
|
149
|
+
return _cannot(
|
|
150
|
+
"tokens are UNMEASURED for %s -- no efficiency record carried an "
|
|
151
|
+
"observed token count. Unmeasured is not within budget: this gate "
|
|
152
|
+
"cannot say whether the run complied, so it reports no verdict "
|
|
153
|
+
"rather than a green one." % workspace)
|
|
154
|
+
|
|
155
|
+
output = tokens["output_tokens"]
|
|
156
|
+
total = sum(tokens[field] for field in TOTAL_FIELDS)
|
|
157
|
+
|
|
158
|
+
breaches = []
|
|
159
|
+
# Tokens are integers, so a strict `>` is exact. No float tolerance: there
|
|
160
|
+
# is no binary-representation slop to absorb, and a tolerance here would
|
|
161
|
+
# silently widen the ceiling the operator asked for.
|
|
162
|
+
if max_output_tokens is not None and output > max_output_tokens:
|
|
163
|
+
breaches.append(
|
|
164
|
+
"output tokens %d exceed the ceiling %d by %d (%.1f%% over)"
|
|
165
|
+
% (output, max_output_tokens, output - max_output_tokens,
|
|
166
|
+
((output - max_output_tokens) / max_output_tokens * 100.0)
|
|
167
|
+
if max_output_tokens else 0.0))
|
|
168
|
+
|
|
169
|
+
if max_total_tokens is not None and total > max_total_tokens:
|
|
170
|
+
breaches.append(
|
|
171
|
+
"total tokens %d exceed the ceiling %d by %d (%.1f%% over) [%s]"
|
|
172
|
+
% (total, max_total_tokens, total - max_total_tokens,
|
|
173
|
+
((total - max_total_tokens) / max_total_tokens * 100.0)
|
|
174
|
+
if max_total_tokens else 0.0, TOTAL_DEFINITION))
|
|
175
|
+
|
|
176
|
+
verdict = {"output_tokens": output, "total_tokens": total,
|
|
177
|
+
"tokens": dict(tokens), "total_definition": TOTAL_DEFINITION,
|
|
178
|
+
"why": None}
|
|
179
|
+
if breaches:
|
|
180
|
+
verdict.update({"status": "over_budget", "exit_code": OVER,
|
|
181
|
+
"reason": "; ".join(breaches)})
|
|
182
|
+
return verdict
|
|
183
|
+
verdict.update({"status": "within_budget", "exit_code": OK,
|
|
184
|
+
"reason": None})
|
|
185
|
+
return verdict
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def render(d):
|
|
189
|
+
if d["status"] == "cannot_evaluate":
|
|
190
|
+
return "CANNOT EVALUATE: %s" % d["why"]
|
|
191
|
+
if d["status"] == "over_budget":
|
|
192
|
+
return "OVER BUDGET: %s" % d["reason"]
|
|
193
|
+
return ("WITHIN BUDGET: measured %d output tokens, %d total tokens [%s]"
|
|
194
|
+
% (d["output_tokens"], d["total_tokens"], d["total_definition"]))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def main(argv=None):
|
|
198
|
+
ap = argparse.ArgumentParser(
|
|
199
|
+
description="Fail CI when a run's token usage regressed past a "
|
|
200
|
+
"budget policy.")
|
|
201
|
+
ap.add_argument("workspace", nargs="?", default=".",
|
|
202
|
+
help="workspace root (or its .loki dir); default .")
|
|
203
|
+
ap.add_argument("--max-output-tokens", type=int,
|
|
204
|
+
help="ceiling on output tokens alone (the work signal)")
|
|
205
|
+
ap.add_argument("--max-total-tokens", type=int,
|
|
206
|
+
help="ceiling on %s (cache reads dominate this)"
|
|
207
|
+
% TOTAL_DEFINITION)
|
|
208
|
+
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
209
|
+
help="emit the verdict as JSON")
|
|
210
|
+
args = ap.parse_args(argv)
|
|
211
|
+
|
|
212
|
+
d = evaluate(args.workspace, args.max_output_tokens, args.max_total_tokens)
|
|
213
|
+
print(json.dumps(d, indent=2) if args.as_json else render(d))
|
|
214
|
+
return d["exit_code"]
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
if __name__ == "__main__":
|
|
218
|
+
sys.exit(main())
|