loki-mode 8.83.0 → 8.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/loki +20 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/receipt-diff.py +302 -0
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v8.
|
|
6
|
+
# Loki Mode v8.85.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -469,4 +469,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
469
469
|
|
|
470
470
|
---
|
|
471
471
|
|
|
472
|
-
**v8.
|
|
472
|
+
**v8.85.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
8.
|
|
1
|
+
8.85.0
|
package/autonomy/loki
CHANGED
|
@@ -33749,6 +33749,21 @@ PYEOF
|
|
|
33749
33749
|
# inconsistent edit is caught. It does NOT prove non-forgeability on
|
|
33750
33750
|
# the unsigned path (a hand-forger can rewrite facts and re-hash);
|
|
33751
33751
|
# neutral non-forgeability requires the signed record.
|
|
33752
|
+
# --human renders the verdict plus every failure reason as prose.
|
|
33753
|
+
# JSON stays the DEFAULT so machine consumers, which pipe this
|
|
33754
|
+
# stdout verbatim, are unaffected. The flag is STRIPPED from the
|
|
33755
|
+
# positional args rather than assumed at a fixed index, so it works
|
|
33756
|
+
# before or after the id.
|
|
33757
|
+
local _pv_human=0 _pv_args=()
|
|
33758
|
+
local _pv_a
|
|
33759
|
+
for _pv_a in "$@"; do
|
|
33760
|
+
case "$_pv_a" in
|
|
33761
|
+
--human) _pv_human=1 ;;
|
|
33762
|
+
*) _pv_args+=("$_pv_a") ;;
|
|
33763
|
+
esac
|
|
33764
|
+
done
|
|
33765
|
+
set -- "${_pv_args[@]+"${_pv_args[@]}"}"
|
|
33766
|
+
|
|
33752
33767
|
local id="${1:-}"
|
|
33753
33768
|
if [ -z "$id" ]; then
|
|
33754
33769
|
echo -e "${RED}Missing proof id.${NC} Use 'loki proof list'."
|
|
@@ -33767,7 +33782,11 @@ PYEOF
|
|
|
33767
33782
|
echo -e "${RED}Verifier not found (autonomy/lib/proof-verify.py).${NC}"
|
|
33768
33783
|
exit 2
|
|
33769
33784
|
fi
|
|
33770
|
-
|
|
33785
|
+
if [ "$_pv_human" -eq 1 ]; then
|
|
33786
|
+
python3 "$verifier" --human "$pj" "${TARGET_DIR:-.}"
|
|
33787
|
+
else
|
|
33788
|
+
python3 "$verifier" "$pj" "${TARGET_DIR:-.}"
|
|
33789
|
+
fi
|
|
33771
33790
|
exit $?
|
|
33772
33791
|
;;
|
|
33773
33792
|
open)
|
package/dashboard/__init__.py
CHANGED
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="8.
|
|
2
|
+
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="8.85.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Xf(Qf(import.meta.url)),Q=X$(X);h5=e_(Zf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var bO={};l0(bO,{runOrThrow:()=>jf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Tf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>yO});async function NQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function jf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Mf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Mf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Tf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return wf?"":Z}var wf,L0,F8,p0,zV0,a0,W8,Q9,v;var S6=p(()=>{wf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),zV0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),W8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as bf}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(bf(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>Kh});import{existsSync as Y9,readFileSync as g3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as rf}from"path";import{homedir as tf}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*LQ/X);if(J>LQ)J=LQ;let z=LQ-J,K=F8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${W8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function Zh(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1232,4 +1232,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1232
1232
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (h_(),f_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1233
1233
|
`),process.stderr.write(v_),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var uW0=await mW0(Bun.argv.slice(2));process.exit(uW0);
|
|
1234
1234
|
|
|
1235
|
-
//# debugId=
|
|
1235
|
+
//# debugId=4F01D2A6D41F31AE64756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "8.
|
|
4
|
+
"version": "8.85.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "8.
|
|
5
|
+
"version": "8.85.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Compare two Evidence Receipts and report what actually changed between runs.
|
|
3
|
+
|
|
4
|
+
Verification of ONE receipt already exists (autonomy/lib/proof-verify.py).
|
|
5
|
+
This is the missing half: two runs, side by side. "Did the change make it
|
|
6
|
+
cheaper?" is the question a cost receipt exists to answer, and it cannot be
|
|
7
|
+
answered from a single receipt.
|
|
8
|
+
|
|
9
|
+
THREE RULES, and they are the whole point of this file:
|
|
10
|
+
|
|
11
|
+
1. A delta against an UNMEASURED value is UNKNOWN, never a number. If run A
|
|
12
|
+
measured cost and run B did not, "-$0.42" is a fabricated saving. The
|
|
13
|
+
measured/unmeasured predicate is record_is_measured() in
|
|
14
|
+
autonomy/lib/efficiency_cost.py -- imported, never restated, because a
|
|
15
|
+
second copy of that predicate is precisely how the honesty rule drifts.
|
|
16
|
+
|
|
17
|
+
2. A genuinely measured ZERO delta reads 0. Zero is falsy, so every guard in
|
|
18
|
+
this file is an explicit `is None` check. A falsy guard turns a real
|
|
19
|
+
"identical cost" finding into UNKNOWN, which is the same lie pointed the
|
|
20
|
+
other way.
|
|
21
|
+
|
|
22
|
+
3. Integrity comes FIRST. A tampered receipt must not silently supply half a
|
|
23
|
+
delta, so verify_integrity() runs on both before anything is compared.
|
|
24
|
+
|
|
25
|
+
Non-comparable receipts (different specs) are REFUSED rather than diffed. The
|
|
26
|
+
delta between a landing page and a compiler is arithmetic, not information.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import importlib.util
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import sys
|
|
34
|
+
|
|
35
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
36
|
+
_LIB = os.path.join(os.path.dirname(_HERE), "autonomy", "lib")
|
|
37
|
+
sys.path.insert(0, _LIB)
|
|
38
|
+
|
|
39
|
+
from efficiency_cost import record_is_measured # noqa: E402
|
|
40
|
+
|
|
41
|
+
# proof-verify.py has a hyphen and is not importable as a module; load it by
|
|
42
|
+
# path the same way tests/test_cost_honesty_end_to_end.py does.
|
|
43
|
+
_spec = importlib.util.spec_from_file_location(
|
|
44
|
+
"proof_verify", os.path.join(_LIB, "proof-verify.py"))
|
|
45
|
+
_pv = importlib.util.module_from_spec(_spec)
|
|
46
|
+
_spec.loader.exec_module(_pv)
|
|
47
|
+
|
|
48
|
+
verify_integrity = _pv.verify_integrity
|
|
49
|
+
|
|
50
|
+
UNKNOWN = "UNKNOWN"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class NotComparable(Exception):
|
|
54
|
+
"""Raised when two receipts must not be diffed at all."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def delta(a, b):
|
|
58
|
+
"""b - a, or UNKNOWN when either side was never measured.
|
|
59
|
+
|
|
60
|
+
`is None` is load-bearing: a measured 0.0 on either side is a real
|
|
61
|
+
observation and must survive as a number.
|
|
62
|
+
"""
|
|
63
|
+
if a is None or b is None:
|
|
64
|
+
return UNKNOWN
|
|
65
|
+
return b - a
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _num(v):
|
|
69
|
+
"""A number as itself, anything else (None, "", bool) as None."""
|
|
70
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
71
|
+
return None
|
|
72
|
+
return v
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def measured_cost(proof):
|
|
76
|
+
"""The cost block as {usd, tokens...}, or None when never measured.
|
|
77
|
+
|
|
78
|
+
cost.available is NOT trusted on its own. A real receipt shipped
|
|
79
|
+
available=true with every field zero (fixed v8.52.0), so the flag is a
|
|
80
|
+
claim, not evidence. record_is_measured reads the VALUES, under the
|
|
81
|
+
per-iteration key names, so the receipt's `usd` is mapped to `cost_usd`.
|
|
82
|
+
"""
|
|
83
|
+
cost = proof.get("cost")
|
|
84
|
+
if not isinstance(cost, dict):
|
|
85
|
+
return None
|
|
86
|
+
rec = {
|
|
87
|
+
"cost_usd": _num(cost.get("usd")),
|
|
88
|
+
"input_tokens": _num(cost.get("input_tokens")),
|
|
89
|
+
"output_tokens": _num(cost.get("output_tokens")),
|
|
90
|
+
"cache_read_tokens": _num(cost.get("cache_read_tokens")),
|
|
91
|
+
"cache_creation_tokens": _num(cost.get("cache_creation_tokens")),
|
|
92
|
+
}
|
|
93
|
+
return rec if record_is_measured(rec) else None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def cache_hit_ratio(cost):
|
|
97
|
+
"""cache_read / (cache_read + input): the metric that dominates cost.
|
|
98
|
+
|
|
99
|
+
None when unmeasured OR when the denominator is zero -- no tokens read
|
|
100
|
+
means no ratio was observed, which is not a 0% hit rate.
|
|
101
|
+
"""
|
|
102
|
+
if cost is None:
|
|
103
|
+
return None
|
|
104
|
+
read = _num(cost.get("cache_read_tokens"))
|
|
105
|
+
fresh = _num(cost.get("input_tokens"))
|
|
106
|
+
if read is None or fresh is None:
|
|
107
|
+
return None
|
|
108
|
+
total = read + fresh
|
|
109
|
+
if total == 0:
|
|
110
|
+
return None
|
|
111
|
+
return read / total
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def gate_map(proof):
|
|
115
|
+
"""{gate name: status} from the facts projection, falling back to flat."""
|
|
116
|
+
facts = proof.get("facts")
|
|
117
|
+
gates = facts.get("quality_gates") if isinstance(facts, dict) else None
|
|
118
|
+
if not isinstance(gates, list):
|
|
119
|
+
qg = proof.get("quality_gates")
|
|
120
|
+
gates = qg.get("gates") if isinstance(qg, dict) else None
|
|
121
|
+
out = {}
|
|
122
|
+
for g in gates or []:
|
|
123
|
+
if isinstance(g, dict) and g.get("name"):
|
|
124
|
+
out[str(g["name"])] = str(g.get("status", "not_run"))
|
|
125
|
+
return out
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _spec_id(proof):
|
|
129
|
+
spec = proof.get("spec")
|
|
130
|
+
return str(spec.get("source") or "") if isinstance(spec, dict) else ""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def compare(a, b, a_path="A", b_path="B"):
|
|
134
|
+
"""Diff two verified receipts. Raises NotComparable when they must not be.
|
|
135
|
+
|
|
136
|
+
Integrity first: a receipt that fails verify_integrity never reaches the
|
|
137
|
+
arithmetic.
|
|
138
|
+
"""
|
|
139
|
+
for proof, path in ((a, a_path), (b, b_path)):
|
|
140
|
+
v = verify_integrity(proof)
|
|
141
|
+
if not v["ok"]:
|
|
142
|
+
raise NotComparable(
|
|
143
|
+
"receipt failed integrity verification: %s\n %s"
|
|
144
|
+
% (path, "\n ".join(v["reasons"] or [v["reason"]])))
|
|
145
|
+
|
|
146
|
+
spec_a, spec_b = _spec_id(a), _spec_id(b)
|
|
147
|
+
if spec_a and spec_b and spec_a != spec_b:
|
|
148
|
+
raise NotComparable(
|
|
149
|
+
"receipts are for different specs, so no delta between them is "
|
|
150
|
+
"meaningful:\n %s: %s\n %s: %s" % (a_path, spec_a, b_path, spec_b))
|
|
151
|
+
|
|
152
|
+
cost_a, cost_b = measured_cost(a), measured_cost(b)
|
|
153
|
+
# WHY each field is unavailable, so a reader never has to guess whether
|
|
154
|
+
# UNKNOWN means "not measured" or "tool could not read it".
|
|
155
|
+
incomparable = []
|
|
156
|
+
if cost_a is None or cost_b is None:
|
|
157
|
+
incomparable.append({
|
|
158
|
+
"field": "cost",
|
|
159
|
+
"why": "cost was not measured in %s" % (
|
|
160
|
+
" and ".join(
|
|
161
|
+
[p for p, c in ((a_path, cost_a), (b_path, cost_b))
|
|
162
|
+
if c is None])),
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
ratio_a, ratio_b = cache_hit_ratio(cost_a), cache_hit_ratio(cost_b)
|
|
166
|
+
if ratio_a is None or ratio_b is None:
|
|
167
|
+
incomparable.append({
|
|
168
|
+
"field": "cache_hit_ratio",
|
|
169
|
+
"why": "no cached-plus-fresh input tokens recorded, so no ratio "
|
|
170
|
+
"was observed (this is not a 0% hit rate)",
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
def iters(p):
|
|
174
|
+
it = p.get("iterations")
|
|
175
|
+
return _num(it.get("count")) if isinstance(it, dict) else None
|
|
176
|
+
|
|
177
|
+
it_a, it_b = iters(a), iters(b)
|
|
178
|
+
if it_a is None or it_b is None:
|
|
179
|
+
incomparable.append(
|
|
180
|
+
{"field": "iterations", "why": "iteration count not recorded"})
|
|
181
|
+
|
|
182
|
+
dur_a, dur_b = _num(a.get("wall_clock_sec")), _num(b.get("wall_clock_sec"))
|
|
183
|
+
if dur_a is None or dur_b is None:
|
|
184
|
+
incomparable.append(
|
|
185
|
+
{"field": "duration_sec", "why": "wall clock not recorded"})
|
|
186
|
+
|
|
187
|
+
ga, gb = gate_map(a), gate_map(b)
|
|
188
|
+
gates = {"regressed": [], "fixed": [], "added": [], "removed": []}
|
|
189
|
+
for name in sorted(set(ga) | set(gb)):
|
|
190
|
+
sa, sb = ga.get(name), gb.get(name)
|
|
191
|
+
if sa is None:
|
|
192
|
+
# Absent in A is ADDED, never a pass that broke.
|
|
193
|
+
gates["added"].append({"gate": name, "status": sb})
|
|
194
|
+
elif sb is None:
|
|
195
|
+
gates["removed"].append({"gate": name, "status": sa})
|
|
196
|
+
elif sa != sb:
|
|
197
|
+
bucket = "fixed" if sb == "passed" else "regressed"
|
|
198
|
+
gates[bucket].append({"gate": name, "from": sa, "to": sb})
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
"comparable": True,
|
|
202
|
+
"a": a_path,
|
|
203
|
+
"b": b_path,
|
|
204
|
+
"spec": spec_a or spec_b,
|
|
205
|
+
"cost_usd": {
|
|
206
|
+
"a": cost_a["cost_usd"] if cost_a else None,
|
|
207
|
+
"b": cost_b["cost_usd"] if cost_b else None,
|
|
208
|
+
"delta": delta(cost_a["cost_usd"] if cost_a else None,
|
|
209
|
+
cost_b["cost_usd"] if cost_b else None),
|
|
210
|
+
},
|
|
211
|
+
"cache_hit_ratio": {
|
|
212
|
+
"a": ratio_a, "b": ratio_b, "delta": delta(ratio_a, ratio_b),
|
|
213
|
+
},
|
|
214
|
+
"iterations": {"a": it_a, "b": it_b, "delta": delta(it_a, it_b)},
|
|
215
|
+
"duration_sec": {"a": dur_a, "b": dur_b, "delta": delta(dur_a, dur_b)},
|
|
216
|
+
"gates": gates,
|
|
217
|
+
"not_comparable": incomparable,
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _fmt(value, unit=""):
|
|
222
|
+
if value == UNKNOWN or value is None:
|
|
223
|
+
return UNKNOWN
|
|
224
|
+
if unit == "$":
|
|
225
|
+
return "%s$%.4f" % ("+" if value >= 0 else "-", abs(value))
|
|
226
|
+
if unit == "%":
|
|
227
|
+
return "%+.1f%%" % (value * 100)
|
|
228
|
+
return "%+g%s" % (value, unit)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def render(d):
|
|
232
|
+
lines = ["Evidence Receipt diff", " A: %s" % d["a"], " B: %s" % d["b"]]
|
|
233
|
+
if d["spec"]:
|
|
234
|
+
lines.append(" spec: %s" % d["spec"])
|
|
235
|
+
lines.append("")
|
|
236
|
+
for label, key, unit in (
|
|
237
|
+
("cost", "cost_usd", "$"),
|
|
238
|
+
("cache hit ratio", "cache_hit_ratio", "%"),
|
|
239
|
+
("iterations", "iterations", ""),
|
|
240
|
+
("duration", "duration_sec", "s"),
|
|
241
|
+
):
|
|
242
|
+
f = d[key]
|
|
243
|
+
a = UNKNOWN if f["a"] is None else (
|
|
244
|
+
"%.4f" % f["a"] if unit == "%" else f["a"])
|
|
245
|
+
b = UNKNOWN if f["b"] is None else (
|
|
246
|
+
"%.4f" % f["b"] if unit == "%" else f["b"])
|
|
247
|
+
lines.append(" %-16s %s -> %s delta %s"
|
|
248
|
+
% (label, a, b, _fmt(f["delta"], unit)))
|
|
249
|
+
|
|
250
|
+
g = d["gates"]
|
|
251
|
+
lines.append("")
|
|
252
|
+
if any(g.values()):
|
|
253
|
+
for gate in g["regressed"]:
|
|
254
|
+
lines.append(" REGRESSED %s: %s -> %s"
|
|
255
|
+
% (gate["gate"], gate["from"], gate["to"]))
|
|
256
|
+
for gate in g["fixed"]:
|
|
257
|
+
lines.append(" FIXED %s: %s -> %s"
|
|
258
|
+
% (gate["gate"], gate["from"], gate["to"]))
|
|
259
|
+
for gate in g["added"]:
|
|
260
|
+
lines.append(" ADDED %s (%s), absent in A"
|
|
261
|
+
% (gate["gate"], gate["status"]))
|
|
262
|
+
for gate in g["removed"]:
|
|
263
|
+
lines.append(" REMOVED %s (was %s), absent in B"
|
|
264
|
+
% (gate["gate"], gate["status"]))
|
|
265
|
+
else:
|
|
266
|
+
lines.append(" gates no verdict changes")
|
|
267
|
+
|
|
268
|
+
if d["not_comparable"]:
|
|
269
|
+
lines.append("")
|
|
270
|
+
lines.append(" NOT COMPARABLE (reported UNKNOWN, not zero):")
|
|
271
|
+
for item in d["not_comparable"]:
|
|
272
|
+
lines.append(" %s: %s" % (item["field"], item["why"]))
|
|
273
|
+
return "\n".join(lines)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def main(argv=None):
|
|
277
|
+
ap = argparse.ArgumentParser(
|
|
278
|
+
description="Compare two Evidence Receipts (proof.json).")
|
|
279
|
+
ap.add_argument("a", help="baseline proof.json")
|
|
280
|
+
ap.add_argument("b", help="proof.json to compare against the baseline")
|
|
281
|
+
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
282
|
+
help="emit the diff as JSON")
|
|
283
|
+
args = ap.parse_args(argv)
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
pa = _pv._load_proof(args.a)
|
|
287
|
+
pb = _pv._load_proof(args.b)
|
|
288
|
+
d = compare(pa, pb, args.a, args.b)
|
|
289
|
+
except (NotComparable, _pv.ProofLoadError) as exc:
|
|
290
|
+
payload = {"comparable": False, "reason": str(exc)}
|
|
291
|
+
if args.as_json:
|
|
292
|
+
print(json.dumps(payload, indent=2))
|
|
293
|
+
else:
|
|
294
|
+
print("REFUSED: %s" % exc)
|
|
295
|
+
return 2
|
|
296
|
+
|
|
297
|
+
print(json.dumps(d, indent=2) if args.as_json else render(d))
|
|
298
|
+
return 0
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
if __name__ == "__main__":
|
|
302
|
+
sys.exit(main())
|