loki-mode 8.95.0 → 8.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/cost-history.py +317 -0
- package/tools/model-advisor.py +13 -2
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v8.
|
|
6
|
+
# Loki Mode v8.96.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -469,4 +469,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
469
469
|
|
|
470
470
|
---
|
|
471
471
|
|
|
472
|
-
**v8.
|
|
472
|
+
**v8.96.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
8.
|
|
1
|
+
8.96.0
|
package/dashboard/__init__.py
CHANGED
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="8.
|
|
2
|
+
var m_=Object.create;var{getPrototypeOf:u_,defineProperty:eK,getOwnPropertyNames:p_}=Object;var d_=Object.prototype.hasOwnProperty;function c_(Z){return this[Z]}var l_,i_,a_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?l_??=new WeakMap:i_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?m_(u_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of p_(Z))if(!d_.call(K,$))eK(K,$,{get:c_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var s_=(Z)=>Z;function n_(Z,X){this[Z]=s_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:n_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var e0=import.meta.require;var kO={};l0(kO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as o_}from"url";import{existsSync as UQ}from"fs";import{homedir as r_}from"os";function t_(){let Z=RO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(RO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(r_(),".loki")}var RO,i0;var H8=p(()=>{RO=Z$(o_(import.meta.url));i0=t_()});import{readFileSync as e_}from"fs";import{resolve as Zf,dirname as Xf}from"path";import{fileURLToPath as Qf}from"url";function h3(){if(h5!==null)return h5;let Z="8.96.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Xf(Qf(import.meta.url)),Q=X$(X);h5=e_(Zf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var bO={};l0(bO,{runOrThrow:()=>jf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Tf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>yO});async function NQ(Z,X=yO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let W=$.byteLength-(z-X);J+=Y.decode($.subarray(0,W),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function jf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=Mf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function Mf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Tf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var yO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return wf?"":Z}var wf,L0,F8,p0,zV0,a0,W8,Q9,v;var S6=p(()=>{wf=(process.env.NO_COLOR??"").length>0;L0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),p0=o7("\x1B[1;33m"),zV0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),W8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),v=o7("\x1B[0m")});import{existsSync as bf}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(bf(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var ZL={};l0(ZL,{runStatus:()=>Kh});import{existsSync as Y9,readFileSync as g3,readdirSync as iO,statSync as aO}from"fs";import{resolve as h8,basename as rf}from"path";import{homedir as tf}from"os";function sO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function nO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*LQ/X);if(J>LQ)J=LQ;let z=LQ-J,K=F8;if(Y>=80)K=L0;else if(Y>=50)K=p0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),W=sO(Z),V=sO(X);return` ${W8}${Q}${v} ${K}[${$}]${v} ${Y}% (${W} / ${V})`}async function Zh(){if(await X9("jq"))return!0;return process.stdout.write(`${L0}Error: jq is required but not installed.${v}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1232,4 +1232,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1232
1232
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (h_(),f_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1233
1233
|
`),process.stderr.write(v_),2}}cO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var uW0=await mW0(Bun.argv.slice(2));process.exit(uW0);
|
|
1234
1234
|
|
|
1235
|
-
//# debugId=
|
|
1235
|
+
//# debugId=3B5AE9CBA3FA521064756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "8.
|
|
4
|
+
"version": "8.96.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "8.
|
|
5
|
+
"version": "8.96.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Track agent cost across MANY runs and report the trend.
|
|
3
|
+
|
|
4
|
+
cost-guard.py gates ONE run against a ceiling. receipt-diff.py compares TWO
|
|
5
|
+
runs. Neither answers the question a team actually asks at the end of a month:
|
|
6
|
+
is our agent spend trending up? That needs a durable history, so this appends
|
|
7
|
+
one line per run to a JSONL file and reports over the whole series.
|
|
8
|
+
|
|
9
|
+
THE FIVE PROPERTIES THAT MAKE THE TREND WORTH BELIEVING:
|
|
10
|
+
|
|
11
|
+
1. AN UNMEASURED RUN IS NEVER RECORDED AS 0. It is appended with usd=null and
|
|
12
|
+
excluded from the trend. Recording zero would drag the median down and
|
|
13
|
+
understate real spend -- the exact "$0.00 means free" lie this repo spent
|
|
14
|
+
thirteen surfaces removing. Whether a number counts as measured is
|
|
15
|
+
record_is_measured() in autonomy/lib/efficiency_cost.py, imported and never
|
|
16
|
+
restated; receipts store the figure under cost.usd while the predicate reads
|
|
17
|
+
cost_usd, so the KEY is mapped here. Mapping a key name is not restating the
|
|
18
|
+
rule.
|
|
19
|
+
|
|
20
|
+
RECORD-NULL rather than REFUSE, deliberately. Refusing is cost-guard.py's
|
|
21
|
+
job and it already does it, per run, with an exit code. Here the history IS
|
|
22
|
+
the product, and a refused append leaves no trace: a month with four broken
|
|
23
|
+
instrumentation runs would look identical to a month with eight clean ones.
|
|
24
|
+
The null row is countable, so the report can say "12 runs, 8 measured" and
|
|
25
|
+
the operator can see the measurement gap instead of inferring it from a
|
|
26
|
+
short file.
|
|
27
|
+
|
|
28
|
+
2. A TREND FROM ONE POINT IS NOT A TREND. Fewer than 2 measured runs reads
|
|
29
|
+
INSUFFICIENT DATA. "Flat" is a claim about change over time and one
|
|
30
|
+
observation cannot support it.
|
|
31
|
+
|
|
32
|
+
3. APPEND IS ATOMIC PER LINE. See _append().
|
|
33
|
+
|
|
34
|
+
4. A CORRUPT LINE IS COUNTED AND REPORTED, never silently skipped. A history
|
|
35
|
+
that quietly drops rows reports a cleaner trend than reality, and drops them
|
|
36
|
+
most often when something upstream is broken -- precisely when the number
|
|
37
|
+
matters.
|
|
38
|
+
|
|
39
|
+
5. AN EMPTY HISTORY IS NOT A FLAT TREND. It exits non-zero. Zero runs is an
|
|
40
|
+
absent measurement, not evidence of stability.
|
|
41
|
+
|
|
42
|
+
Usage:
|
|
43
|
+
tools/cost-history.py record [workspace] [--file .loki/cost-history.jsonl]
|
|
44
|
+
tools/cost-history.py report [--file ...] [--json]
|
|
45
|
+
|
|
46
|
+
Exit: 0 report produced / run recorded, 1 nothing to report (empty or no
|
|
47
|
+
measured runs), 2 cannot record.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
import argparse
|
|
51
|
+
import json
|
|
52
|
+
import os
|
|
53
|
+
import sys
|
|
54
|
+
import time
|
|
55
|
+
|
|
56
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
57
|
+
sys.path.insert(0, os.path.join(os.path.dirname(_HERE), "autonomy", "lib"))
|
|
58
|
+
|
|
59
|
+
from efficiency_cost import collect_efficiency, record_is_measured # noqa: E402
|
|
60
|
+
|
|
61
|
+
OK, NOTHING, CANNOT = 0, 1, 2
|
|
62
|
+
|
|
63
|
+
DEFAULT_FILE = os.path.join(".loki", "cost-history.jsonl")
|
|
64
|
+
|
|
65
|
+
# Below this the two halves are the same number to the cent, and calling that
|
|
66
|
+
# a direction is noise dressed as signal.
|
|
67
|
+
FLAT_PCT = 5.0
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _num(v):
|
|
71
|
+
"""A number as itself; None, "", or a bool as None."""
|
|
72
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
73
|
+
return None
|
|
74
|
+
return v
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def measured_usd(cost):
|
|
78
|
+
"""The USD figure from a cost block, or None when it was never measured.
|
|
79
|
+
|
|
80
|
+
`available` is NOT trusted on its own: a real receipt shipped
|
|
81
|
+
available=true with every field zero (v8.52.0), so the flag is a claim and
|
|
82
|
+
the VALUES are the evidence.
|
|
83
|
+
"""
|
|
84
|
+
if not isinstance(cost, dict):
|
|
85
|
+
return None
|
|
86
|
+
rec = {
|
|
87
|
+
"cost_usd": _num(cost.get("usd")),
|
|
88
|
+
"input_tokens": _num(cost.get("input_tokens")),
|
|
89
|
+
"output_tokens": _num(cost.get("output_tokens")),
|
|
90
|
+
"cache_read_tokens": _num(cost.get("cache_read_tokens")),
|
|
91
|
+
"cache_creation_tokens": _num(cost.get("cache_creation_tokens")),
|
|
92
|
+
}
|
|
93
|
+
if not record_is_measured(rec):
|
|
94
|
+
return None
|
|
95
|
+
return _num(rec["cost_usd"])
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _loki_dir(workspace):
|
|
99
|
+
"""Accept either a workspace root or a .loki dir; collect_ wants .loki."""
|
|
100
|
+
if os.path.basename(os.path.normpath(workspace)) == ".loki":
|
|
101
|
+
return workspace
|
|
102
|
+
return os.path.join(workspace, ".loki")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _append(path, entry):
|
|
106
|
+
"""Append ONE newline-terminated line with ONE write() call, in "a" mode.
|
|
107
|
+
|
|
108
|
+
WHY THAT IS SUFFICIENT HERE, rather than the usual write-temp-and-rename.
|
|
109
|
+
POSIX gives an O_APPEND write a seek-to-end that cannot be interleaved, so
|
|
110
|
+
concurrent appenders cannot overwrite each other, and a line small enough
|
|
111
|
+
to land in a single write is not torn between two of them. Rename-based
|
|
112
|
+
atomicity would be strictly WORSE for this file: it rewrites the whole
|
|
113
|
+
history every time, which turns a crash into the loss of every prior run
|
|
114
|
+
instead of a partial final line.
|
|
115
|
+
|
|
116
|
+
That partial final line is the residual risk and it is handled rather than
|
|
117
|
+
prevented: a torn or truncated row is COUNTED as corrupt by load() and
|
|
118
|
+
named in the report. This is why the newline terminator is load-bearing --
|
|
119
|
+
it is what makes a complete row distinguishable from a truncated one.
|
|
120
|
+
|
|
121
|
+
ponytail: no locking. O_APPEND is the lock. Add one only if this ever needs
|
|
122
|
+
to write multi-line entries, which would break the single-write property.
|
|
123
|
+
"""
|
|
124
|
+
parent = os.path.dirname(os.path.abspath(path))
|
|
125
|
+
if parent:
|
|
126
|
+
os.makedirs(parent, exist_ok=True)
|
|
127
|
+
line = json.dumps(entry, sort_keys=True) + "\n"
|
|
128
|
+
with open(path, "a", encoding="utf-8") as handle:
|
|
129
|
+
handle.write(line)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def load(path):
|
|
133
|
+
"""Read the history. Returns (entries, corrupt_count).
|
|
134
|
+
|
|
135
|
+
A line that is not JSON, or is JSON but not an object, is corrupt and
|
|
136
|
+
COUNTED. It is never dropped on the floor.
|
|
137
|
+
"""
|
|
138
|
+
entries, corrupt = [], 0
|
|
139
|
+
try:
|
|
140
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
141
|
+
raw = handle.read()
|
|
142
|
+
except OSError:
|
|
143
|
+
return None, 0
|
|
144
|
+
for line in raw.splitlines():
|
|
145
|
+
if not line.strip():
|
|
146
|
+
continue
|
|
147
|
+
try:
|
|
148
|
+
obj = json.loads(line)
|
|
149
|
+
except ValueError:
|
|
150
|
+
corrupt += 1
|
|
151
|
+
continue
|
|
152
|
+
if not isinstance(obj, dict) or "usd" not in obj:
|
|
153
|
+
corrupt += 1
|
|
154
|
+
continue
|
|
155
|
+
entries.append(obj)
|
|
156
|
+
return entries, corrupt
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def median(values):
|
|
160
|
+
"""Median of a non-empty list. Even length averages the middle pair."""
|
|
161
|
+
s = sorted(values)
|
|
162
|
+
n = len(s)
|
|
163
|
+
mid = n // 2
|
|
164
|
+
return s[mid] if n % 2 else (s[mid - 1] + s[mid]) / 2.0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def direction(values):
|
|
168
|
+
"""(label, explanation) for a series oldest to newest.
|
|
169
|
+
|
|
170
|
+
Compares the median of the older half against the median of the newer
|
|
171
|
+
half, not first-vs-last: a single expensive outlier at either end should
|
|
172
|
+
not name the trend for a whole quarter. With 2 or 3 points the halves are
|
|
173
|
+
small, so the comparison it used is always printed alongside the verdict --
|
|
174
|
+
the operator can see how thin the evidence is.
|
|
175
|
+
"""
|
|
176
|
+
if len(values) < 2:
|
|
177
|
+
return "INSUFFICIENT DATA", (
|
|
178
|
+
"%d measured run(s); a trend needs at least 2. One observation "
|
|
179
|
+
"cannot show change over time, and 'flat' would be a claim."
|
|
180
|
+
% len(values))
|
|
181
|
+
half = len(values) // 2
|
|
182
|
+
older = median(values[:half])
|
|
183
|
+
newer = median(values[len(values) - half:])
|
|
184
|
+
basis = ("median of the oldest %d ($%.4f) vs the newest %d ($%.4f)"
|
|
185
|
+
% (half, older, half, newer))
|
|
186
|
+
if older == 0:
|
|
187
|
+
if newer == 0:
|
|
188
|
+
return "flat", basis + "; both zero"
|
|
189
|
+
return "rising", basis + "; percent change is undefined against zero"
|
|
190
|
+
pct = (newer - older) / older * 100.0
|
|
191
|
+
if abs(pct) < FLAT_PCT:
|
|
192
|
+
return "flat", "%s; %+.1f%%, within the %.0f%% flat band" % (
|
|
193
|
+
basis, pct, FLAT_PCT)
|
|
194
|
+
return ("rising" if pct > 0 else "falling"), "%s; %+.1f%%" % (basis, pct)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def record(workspace, path, now=None):
|
|
198
|
+
"""Append this workspace's run to the history. Returns a verdict dict."""
|
|
199
|
+
cost, model = collect_efficiency(_loki_dir(workspace))
|
|
200
|
+
usd = measured_usd(cost)
|
|
201
|
+
entry = {
|
|
202
|
+
"ts": now if now is not None else time.time(),
|
|
203
|
+
"workspace": os.path.abspath(workspace),
|
|
204
|
+
"usd": usd, # None when unmeasured. NEVER 0 as a stand-in.
|
|
205
|
+
"measured": usd is not None,
|
|
206
|
+
"model": model or None,
|
|
207
|
+
}
|
|
208
|
+
try:
|
|
209
|
+
_append(path, entry)
|
|
210
|
+
except OSError as exc:
|
|
211
|
+
return {"status": "cannot_record", "exit_code": CANNOT,
|
|
212
|
+
"why": "could not append to %s: %s" % (path, exc),
|
|
213
|
+
"entry": None}
|
|
214
|
+
return {"status": "recorded", "exit_code": OK, "why": None,
|
|
215
|
+
"entry": entry}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def report(path):
|
|
219
|
+
"""Summarize the history. Returns a verdict dict."""
|
|
220
|
+
entries, corrupt = load(path)
|
|
221
|
+
if entries is None:
|
|
222
|
+
return {"status": "no_history", "exit_code": NOTHING,
|
|
223
|
+
"why": "no history file at %s; record a run first." % path,
|
|
224
|
+
"runs": 0, "measured": 0, "corrupt_lines": 0,
|
|
225
|
+
"costs": [], "median_usd": None,
|
|
226
|
+
"direction": "INSUFFICIENT DATA", "basis": None}
|
|
227
|
+
|
|
228
|
+
costs = [e["usd"] for e in entries
|
|
229
|
+
if e.get("measured") and _num(e.get("usd")) is not None]
|
|
230
|
+
|
|
231
|
+
if not entries:
|
|
232
|
+
# An empty file is not a flat trend. Zero runs is an absent
|
|
233
|
+
# measurement, and reporting stability from it would be a claim made
|
|
234
|
+
# out of nothing.
|
|
235
|
+
return {"status": "empty", "exit_code": NOTHING,
|
|
236
|
+
"why": "history %s is empty (%d corrupt line(s)); zero runs "
|
|
237
|
+
"is not a flat trend." % (path, corrupt),
|
|
238
|
+
"runs": 0, "measured": 0, "corrupt_lines": corrupt,
|
|
239
|
+
"costs": [], "median_usd": None,
|
|
240
|
+
"direction": "INSUFFICIENT DATA", "basis": None}
|
|
241
|
+
|
|
242
|
+
label, basis = direction(costs)
|
|
243
|
+
return {
|
|
244
|
+
"status": "ok" if costs else "no_measured_runs",
|
|
245
|
+
"exit_code": OK if costs else NOTHING,
|
|
246
|
+
"why": None if costs else (
|
|
247
|
+
"%d run(s) recorded but none carried a measured cost; there is "
|
|
248
|
+
"nothing to trend. Unmeasured runs are kept as null, not 0."
|
|
249
|
+
% len(entries)),
|
|
250
|
+
"runs": len(entries),
|
|
251
|
+
"measured": len(costs),
|
|
252
|
+
"corrupt_lines": corrupt,
|
|
253
|
+
"costs": costs,
|
|
254
|
+
"median_usd": median(costs) if costs else None,
|
|
255
|
+
"direction": label,
|
|
256
|
+
"basis": basis,
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def render(d):
|
|
261
|
+
if d["status"] in ("no_history", "empty"):
|
|
262
|
+
return "NO TREND: %s" % d["why"]
|
|
263
|
+
if d["status"] == "cannot_record":
|
|
264
|
+
return "CANNOT RECORD: %s" % d["why"]
|
|
265
|
+
if d["status"] == "recorded":
|
|
266
|
+
e = d["entry"]
|
|
267
|
+
return ("recorded %s: %s" % (
|
|
268
|
+
e["workspace"],
|
|
269
|
+
"$%.4f" % e["usd"] if e["measured"]
|
|
270
|
+
else "UNMEASURED (stored as null, excluded from the trend)"))
|
|
271
|
+
|
|
272
|
+
lines = ["%d run(s), %d measured" % (d["runs"], d["measured"])]
|
|
273
|
+
if d["corrupt_lines"]:
|
|
274
|
+
# Named, not swallowed. A dropped row makes the trend look cleaner
|
|
275
|
+
# than reality.
|
|
276
|
+
lines.append("%d CORRUPT line(s) in the history -- counted, not "
|
|
277
|
+
"skipped; the trend below omits them."
|
|
278
|
+
% d["corrupt_lines"])
|
|
279
|
+
if d["status"] == "no_measured_runs":
|
|
280
|
+
lines.append("NO TREND: %s" % d["why"])
|
|
281
|
+
return "\n".join(lines)
|
|
282
|
+
lines.append("costs oldest to newest: "
|
|
283
|
+
+ ", ".join("$%.4f" % c for c in d["costs"]))
|
|
284
|
+
lines.append("median: $%.4f" % d["median_usd"])
|
|
285
|
+
lines.append("direction: %s (%s)" % (d["direction"], d["basis"]))
|
|
286
|
+
return "\n".join(lines)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def main(argv=None):
|
|
290
|
+
ap = argparse.ArgumentParser(
|
|
291
|
+
description="Track agent cost across many runs and report the trend.")
|
|
292
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
293
|
+
|
|
294
|
+
rec = sub.add_parser("record", help="append this run's cost to the history")
|
|
295
|
+
rec.add_argument("workspace", nargs="?", default=".",
|
|
296
|
+
help="workspace root (or its .loki dir); default .")
|
|
297
|
+
rec.add_argument("--file", default=DEFAULT_FILE,
|
|
298
|
+
help="history JSONL (default %s)" % DEFAULT_FILE)
|
|
299
|
+
|
|
300
|
+
rep = sub.add_parser("report", help="summarize the recorded history")
|
|
301
|
+
rep.add_argument("--file", default=DEFAULT_FILE,
|
|
302
|
+
help="history JSONL (default %s)" % DEFAULT_FILE)
|
|
303
|
+
rep.add_argument("--json", action="store_true", dest="as_json",
|
|
304
|
+
help="emit the report as JSON")
|
|
305
|
+
|
|
306
|
+
args = ap.parse_args(argv)
|
|
307
|
+
if args.cmd == "record":
|
|
308
|
+
d = record(args.workspace, args.file)
|
|
309
|
+
print(render(d))
|
|
310
|
+
else:
|
|
311
|
+
d = report(args.file)
|
|
312
|
+
print(json.dumps(d, indent=2) if args.as_json else render(d))
|
|
313
|
+
return d["exit_code"]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
if __name__ == "__main__":
|
|
317
|
+
sys.exit(main())
|
package/tools/model-advisor.py
CHANGED
|
@@ -484,10 +484,21 @@ def main(argv=None):
|
|
|
484
484
|
ap.add_argument("--json", action="store_true")
|
|
485
485
|
args = ap.parse_args(argv)
|
|
486
486
|
|
|
487
|
+
# A workspace that does not EXIST is not the same fact as a workspace with
|
|
488
|
+
# no cost history, and exit 0 collapsed them. A CI job doing
|
|
489
|
+
# `model-advisor.py "$WS" && ...` on a mistyped or unmounted path saw green
|
|
490
|
+
# and carried on. Every sibling tool distinguishes these (run-replay 66,
|
|
491
|
+
# cost-guard 2, receipt-bundle 3); this one did not.
|
|
492
|
+
if not os.path.isdir(args.workspace):
|
|
493
|
+
sys.stderr.write(
|
|
494
|
+
"cannot advise: workspace does not exist: %s\n" % args.workspace)
|
|
495
|
+
return 66
|
|
496
|
+
|
|
487
497
|
adv = advise(args.workspace)
|
|
488
498
|
print(json.dumps(adv, indent=2) if args.json else render(adv))
|
|
489
|
-
# Exit 0
|
|
490
|
-
# failure
|
|
499
|
+
# Exit 0 for a real workspace with no basis: that IS a successful, honest
|
|
500
|
+
# answer rather than a tool failure, and the output says so in words.
|
|
501
|
+
# Callers read has_basis.
|
|
491
502
|
return 0
|
|
492
503
|
|
|
493
504
|
|