loki-mode 9.21.0 → 9.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/docs/OUTCOME-CANARY-EVALUATION.md +65 -0
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/outcome-canary-evaluate.py +261 -0
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v9.
|
|
6
|
+
# Loki Mode v9.22.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -470,4 +470,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
470
470
|
|
|
471
471
|
---
|
|
472
472
|
|
|
473
|
-
**v9.
|
|
473
|
+
**v9.22.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
9.
|
|
1
|
+
9.22.0
|
package/dashboard/__init__.py
CHANGED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Outcome Canary Evaluation
|
|
2
|
+
|
|
3
|
+
`outcome-canary-evaluate.py` converts consented, locally recorded canary
|
|
4
|
+
observations into one deterministic aggregate verdict. It never invokes a provider,
|
|
5
|
+
changes an assignment, or promotes a route.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python3 tools/outcome-canary-evaluate.py report.json observations.json \
|
|
9
|
+
--enable-evaluation --control-route safe --canary-percent 10 --json
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Evaluation is opt-in. Without `--enable-evaluation`, the command refuses.
|
|
13
|
+
|
|
14
|
+
## Observation contract
|
|
15
|
+
|
|
16
|
+
The input is one `loki-outcome-canary-observations/v1` JSON object:
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"observations": "loki-outcome-canary-observations/v1",
|
|
21
|
+
"report_sha256": "<sha256 of the exact router report>",
|
|
22
|
+
"source_sha256": "<source_sha256 from that report>",
|
|
23
|
+
"items": [
|
|
24
|
+
{
|
|
25
|
+
"subject": "locally chosen opaque key",
|
|
26
|
+
"assignment": "control",
|
|
27
|
+
"route": "safe",
|
|
28
|
+
"accepted": true,
|
|
29
|
+
"risk": 0.1
|
|
30
|
+
}
|
|
31
|
+
]
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Each item has exactly those five fields. Subjects must be unique and non-empty;
|
|
36
|
+
`accepted` is a JSON Boolean; and `risk` is a finite number from zero through one.
|
|
37
|
+
The file is capped at 5 MiB and 100,000 observations. Duplicate JSON keys are
|
|
38
|
+
rejected.
|
|
39
|
+
|
|
40
|
+
The evaluator hashes the exact report and observation bytes, rechecks the report's
|
|
41
|
+
underlying source digest, and reruns the released deterministic canary assignment
|
|
42
|
+
for every subject. A recorded arm or route that does not match that assignment
|
|
43
|
+
refuses the entire evaluation. Output contains no subject keys.
|
|
44
|
+
|
|
45
|
+
## Verdict policy
|
|
46
|
+
|
|
47
|
+
Both arms must reach `--min-samples` (default 5). The tool computes integer accepted
|
|
48
|
+
basis points and mean observed risk for each arm.
|
|
49
|
+
|
|
50
|
+
- `ROLLBACK`: canary acceptance is below control, or canary mean risk exceeds
|
|
51
|
+
`--max-risk` (default 0.25).
|
|
52
|
+
- `PROMOTE`: canary acceptance lift reaches `--min-lift-bps` (default 1) and canary
|
|
53
|
+
mean risk is no higher than control.
|
|
54
|
+
- `HOLD`: the evidence is valid and sufficiently sampled but meets neither rule.
|
|
55
|
+
|
|
56
|
+
These are offline recommendations, not routing actions. A consented operator still
|
|
57
|
+
controls whether to apply any change. Malformed, drifted, mismatched, sparse, or
|
|
58
|
+
unbound evidence returns `REFUSED` rather than a partial verdict.
|
|
59
|
+
|
|
60
|
+
## Output and exit codes
|
|
61
|
+
|
|
62
|
+
`--json` emits the aggregate arms, policy, exact evidence digests, verdict, and
|
|
63
|
+
refusal reasons. The default output is a short human-readable summary. Exit 0 means
|
|
64
|
+
a verdict was produced (including `ROLLBACK`), 3 means evaluation was refused, 64
|
|
65
|
+
is an invocation error, and 66 is a missing input file.
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.
|
|
2
|
+
var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.22.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Yf(Jf(import.meta.url)),Q=X$(X);h5=Xf(Qf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var _O={};l0(_O,{runOrThrow:()=>Tf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Cf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>bO});async function NQ(Z,X=bO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let V=$.byteLength-(z-X);J+=Y.decode($.subarray(0,V),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Tf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=wf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function wf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Cf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var bO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return Ff?"":Z}var Ff,A0,F8,_0,$W0,a0,V8,Q9,h;var S6=p(()=>{Ff=(process.env.NO_COLOR??"").length>0;A0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),_0=o7("\x1B[1;33m"),$W0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),V8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),h=o7("\x1B[0m")});import{existsSync as ff}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(ff(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var XA={};l0(XA,{runStatus:()=>Vh});import{existsSync as Y9,readFileSync as g3,readdirSync as aO,statSync as sO}from"fs";import{resolve as h8,basename as ef}from"path";import{homedir as Zh}from"os";function nO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function oO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*AQ/X);if(J>AQ)J=AQ;let z=AQ-J,K=F8;if(Y>=80)K=A0;else if(Y>=50)K=_0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),V=nO(Z),W=nO(X);return` ${V8}${Q}${h} ${K}[${$}]${h} ${Y}% (${V} / ${W})`}async function Qh(){if(await X9("jq"))return!0;return process.stdout.write(`${A0}Error: jq is required but not installed.${h}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1236,4 +1236,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1236
1236
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (g_(),v_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1237
1237
|
`),process.stderr.write(m_),2}}lO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var dV0=await pV0(Bun.argv.slice(2));process.exit(dV0);
|
|
1238
1238
|
|
|
1239
|
-
//# debugId=
|
|
1239
|
+
//# debugId=DE8569120D6E31F464756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "9.
|
|
4
|
+
"version": "9.22.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "9.
|
|
5
|
+
"version": "9.22.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Evaluate source-bound canary observations without changing a route."""
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import importlib.util
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import pathlib
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
OK, REFUSED, USAGE, NO_INPUT = 0, 3, 64, 66
|
|
13
|
+
DOMAIN = "loki-outcome-canary-evaluation/v1"
|
|
14
|
+
OBSERVATIONS = "loki-outcome-canary-observations/v1"
|
|
15
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
16
|
+
MAX_ITEMS = 100_000
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Parser(argparse.ArgumentParser):
|
|
20
|
+
def error(self, message):
|
|
21
|
+
self.print_usage(sys.stderr)
|
|
22
|
+
self.exit(USAGE, f"{self.prog}: error: {message}\n")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DuplicateKey(ValueError):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _object(pairs):
|
|
30
|
+
out = {}
|
|
31
|
+
for key, value in pairs:
|
|
32
|
+
if key in out:
|
|
33
|
+
raise DuplicateKey(f"duplicate key: {key}")
|
|
34
|
+
out[key] = value
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _number(value, low, high):
|
|
39
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
40
|
+
return None
|
|
41
|
+
if not math.isfinite(value) or value < low or value > high:
|
|
42
|
+
return None
|
|
43
|
+
return float(value)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _integer(value, low, high):
|
|
47
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < low or value > high:
|
|
48
|
+
return None
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _sha256(data):
|
|
53
|
+
return hashlib.sha256(data).hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _read_json(path):
|
|
57
|
+
with open(path, "rb") as handle:
|
|
58
|
+
data = handle.read(MAX_BYTES + 1)
|
|
59
|
+
if len(data) > MAX_BYTES:
|
|
60
|
+
raise ValueError(f"input exceeds {MAX_BYTES} bytes")
|
|
61
|
+
return json.loads(data.decode("utf-8"), object_pairs_hook=_object), _sha256(data)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _load_planner():
|
|
65
|
+
path = pathlib.Path(__file__).with_name("outcome-canary.py")
|
|
66
|
+
spec = importlib.util.spec_from_file_location("outcome_canary", path)
|
|
67
|
+
if spec is None or spec.loader is None:
|
|
68
|
+
raise RuntimeError("cannot load outcome canary planner")
|
|
69
|
+
module = importlib.util.module_from_spec(spec)
|
|
70
|
+
spec.loader.exec_module(module)
|
|
71
|
+
return module
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _arm(items):
|
|
75
|
+
trials = len(items)
|
|
76
|
+
accepted = sum(1 for item in items if item["accepted"])
|
|
77
|
+
return {
|
|
78
|
+
"trials": trials,
|
|
79
|
+
"accepted": accepted,
|
|
80
|
+
"accepted_bps": (accepted * 10_000) // trials,
|
|
81
|
+
"mean_risk": sum(item["risk"] for item in items) / trials,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def evaluate(report_path, observations_path, control_route, canary_percent=10.0,
|
|
86
|
+
max_risk=.25, min_samples=5, min_lift_bps=1, enable_evaluation=False):
|
|
87
|
+
"""Return a deterministic aggregate verdict or an explained refusal."""
|
|
88
|
+
reasons = []
|
|
89
|
+
percent = _number(canary_percent, 0, 100)
|
|
90
|
+
ceiling = _number(max_risk, 0, 1)
|
|
91
|
+
samples = _integer(min_samples, 1, MAX_ITEMS)
|
|
92
|
+
lift = _integer(min_lift_bps, 0, 10_000)
|
|
93
|
+
if not enable_evaluation:
|
|
94
|
+
reasons.append("evaluation is opt-in; pass --enable-evaluation")
|
|
95
|
+
if not isinstance(control_route, str) or not control_route.strip():
|
|
96
|
+
reasons.append("control route must be non-empty")
|
|
97
|
+
if percent is None:
|
|
98
|
+
reasons.append("canary percent must be a finite number between 0 and 100")
|
|
99
|
+
if ceiling is None:
|
|
100
|
+
reasons.append("max risk must be a finite number between 0 and 1")
|
|
101
|
+
if samples is None:
|
|
102
|
+
reasons.append(f"min samples must be an integer between 1 and {MAX_ITEMS}")
|
|
103
|
+
if lift is None:
|
|
104
|
+
reasons.append("min lift bps must be an integer between 0 and 10000")
|
|
105
|
+
|
|
106
|
+
out = {
|
|
107
|
+
"evaluation": DOMAIN,
|
|
108
|
+
"report": os.path.abspath(report_path),
|
|
109
|
+
"report_sha256": None,
|
|
110
|
+
"source_sha256": None,
|
|
111
|
+
"observations": os.path.abspath(observations_path),
|
|
112
|
+
"observations_sha256": None,
|
|
113
|
+
"control_route": control_route,
|
|
114
|
+
"canary_route": None,
|
|
115
|
+
"canary_percent": percent,
|
|
116
|
+
"max_risk": ceiling,
|
|
117
|
+
"min_samples": samples,
|
|
118
|
+
"min_lift_bps": lift,
|
|
119
|
+
"control": None,
|
|
120
|
+
"canary": None,
|
|
121
|
+
"verdict": None,
|
|
122
|
+
"refusal_reasons": reasons,
|
|
123
|
+
}
|
|
124
|
+
try:
|
|
125
|
+
observations, observations_sha256 = _read_json(observations_path)
|
|
126
|
+
out["observations_sha256"] = observations_sha256
|
|
127
|
+
except Exception as exc:
|
|
128
|
+
out["refusal_reasons"].append(f"observations are malformed: {exc}")
|
|
129
|
+
return out
|
|
130
|
+
if not isinstance(observations, dict):
|
|
131
|
+
out["refusal_reasons"].append("observations are not a JSON object")
|
|
132
|
+
return out
|
|
133
|
+
if observations.get("observations") != OBSERVATIONS:
|
|
134
|
+
out["refusal_reasons"].append(f"observations version is not {OBSERVATIONS}")
|
|
135
|
+
items = observations.get("items")
|
|
136
|
+
if not isinstance(items, list):
|
|
137
|
+
out["refusal_reasons"].append("observations have no items list")
|
|
138
|
+
return out
|
|
139
|
+
if len(items) > MAX_ITEMS:
|
|
140
|
+
out["refusal_reasons"].append(f"observations exceed {MAX_ITEMS} items")
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
planner = _load_planner()
|
|
145
|
+
report, report_sha256, report_reasons = planner.load_report(report_path)
|
|
146
|
+
except Exception as exc:
|
|
147
|
+
out["refusal_reasons"].append(f"report is malformed: {exc}")
|
|
148
|
+
return out
|
|
149
|
+
out["report_sha256"] = report_sha256
|
|
150
|
+
out["refusal_reasons"].extend(report_reasons)
|
|
151
|
+
if report is None:
|
|
152
|
+
return out
|
|
153
|
+
out["source_sha256"] = report.get("source_sha256")
|
|
154
|
+
out["canary_route"] = report.get("selected_route")
|
|
155
|
+
if observations.get("report_sha256") != report_sha256:
|
|
156
|
+
out["refusal_reasons"].append("observations are not bound to the exact router report")
|
|
157
|
+
if observations.get("source_sha256") != report.get("source_sha256"):
|
|
158
|
+
out["refusal_reasons"].append("observations are not bound to the exact evidence source")
|
|
159
|
+
|
|
160
|
+
seen = set()
|
|
161
|
+
arms = {"control": [], "canary": []}
|
|
162
|
+
for index, item in enumerate(items):
|
|
163
|
+
label = f"item {index}"
|
|
164
|
+
if not isinstance(item, dict):
|
|
165
|
+
out["refusal_reasons"].append(f"{label} is not an object")
|
|
166
|
+
continue
|
|
167
|
+
if set(item) != {"subject", "assignment", "route", "accepted", "risk"}:
|
|
168
|
+
out["refusal_reasons"].append(f"{label} has a non-canonical shape")
|
|
169
|
+
continue
|
|
170
|
+
subject = item.get("subject")
|
|
171
|
+
assignment = item.get("assignment")
|
|
172
|
+
route = item.get("route")
|
|
173
|
+
accepted = item.get("accepted")
|
|
174
|
+
risk = _number(item.get("risk"), 0, 1)
|
|
175
|
+
if not isinstance(subject, str) or not subject.strip():
|
|
176
|
+
out["refusal_reasons"].append(f"{label} has an invalid subject")
|
|
177
|
+
continue
|
|
178
|
+
if subject in seen:
|
|
179
|
+
out["refusal_reasons"].append(f"{label} repeats a subject")
|
|
180
|
+
continue
|
|
181
|
+
seen.add(subject)
|
|
182
|
+
if assignment not in arms or not isinstance(route, str) or not isinstance(accepted, bool) or risk is None:
|
|
183
|
+
out["refusal_reasons"].append(f"{label} has invalid measured values")
|
|
184
|
+
continue
|
|
185
|
+
plan = planner.plan(report_path, subject, control_route, percent, ceiling, True)
|
|
186
|
+
if plan.get("refusal_reasons"):
|
|
187
|
+
out["refusal_reasons"].append(f"{label} cannot be rebound to a valid canary plan")
|
|
188
|
+
continue
|
|
189
|
+
if assignment != plan.get("assignment") or route != plan.get("route"):
|
|
190
|
+
out["refusal_reasons"].append(f"{label} does not match its deterministic assignment")
|
|
191
|
+
continue
|
|
192
|
+
arms[assignment].append({"accepted": accepted, "risk": risk})
|
|
193
|
+
|
|
194
|
+
if out["refusal_reasons"]:
|
|
195
|
+
return out
|
|
196
|
+
for name in ("control", "canary"):
|
|
197
|
+
if len(arms[name]) < samples:
|
|
198
|
+
out["refusal_reasons"].append(
|
|
199
|
+
f"{name} arm has {len(arms[name])} samples; requires at least {samples}"
|
|
200
|
+
)
|
|
201
|
+
if out["refusal_reasons"]:
|
|
202
|
+
return out
|
|
203
|
+
|
|
204
|
+
control = _arm(arms["control"])
|
|
205
|
+
canary = _arm(arms["canary"])
|
|
206
|
+
out["control"], out["canary"] = control, canary
|
|
207
|
+
accepted_delta = canary["accepted_bps"] - control["accepted_bps"]
|
|
208
|
+
out["accepted_delta_bps"] = accepted_delta
|
|
209
|
+
if canary["mean_risk"] > ceiling or accepted_delta < 0:
|
|
210
|
+
out["verdict"] = "ROLLBACK"
|
|
211
|
+
elif accepted_delta >= lift and canary["mean_risk"] <= control["mean_risk"]:
|
|
212
|
+
out["verdict"] = "PROMOTE"
|
|
213
|
+
else:
|
|
214
|
+
out["verdict"] = "HOLD"
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def main(argv=None):
|
|
219
|
+
parser = Parser(prog="outcome-canary-evaluate")
|
|
220
|
+
parser.add_argument("report")
|
|
221
|
+
parser.add_argument("observations")
|
|
222
|
+
parser.add_argument("--enable-evaluation", action="store_true")
|
|
223
|
+
parser.add_argument("--control-route", required=True)
|
|
224
|
+
parser.add_argument("--canary-percent", type=float, default=10.0)
|
|
225
|
+
parser.add_argument("--max-risk", type=float, default=.25)
|
|
226
|
+
parser.add_argument("--min-samples", type=int, default=5)
|
|
227
|
+
parser.add_argument("--min-lift-bps", type=int, default=1)
|
|
228
|
+
parser.add_argument("--json", action="store_true")
|
|
229
|
+
args = parser.parse_args(argv)
|
|
230
|
+
for path in (args.report, args.observations):
|
|
231
|
+
if not os.path.isfile(path):
|
|
232
|
+
print(f"outcome-canary-evaluate: no such file: {path}", file=sys.stderr)
|
|
233
|
+
return NO_INPUT
|
|
234
|
+
result = evaluate(
|
|
235
|
+
args.report, args.observations, args.control_route, args.canary_percent,
|
|
236
|
+
args.max_risk, args.min_samples, args.min_lift_bps, args.enable_evaluation,
|
|
237
|
+
)
|
|
238
|
+
if args.json:
|
|
239
|
+
print(json.dumps(result, sort_keys=True))
|
|
240
|
+
elif result["refusal_reasons"]:
|
|
241
|
+
print("Canary evaluation: REFUSED")
|
|
242
|
+
for reason in result["refusal_reasons"]:
|
|
243
|
+
print(f" {reason}")
|
|
244
|
+
else:
|
|
245
|
+
print(f"Canary evaluation: {result['verdict']}")
|
|
246
|
+
print(
|
|
247
|
+
f" control={result['control']['accepted']}/{result['control']['trials']} "
|
|
248
|
+
f"({result['control']['accepted_bps']} bps)"
|
|
249
|
+
)
|
|
250
|
+
print(
|
|
251
|
+
f" canary={result['canary']['accepted']}/{result['canary']['trials']} "
|
|
252
|
+
f"({result['canary']['accepted_bps']} bps)"
|
|
253
|
+
)
|
|
254
|
+
print(f" accepted delta: {result['accepted_delta_bps']} bps")
|
|
255
|
+
print(f" report sha256: {result['report_sha256']}")
|
|
256
|
+
print(f" observations sha256: {result['observations_sha256']}")
|
|
257
|
+
return REFUSED if result["refusal_reasons"] else OK
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
if __name__ == "__main__":
|
|
261
|
+
raise SystemExit(main())
|