loki-mode 9.21.0 → 9.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/SKILL.md CHANGED
@@ -3,7 +3,7 @@ name: loki-mode
3
3
  description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
4
4
  ---
5
5
 
6
- # Loki Mode v9.21.0
6
+ # Loki Mode v9.22.0
7
7
 
8
8
  **You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
9
9
 
@@ -470,4 +470,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
470
470
 
471
471
  ---
472
472
 
473
- **v9.21.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
473
+ **v9.22.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
package/VERSION CHANGED
@@ -1 +1 @@
1
- 9.21.0
1
+ 9.22.0
@@ -7,7 +7,7 @@ Modules:
7
7
  control: Session control API (start/stop/pause/resume)
8
8
  """
9
9
 
10
- __version__ = "9.21.0"
10
+ __version__ = "9.22.0"
11
11
 
12
12
  # Expose the control app for easy import
13
13
  try:
@@ -0,0 +1,65 @@
1
+ # Outcome Canary Evaluation
2
+
3
+ `outcome-canary-evaluate.py` converts consented, locally recorded canary
4
+ observations into one deterministic aggregate verdict. It never invokes a provider,
5
+ changes an assignment, or promotes a route.
6
+
7
+ ```bash
8
+ python3 tools/outcome-canary-evaluate.py report.json observations.json \
9
+ --enable-evaluation --control-route safe --canary-percent 10 --json
10
+ ```
11
+
12
+ Evaluation is opt-in. Without `--enable-evaluation`, the command refuses.
13
+
14
+ ## Observation contract
15
+
16
+ The input is one `loki-outcome-canary-observations/v1` JSON object:
17
+
18
+ ```json
19
+ {
20
+ "observations": "loki-outcome-canary-observations/v1",
21
+ "report_sha256": "<sha256 of the exact router report>",
22
+ "source_sha256": "<source_sha256 from that report>",
23
+ "items": [
24
+ {
25
+ "subject": "locally chosen opaque key",
26
+ "assignment": "control",
27
+ "route": "safe",
28
+ "accepted": true,
29
+ "risk": 0.1
30
+ }
31
+ ]
32
+ }
33
+ ```
34
+
35
+ Each item has exactly those five fields. Subjects must be unique and non-empty;
36
+ `accepted` is a JSON Boolean; and `risk` is a finite number from zero through one.
37
+ The file is capped at 5 MiB and 100,000 observations. Duplicate JSON keys are
38
+ rejected.
39
+
40
+ The evaluator hashes the exact report and observation bytes, rechecks the report's
41
+ underlying source digest, and reruns the released deterministic canary assignment
42
+ for every subject. A recorded arm or route that does not match that assignment
43
+ refuses the entire evaluation. Output contains no subject keys.
44
+
45
+ ## Verdict policy
46
+
47
+ Both arms must reach `--min-samples` (default 5). The tool computes integer accepted
48
+ basis points and mean observed risk for each arm.
49
+
50
+ - `ROLLBACK`: canary acceptance is below control, or canary mean risk exceeds
51
+ `--max-risk` (default 0.25).
52
+ - `PROMOTE`: canary acceptance lift reaches `--min-lift-bps` (default 1) and canary
53
+ mean risk is no higher than control.
54
+ - `HOLD`: the evidence is valid and sufficiently sampled but meets neither rule.
55
+
56
+ These are offline recommendations, not routing actions. A consented operator still
57
+ controls whether to apply any change. Malformed, drifted, mismatched, sparse, or
58
+ unbound evidence returns `REFUSED` rather than a partial verdict.
59
+
60
+ ## Output and exit codes
61
+
62
+ `--json` emits the aggregate arms, policy, exact evidence digests, verdict, and
63
+ refusal reasons. The default output is a short human-readable summary. Exit 0 means
64
+ a verdict was produced (including `ROLLBACK`), 3 means evaluation was refused, 64
65
+ is an invocation error, and 66 is a missing input file.
@@ -1,5 +1,5 @@
1
1
  // @bun
2
- var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.21.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Yf(Jf(import.meta.url)),Q=X$(X);h5=Xf(Qf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var _O={};l0(_O,{runOrThrow:()=>Tf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Cf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>bO});async function NQ(Z,X=bO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let V=$.byteLength-(z-X);J+=Y.decode($.subarray(0,V),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Tf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=wf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function wf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Cf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var bO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return Ff?"":Z}var Ff,A0,F8,_0,$W0,a0,V8,Q9,h;var S6=p(()=>{Ff=(process.env.NO_COLOR??"").length>0;A0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),_0=o7("\x1B[1;33m"),$W0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),V8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),h=o7("\x1B[0m")});import{existsSync as ff}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(ff(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var XA={};l0(XA,{runStatus:()=>Vh});import{existsSync as Y9,readFileSync as g3,readdirSync as aO,statSync as sO}from"fs";import{resolve as h8,basename as ef}from"path";import{homedir as Zh}from"os";function nO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function oO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*AQ/X);if(J>AQ)J=AQ;let z=AQ-J,K=F8;if(Y>=80)K=A0;else if(Y>=50)K=_0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),V=nO(Z),W=nO(X);return` ${V8}${Q}${h} ${K}[${$}]${h} ${Y}% (${V} / ${W})`}async function Qh(){if(await X9("jq"))return!0;return process.stdout.write(`${A0}Error: jq is required but not installed.${h}
2
+ var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.22.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Yf(Jf(import.meta.url)),Q=X$(X);h5=Xf(Qf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var _O={};l0(_O,{runOrThrow:()=>Tf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Cf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>bO});async function NQ(Z,X=bO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let V=$.byteLength-(z-X);J+=Y.decode($.subarray(0,V),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Tf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=wf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function wf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Cf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var bO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return Ff?"":Z}var Ff,A0,F8,_0,$W0,a0,V8,Q9,h;var S6=p(()=>{Ff=(process.env.NO_COLOR??"").length>0;A0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),_0=o7("\x1B[1;33m"),$W0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),V8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),h=o7("\x1B[0m")});import{existsSync as ff}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(ff(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var XA={};l0(XA,{runStatus:()=>Vh});import{existsSync as Y9,readFileSync as g3,readdirSync as aO,statSync as sO}from"fs";import{resolve as h8,basename as ef}from"path";import{homedir as Zh}from"os";function nO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function oO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*AQ/X);if(J>AQ)J=AQ;let z=AQ-J,K=F8;if(Y>=80)K=A0;else if(Y>=50)K=_0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),V=nO(Z),W=nO(X);return` ${V8}${Q}${h} ${K}[${$}]${h} ${Y}% (${V} / ${W})`}async function Qh(){if(await X9("jq"))return!0;return process.stdout.write(`${A0}Error: jq is required but not installed.${h}
3
3
  `),process.stdout.write(`Install with:
4
4
  `),process.stdout.write(` brew install jq (macOS)
5
5
  `),process.stdout.write(` apt install jq (Debian/Ubuntu)
@@ -1236,4 +1236,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
1236
1236
  `),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (g_(),v_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
1237
1237
  `),process.stderr.write(m_),2}}lO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var dV0=await pV0(Bun.argv.slice(2));process.exit(dV0);
1238
1238
 
1239
- //# debugId=8B493F2C51A0A0E764756E2164756E21
1239
+ //# debugId=DE8569120D6E31F464756E2164756E21
package/mcp/__init__.py CHANGED
@@ -75,4 +75,4 @@ try:
75
75
  except ImportError:
76
76
  __all__ = ['mcp']
77
77
 
78
- __version__ = '9.21.0'
78
+ __version__ = '9.22.0'
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "loki-mode",
3
3
  "mcpName": "io.github.asklokesh/loki-mode",
4
- "version": "9.21.0",
4
+ "version": "9.22.0",
5
5
  "description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
6
6
  "keywords": [
7
7
  "agent",
@@ -2,7 +2,7 @@
2
2
  "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
3
3
  "name": "loki-mode",
4
4
  "displayName": "Loki Mode",
5
- "version": "9.21.0",
5
+ "version": "9.22.0",
6
6
  "description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
7
7
  "author": {
8
8
  "name": "Autonomi",
@@ -0,0 +1,261 @@
1
+ #!/usr/bin/env python3
2
+ """Evaluate source-bound canary observations without changing a route."""
3
+ import argparse
4
+ import hashlib
5
+ import importlib.util
6
+ import json
7
+ import math
8
+ import os
9
+ import pathlib
10
+ import sys
11
+
12
+ OK, REFUSED, USAGE, NO_INPUT = 0, 3, 64, 66
13
+ DOMAIN = "loki-outcome-canary-evaluation/v1"
14
+ OBSERVATIONS = "loki-outcome-canary-observations/v1"
15
+ MAX_BYTES = 5 * 1024 * 1024
16
+ MAX_ITEMS = 100_000
17
+
18
+
19
+ class Parser(argparse.ArgumentParser):
20
+ def error(self, message):
21
+ self.print_usage(sys.stderr)
22
+ self.exit(USAGE, f"{self.prog}: error: {message}\n")
23
+
24
+
25
+ class DuplicateKey(ValueError):
26
+ pass
27
+
28
+
29
+ def _object(pairs):
30
+ out = {}
31
+ for key, value in pairs:
32
+ if key in out:
33
+ raise DuplicateKey(f"duplicate key: {key}")
34
+ out[key] = value
35
+ return out
36
+
37
+
38
+ def _number(value, low, high):
39
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
40
+ return None
41
+ if not math.isfinite(value) or value < low or value > high:
42
+ return None
43
+ return float(value)
44
+
45
+
46
+ def _integer(value, low, high):
47
+ if isinstance(value, bool) or not isinstance(value, int) or value < low or value > high:
48
+ return None
49
+ return value
50
+
51
+
52
+ def _sha256(data):
53
+ return hashlib.sha256(data).hexdigest()
54
+
55
+
56
+ def _read_json(path):
57
+ with open(path, "rb") as handle:
58
+ data = handle.read(MAX_BYTES + 1)
59
+ if len(data) > MAX_BYTES:
60
+ raise ValueError(f"input exceeds {MAX_BYTES} bytes")
61
+ return json.loads(data.decode("utf-8"), object_pairs_hook=_object), _sha256(data)
62
+
63
+
64
+ def _load_planner():
65
+ path = pathlib.Path(__file__).with_name("outcome-canary.py")
66
+ spec = importlib.util.spec_from_file_location("outcome_canary", path)
67
+ if spec is None or spec.loader is None:
68
+ raise RuntimeError("cannot load outcome canary planner")
69
+ module = importlib.util.module_from_spec(spec)
70
+ spec.loader.exec_module(module)
71
+ return module
72
+
73
+
74
+ def _arm(items):
75
+ trials = len(items)
76
+ accepted = sum(1 for item in items if item["accepted"])
77
+ return {
78
+ "trials": trials,
79
+ "accepted": accepted,
80
+ "accepted_bps": (accepted * 10_000) // trials,
81
+ "mean_risk": sum(item["risk"] for item in items) / trials,
82
+ }
83
+
84
+
85
+ def evaluate(report_path, observations_path, control_route, canary_percent=10.0,
86
+ max_risk=.25, min_samples=5, min_lift_bps=1, enable_evaluation=False):
87
+ """Return a deterministic aggregate verdict or an explained refusal."""
88
+ reasons = []
89
+ percent = _number(canary_percent, 0, 100)
90
+ ceiling = _number(max_risk, 0, 1)
91
+ samples = _integer(min_samples, 1, MAX_ITEMS)
92
+ lift = _integer(min_lift_bps, 0, 10_000)
93
+ if not enable_evaluation:
94
+ reasons.append("evaluation is opt-in; pass --enable-evaluation")
95
+ if not isinstance(control_route, str) or not control_route.strip():
96
+ reasons.append("control route must be non-empty")
97
+ if percent is None:
98
+ reasons.append("canary percent must be a finite number between 0 and 100")
99
+ if ceiling is None:
100
+ reasons.append("max risk must be a finite number between 0 and 1")
101
+ if samples is None:
102
+ reasons.append(f"min samples must be an integer between 1 and {MAX_ITEMS}")
103
+ if lift is None:
104
+ reasons.append("min lift bps must be an integer between 0 and 10000")
105
+
106
+ out = {
107
+ "evaluation": DOMAIN,
108
+ "report": os.path.abspath(report_path),
109
+ "report_sha256": None,
110
+ "source_sha256": None,
111
+ "observations": os.path.abspath(observations_path),
112
+ "observations_sha256": None,
113
+ "control_route": control_route,
114
+ "canary_route": None,
115
+ "canary_percent": percent,
116
+ "max_risk": ceiling,
117
+ "min_samples": samples,
118
+ "min_lift_bps": lift,
119
+ "control": None,
120
+ "canary": None,
121
+ "verdict": None,
122
+ "refusal_reasons": reasons,
123
+ }
124
+ try:
125
+ observations, observations_sha256 = _read_json(observations_path)
126
+ out["observations_sha256"] = observations_sha256
127
+ except Exception as exc:
128
+ out["refusal_reasons"].append(f"observations are malformed: {exc}")
129
+ return out
130
+ if not isinstance(observations, dict):
131
+ out["refusal_reasons"].append("observations are not a JSON object")
132
+ return out
133
+ if observations.get("observations") != OBSERVATIONS:
134
+ out["refusal_reasons"].append(f"observations version is not {OBSERVATIONS}")
135
+ items = observations.get("items")
136
+ if not isinstance(items, list):
137
+ out["refusal_reasons"].append("observations have no items list")
138
+ return out
139
+ if len(items) > MAX_ITEMS:
140
+ out["refusal_reasons"].append(f"observations exceed {MAX_ITEMS} items")
141
+ return out
142
+
143
+ try:
144
+ planner = _load_planner()
145
+ report, report_sha256, report_reasons = planner.load_report(report_path)
146
+ except Exception as exc:
147
+ out["refusal_reasons"].append(f"report is malformed: {exc}")
148
+ return out
149
+ out["report_sha256"] = report_sha256
150
+ out["refusal_reasons"].extend(report_reasons)
151
+ if report is None:
152
+ return out
153
+ out["source_sha256"] = report.get("source_sha256")
154
+ out["canary_route"] = report.get("selected_route")
155
+ if observations.get("report_sha256") != report_sha256:
156
+ out["refusal_reasons"].append("observations are not bound to the exact router report")
157
+ if observations.get("source_sha256") != report.get("source_sha256"):
158
+ out["refusal_reasons"].append("observations are not bound to the exact evidence source")
159
+
160
+ seen = set()
161
+ arms = {"control": [], "canary": []}
162
+ for index, item in enumerate(items):
163
+ label = f"item {index}"
164
+ if not isinstance(item, dict):
165
+ out["refusal_reasons"].append(f"{label} is not an object")
166
+ continue
167
+ if set(item) != {"subject", "assignment", "route", "accepted", "risk"}:
168
+ out["refusal_reasons"].append(f"{label} has a non-canonical shape")
169
+ continue
170
+ subject = item.get("subject")
171
+ assignment = item.get("assignment")
172
+ route = item.get("route")
173
+ accepted = item.get("accepted")
174
+ risk = _number(item.get("risk"), 0, 1)
175
+ if not isinstance(subject, str) or not subject.strip():
176
+ out["refusal_reasons"].append(f"{label} has an invalid subject")
177
+ continue
178
+ if subject in seen:
179
+ out["refusal_reasons"].append(f"{label} repeats a subject")
180
+ continue
181
+ seen.add(subject)
182
+ if assignment not in arms or not isinstance(route, str) or not isinstance(accepted, bool) or risk is None:
183
+ out["refusal_reasons"].append(f"{label} has invalid measured values")
184
+ continue
185
+ plan = planner.plan(report_path, subject, control_route, percent, ceiling, True)
186
+ if plan.get("refusal_reasons"):
187
+ out["refusal_reasons"].append(f"{label} cannot be rebound to a valid canary plan")
188
+ continue
189
+ if assignment != plan.get("assignment") or route != plan.get("route"):
190
+ out["refusal_reasons"].append(f"{label} does not match its deterministic assignment")
191
+ continue
192
+ arms[assignment].append({"accepted": accepted, "risk": risk})
193
+
194
+ if out["refusal_reasons"]:
195
+ return out
196
+ for name in ("control", "canary"):
197
+ if len(arms[name]) < samples:
198
+ out["refusal_reasons"].append(
199
+ f"{name} arm has {len(arms[name])} samples; requires at least {samples}"
200
+ )
201
+ if out["refusal_reasons"]:
202
+ return out
203
+
204
+ control = _arm(arms["control"])
205
+ canary = _arm(arms["canary"])
206
+ out["control"], out["canary"] = control, canary
207
+ accepted_delta = canary["accepted_bps"] - control["accepted_bps"]
208
+ out["accepted_delta_bps"] = accepted_delta
209
+ if canary["mean_risk"] > ceiling or accepted_delta < 0:
210
+ out["verdict"] = "ROLLBACK"
211
+ elif accepted_delta >= lift and canary["mean_risk"] <= control["mean_risk"]:
212
+ out["verdict"] = "PROMOTE"
213
+ else:
214
+ out["verdict"] = "HOLD"
215
+ return out
216
+
217
+
218
+ def main(argv=None):
219
+ parser = Parser(prog="outcome-canary-evaluate")
220
+ parser.add_argument("report")
221
+ parser.add_argument("observations")
222
+ parser.add_argument("--enable-evaluation", action="store_true")
223
+ parser.add_argument("--control-route", required=True)
224
+ parser.add_argument("--canary-percent", type=float, default=10.0)
225
+ parser.add_argument("--max-risk", type=float, default=.25)
226
+ parser.add_argument("--min-samples", type=int, default=5)
227
+ parser.add_argument("--min-lift-bps", type=int, default=1)
228
+ parser.add_argument("--json", action="store_true")
229
+ args = parser.parse_args(argv)
230
+ for path in (args.report, args.observations):
231
+ if not os.path.isfile(path):
232
+ print(f"outcome-canary-evaluate: no such file: {path}", file=sys.stderr)
233
+ return NO_INPUT
234
+ result = evaluate(
235
+ args.report, args.observations, args.control_route, args.canary_percent,
236
+ args.max_risk, args.min_samples, args.min_lift_bps, args.enable_evaluation,
237
+ )
238
+ if args.json:
239
+ print(json.dumps(result, sort_keys=True))
240
+ elif result["refusal_reasons"]:
241
+ print("Canary evaluation: REFUSED")
242
+ for reason in result["refusal_reasons"]:
243
+ print(f" {reason}")
244
+ else:
245
+ print(f"Canary evaluation: {result['verdict']}")
246
+ print(
247
+ f" control={result['control']['accepted']}/{result['control']['trials']} "
248
+ f"({result['control']['accepted_bps']} bps)"
249
+ )
250
+ print(
251
+ f" canary={result['canary']['accepted']}/{result['canary']['trials']} "
252
+ f"({result['canary']['accepted_bps']} bps)"
253
+ )
254
+ print(f" accepted delta: {result['accepted_delta_bps']} bps")
255
+ print(f" report sha256: {result['report_sha256']}")
256
+ print(f" observations sha256: {result['observations_sha256']}")
257
+ return REFUSED if result["refusal_reasons"] else OK
258
+
259
+
260
+ if __name__ == "__main__":
261
+ raise SystemExit(main())