loki-mode 9.20.0 → 9.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/loki +73 -12
- package/autonomy/run.sh +7 -2
- package/bin/loki +1 -0
- package/dashboard/__init__.py +1 -1
- package/docs/OUTCOME-CANARY-EVALUATION.md +65 -0
- package/docs/OUTCOME-CANARY.md +61 -0
- package/docs/OUTCOME-ROUTER.md +7 -0
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/outcome-canary-evaluate.py +261 -0
- package/tools/outcome-canary.py +194 -0
- package/tools/outcome-router.py +80 -0
package/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: loki-mode
|
|
|
3
3
|
description: Autonomous spec-driven build system with a built-in trust layer. It does not call work done until it is verified (RARV-C closure loop, 8 quality gates, completion council, verified-completion evidence gate). Triggers on "Loki Mode". Takes a spec (PRD, GitHub issue, OpenAPI doc, etc.) to deployed product with minimal human intervention. Provider-agnostic. Requires --dangerously-skip-permissions flag.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Loki Mode v9.
|
|
6
|
+
# Loki Mode v9.22.0
|
|
7
7
|
|
|
8
8
|
**You are an autonomous agent. You make decisions. You do not ask questions. You do not stop.**
|
|
9
9
|
|
|
@@ -470,4 +470,4 @@ See `CHANGELOG.md` entries [7.5.7], [7.5.8], [7.5.13] for the per-fix list and r
|
|
|
470
470
|
|
|
471
471
|
---
|
|
472
472
|
|
|
473
|
-
**v9.
|
|
473
|
+
**v9.22.0 | [Autonomi](https://www.autonomi.dev/) flagship product | ~410 lines core**
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
9.
|
|
1
|
+
9.22.0
|
package/autonomy/loki
CHANGED
|
@@ -2021,6 +2021,8 @@ cmd_start() {
|
|
|
2021
2021
|
local mirofish_bg=false
|
|
2022
2022
|
local mirofish_disabled=false
|
|
2023
2023
|
local no_plan=false # v6.81.1: --no-plan opts out of auto-plan display
|
|
2024
|
+
local repo_depth="full" # repository targets default to full verification
|
|
2025
|
+
local repo_depth_explicit="" # "" | "fast" | "full" -- what the operator asked for
|
|
2024
2026
|
local remote_url="${LOKI_REMOTE_URL:-}" # --remote: submit to a deployed cluster
|
|
2025
2027
|
|
|
2026
2028
|
# v6.84.0: unified entry point -- explicit mode overrides + issue-mode args
|
|
@@ -2092,6 +2094,10 @@ cmd_start() {
|
|
|
2092
2094
|
echo " --mirofish-bg Run MiroFish pipeline in background"
|
|
2093
2095
|
echo " --no-mirofish Disable MiroFish even if env var is set"
|
|
2094
2096
|
echo " --no-plan Skip auto-shown PRD analysis at startup"
|
|
2097
|
+
echo " --fast REPO mode: capped lightweight pass, gates reduced"
|
|
2098
|
+
echo " (no council verdict; alias --repo-fast)"
|
|
2099
|
+
echo " --full REPO mode: explicit full verification, the default"
|
|
2100
|
+
echo " (wins over --fast; alias --repo-full)"
|
|
2095
2101
|
echo " --brief \"TEXT\" Zero-config fast first run from a one-line brief"
|
|
2096
2102
|
echo " --yes, -y Skip confirmation prompts (auto-confirm)"
|
|
2097
2103
|
echo " --quiet Suppress [INFO]/[STEP]; warnings and errors still print"
|
|
@@ -2445,6 +2451,22 @@ cmd_start() {
|
|
|
2445
2451
|
no_plan=true
|
|
2446
2452
|
shift
|
|
2447
2453
|
;;
|
|
2454
|
+
--fast|--repo-fast)
|
|
2455
|
+
# Fail closed on a conflict, same doctrine as --isolation below:
|
|
2456
|
+
# never silently downgrade to weaker verification than asked for.
|
|
2457
|
+
# --full wins regardless of order, so `--fast --full` and
|
|
2458
|
+
# `--full --fast` both keep the gates ON.
|
|
2459
|
+
if [ "$repo_depth_explicit" != "full" ]; then
|
|
2460
|
+
repo_depth="fast"
|
|
2461
|
+
repo_depth_explicit="fast"
|
|
2462
|
+
fi
|
|
2463
|
+
shift
|
|
2464
|
+
;;
|
|
2465
|
+
--full|--repo-full)
|
|
2466
|
+
repo_depth="full"
|
|
2467
|
+
repo_depth_explicit="full"
|
|
2468
|
+
shift
|
|
2469
|
+
;;
|
|
2448
2470
|
--brief)
|
|
2449
2471
|
# R7: explicit one-line brief (escape hatch for single-word
|
|
2450
2472
|
# briefs that detect_arg_type would otherwise treat as a PRD
|
|
@@ -2839,27 +2861,50 @@ cmd_start() {
|
|
|
2839
2861
|
# synthesized brief that bounds the work to ONE high-value improvement
|
|
2840
2862
|
# instead of "build out the whole project".
|
|
2841
2863
|
#
|
|
2842
|
-
#
|
|
2843
|
-
#
|
|
2844
|
-
#
|
|
2845
|
-
#
|
|
2846
|
-
#
|
|
2847
|
-
# full-depth signal the no-arg path
|
|
2864
|
+
# The DEFAULT and `--full` deliberately do NOT call
|
|
2865
|
+
# set_ttfv_lightweight_profile: that helper turns the council and the
|
|
2866
|
+
# code-review phase OFF, and the whole point of this entry point is to hand
|
|
2867
|
+
# back VERIFIED evidence. Scope is bounded by the brief text, not by
|
|
2868
|
+
# disabling the gates that check the work. Those runs keep full depth and
|
|
2869
|
+
# report LOKI_TTFV=repo -- the same honest full-depth signal the no-arg path
|
|
2870
|
+
# uses.
|
|
2871
|
+
#
|
|
2872
|
+
# `--fast` is the opt-in exception: it applies the SAME lightweight profile
|
|
2873
|
+
# the brief path uses (capped iterations, council off, simple tier, heavy
|
|
2874
|
+
# phases off) and reports LOKI_TTFV=repo-fast, a distinct signal so the
|
|
2875
|
+
# end-of-run wording in run.sh cannot claim council verdicts that never ran.
|
|
2848
2876
|
if [ "$detected_type" = "repo" ]; then
|
|
2849
2877
|
local version
|
|
2850
2878
|
version=$(get_version)
|
|
2879
|
+
# Same cap the brief sub-path reads, so "fast" means one thing.
|
|
2880
|
+
local _repo_max_iter="${LOKI_MAX_ITERATIONS:-3}"
|
|
2851
2881
|
mkdir -p "$LOKI_DIR" 2>/dev/null || true
|
|
2852
2882
|
find "$LOKI_DIR" -maxdepth 1 -name 'repo-prd-*.md' -mtime +1 -delete 2>/dev/null || true
|
|
2853
2883
|
local repo_prd="$LOKI_DIR/repo-prd-$$.md"
|
|
2854
|
-
synthesize_repo_prd "$repo_prd"
|
|
2884
|
+
synthesize_repo_prd "$repo_prd" "$repo_depth"
|
|
2855
2885
|
prd_file="$repo_prd"
|
|
2856
2886
|
|
|
2887
|
+
if [ "$repo_depth" = "fast" ]; then
|
|
2888
|
+
set_ttfv_lightweight_profile "$_repo_max_iter"
|
|
2889
|
+
export LOKI_TTFV=repo-fast
|
|
2890
|
+
no_plan=true
|
|
2891
|
+
fi
|
|
2892
|
+
|
|
2857
2893
|
echo -e "${BOLD}Loki Mode v$version - Repo improvement${NC}"
|
|
2858
2894
|
echo ""
|
|
2859
2895
|
echo -e "${CYAN}Repo:${NC} $LOKI_REPO_TARGET"
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2896
|
+
if [ "$repo_depth" = "fast" ]; then
|
|
2897
|
+
echo -e "${DIM}Loki will inspect this project, pick one high-value improvement,${NC}"
|
|
2898
|
+
echo -e "${DIM}implement it, and test it.${NC}"
|
|
2899
|
+
echo -e "${DIM}Fast pass: $_repo_max_iter iterations max, simple tier, council and${NC}"
|
|
2900
|
+
echo -e "${DIM}heavy phases OFF. Reduced gates: you get measured run evidence${NC}"
|
|
2901
|
+
echo -e "${DIM}(diffs, cost, time), NOT a council-verified verdict.${NC}"
|
|
2902
|
+
echo -e "${DIM}Run full verification with: loki start \"$LOKI_REPO_TARGET\" --full${NC}"
|
|
2903
|
+
else
|
|
2904
|
+
echo -e "${DIM}Loki will inspect this project, pick one high-value improvement,${NC}"
|
|
2905
|
+
echo -e "${DIM}implement it, test it, and produce verified evidence.${NC}"
|
|
2906
|
+
echo -e "${DIM}Full depth: quality gates and completion council stay ON.${NC}"
|
|
2907
|
+
fi
|
|
2863
2908
|
echo -e "${DIM}Scope it yourself instead with: loki start \"<what you want>\".${NC}"
|
|
2864
2909
|
echo ""
|
|
2865
2910
|
fi
|
|
@@ -15070,8 +15115,16 @@ BRIEFEOF
|
|
|
15070
15115
|
# Build a bounded input for `loki start <repo-directory>`. This intentionally
|
|
15071
15116
|
# contains no caller-controlled path: repository identity comes from cwd, while
|
|
15072
15117
|
# the task remains stable, reviewable, and safe to render into a prompt.
|
|
15118
|
+
#
|
|
15119
|
+
# The trailing Mode line is the only variable part, and it must match what the
|
|
15120
|
+
# run actually does: a fast pass runs with the council off, so a PRD telling the
|
|
15121
|
+
# agent it is under "full verification" would be a false claim in the prompt
|
|
15122
|
+
# itself. The body stays a quoted heredoc (no expansion, nothing injectable);
|
|
15123
|
+
# the Mode line is appended from a fixed string chosen by a validated depth.
|
|
15124
|
+
# Usage: synthesize_repo_prd <output_file> [full|fast]
|
|
15073
15125
|
synthesize_repo_prd() {
|
|
15074
15126
|
local out_file="$1"
|
|
15127
|
+
local depth="${2:-full}"
|
|
15075
15128
|
mkdir -p "$(dirname "$out_file")" 2>/dev/null || true
|
|
15076
15129
|
cat > "$out_file" <<'REPOEOF'
|
|
15077
15130
|
# Existing Repository Improvement
|
|
@@ -15091,9 +15144,17 @@ Select one small, high-value user-facing improvement and implement it completely
|
|
|
15091
15144
|
- Reuse the repository's established architecture and dependencies.
|
|
15092
15145
|
- Keep the change easy to review and roll back.
|
|
15093
15146
|
- Do not claim deployment, adoption, or customer value without evidence.
|
|
15094
|
-
|
|
15095
|
-
**Mode:** Existing repository improvement (full verification)
|
|
15096
15147
|
REPOEOF
|
|
15148
|
+
if [ "$depth" = "fast" ]; then
|
|
15149
|
+
printf '%s\n' \
|
|
15150
|
+
"- This is a fast, capped pass. Keep the change small enough to finish and prove." \
|
|
15151
|
+
"" \
|
|
15152
|
+
"**Mode:** Existing repository improvement (fast pass, reduced gates)." \
|
|
15153
|
+
"The completion council and heavy review phases are OFF on this run." \
|
|
15154
|
+
"Do not describe the result as council-verified or fully verified." >> "$out_file"
|
|
15155
|
+
else
|
|
15156
|
+
printf '\n**Mode:** Existing repository improvement (full verification)\n' >> "$out_file"
|
|
15157
|
+
fi
|
|
15097
15158
|
}
|
|
15098
15159
|
|
|
15099
15160
|
# Quick mode - lightweight single-task execution
|
package/autonomy/run.sh
CHANGED
|
@@ -7784,9 +7784,14 @@ print_ttfv_next_steps() {
|
|
|
7784
7784
|
echo "============================================================"
|
|
7785
7785
|
echo ""
|
|
7786
7786
|
echo " What I did:"
|
|
7787
|
-
if [ "$mode" = "brief" ]; then
|
|
7787
|
+
if [ "$mode" = "brief" ] || [ "$mode" = "repo-fast" ]; then
|
|
7788
|
+
if [ "$mode" = "repo-fast" ]; then
|
|
7789
|
+
echo " - Inspected your repository on a fast, lightweight first"
|
|
7790
|
+
echo " pass (council off, simple tier, capped iterations)."
|
|
7791
|
+
else
|
|
7788
7792
|
echo " - Worked from your one-line brief on a fast, lightweight first"
|
|
7789
7793
|
echo " pass (council off, simple tier, capped iterations)."
|
|
7794
|
+
fi
|
|
7790
7795
|
echo " - Generated a proof-of-run (diffs, cost, time)."
|
|
7791
7796
|
else
|
|
7792
7797
|
echo " - Analyzed your codebase and generated a PRD, then ran a full"
|
|
@@ -7808,7 +7813,7 @@ print_ttfv_next_steps() {
|
|
|
7808
7813
|
echo " loki proof list"
|
|
7809
7814
|
fi
|
|
7810
7815
|
echo ""
|
|
7811
|
-
if [ "$mode" = "brief" ]; then
|
|
7816
|
+
if [ "$mode" = "brief" ] || [ "$mode" = "repo-fast" ]; then
|
|
7812
7817
|
echo " Go deeper (full RARV-C depth, council-gated):"
|
|
7813
7818
|
echo " loki start # continue / harden this project"
|
|
7814
7819
|
echo " loki start ./prd.md # build from a full PRD"
|
package/bin/loki
CHANGED
|
@@ -276,6 +276,7 @@ _loki_start_needs_bash() {
|
|
|
276
276
|
for a in "$@"; do
|
|
277
277
|
case "$a" in
|
|
278
278
|
--parallel|--bg|--background|--github|--api|--sandbox \
|
|
279
|
+
|--fast|--repo-fast|--full|--repo-full \
|
|
279
280
|
|--bmad-project|--openspec \
|
|
280
281
|
|--mirofish|--mirofish-docker|--mirofish-rounds|--mirofish-timeout|--mirofish-bg \
|
|
281
282
|
|--issue|--dry-run|--no-start|--output|--worktree|-w|--pr|--ship|--detach|-d)
|
package/dashboard/__init__.py
CHANGED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Outcome Canary Evaluation
|
|
2
|
+
|
|
3
|
+
`outcome-canary-evaluate.py` converts consented, locally recorded canary
|
|
4
|
+
observations into one deterministic aggregate verdict. It never invokes a provider,
|
|
5
|
+
changes an assignment, or promotes a route.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python3 tools/outcome-canary-evaluate.py report.json observations.json \
|
|
9
|
+
--enable-evaluation --control-route safe --canary-percent 10 --json
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Evaluation is opt-in. Without `--enable-evaluation`, the command refuses.
|
|
13
|
+
|
|
14
|
+
## Observation contract
|
|
15
|
+
|
|
16
|
+
The input is one `loki-outcome-canary-observations/v1` JSON object:
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"observations": "loki-outcome-canary-observations/v1",
|
|
21
|
+
"report_sha256": "<sha256 of the exact router report>",
|
|
22
|
+
"source_sha256": "<source_sha256 from that report>",
|
|
23
|
+
"items": [
|
|
24
|
+
{
|
|
25
|
+
"subject": "locally chosen opaque key",
|
|
26
|
+
"assignment": "control",
|
|
27
|
+
"route": "safe",
|
|
28
|
+
"accepted": true,
|
|
29
|
+
"risk": 0.1
|
|
30
|
+
}
|
|
31
|
+
]
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Each item has exactly those five fields. Subjects must be unique and non-empty;
|
|
36
|
+
`accepted` is a JSON Boolean; and `risk` is a finite number from zero through one.
|
|
37
|
+
The file is capped at 5 MiB and 100,000 observations. Duplicate JSON keys are
|
|
38
|
+
rejected.
|
|
39
|
+
|
|
40
|
+
The evaluator hashes the exact report and observation bytes, rechecks the report's
|
|
41
|
+
underlying source digest, and reruns the released deterministic canary assignment
|
|
42
|
+
for every subject. A recorded arm or route that does not match that assignment
|
|
43
|
+
refuses the entire evaluation. Output contains no subject keys.
|
|
44
|
+
|
|
45
|
+
## Verdict policy
|
|
46
|
+
|
|
47
|
+
Both arms must reach `--min-samples` (default 5). The tool computes integer accepted
|
|
48
|
+
basis points and mean observed risk for each arm.
|
|
49
|
+
|
|
50
|
+
- `ROLLBACK`: canary acceptance is below control, or canary mean risk exceeds
|
|
51
|
+
`--max-risk` (default 0.25).
|
|
52
|
+
- `PROMOTE`: canary acceptance lift reaches `--min-lift-bps` (default 1) and canary
|
|
53
|
+
mean risk is no higher than control.
|
|
54
|
+
- `HOLD`: the evidence is valid and sufficiently sampled but meets neither rule.
|
|
55
|
+
|
|
56
|
+
These are offline recommendations, not routing actions. A consented operator still
|
|
57
|
+
controls whether to apply any change. Malformed, drifted, mismatched, sparse, or
|
|
58
|
+
unbound evidence returns `REFUSED` rather than a partial verdict.
|
|
59
|
+
|
|
60
|
+
## Output and exit codes
|
|
61
|
+
|
|
62
|
+
`--json` emits the aggregate arms, policy, exact evidence digests, verdict, and
|
|
63
|
+
refusal reasons. The default output is a short human-readable summary. Exit 0 means
|
|
64
|
+
a verdict was produced (including `ROLLBACK`), 3 means evaluation was refused, 64
|
|
65
|
+
is an invocation error, and 66 is a missing input file.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Outcome Canary
|
|
2
|
+
|
|
3
|
+
`python3 tools/outcome-canary.py report.json --enable-canary --subject <key> --control-route <name>`
|
|
4
|
+
plans a reversible canary split between the route an [outcome router](./OUTCOME-ROUTER.md)
|
|
5
|
+
report selected and an operator-named control. It is a planner: it never invokes or
|
|
6
|
+
switches a provider, never weakens a gate, and writes nothing.
|
|
7
|
+
|
|
8
|
+
The canary is opt-in. Without `--enable-canary` it refuses, so a plan can never be
|
|
9
|
+
produced by accident.
|
|
10
|
+
|
|
11
|
+
## Evidence it demands
|
|
12
|
+
|
|
13
|
+
The tool refuses unless the evidence stands up on its own terms:
|
|
14
|
+
|
|
15
|
+
- the report parses, is a `loki-outcome-router/v1` object, and carries `source`,
|
|
16
|
+
a 64-hex `source_sha256`, and a `candidates` list
|
|
17
|
+
- the recorded source file still exists and still hashes to `source_sha256`, so a
|
|
18
|
+
plan cannot rest on trials that changed after they were measured
|
|
19
|
+
- the exact router-report bytes are hashed as `report_sha256`; that digest is
|
|
20
|
+
emitted in the plan and participates in assignment, so a report edited after
|
|
21
|
+
review cannot reuse the prior plan binding
|
|
22
|
+
- the report selected a primary route, the control route is present, both are
|
|
23
|
+
`eligible`, and the two differ
|
|
24
|
+
- `trials` and `mean_risk` on both routes are strictly valid numbers: booleans,
|
|
25
|
+
`NaN`, and `Infinity` are rejected, not coerced
|
|
26
|
+
- both routes have equal trial counts, so the two arms rest on matched evidence
|
|
27
|
+
- neither route's mean risk exceeds `--max-risk` (default `.25`)
|
|
28
|
+
- `--canary-percent` (default `10`) is a finite number from 0 to 100
|
|
29
|
+
|
|
30
|
+
Every failed check is reported in `refusal_reasons`; the plan is refused as a whole
|
|
31
|
+
rather than partially applied.
|
|
32
|
+
|
|
33
|
+
## Assignment
|
|
34
|
+
|
|
35
|
+
Assignment is deterministic and needs no stored state. The tool hashes a
|
|
36
|
+
NUL-joined, domain-separated string of `loki-outcome-canary/v1`, the subject, the
|
|
37
|
+
router-report digest, the source digest, the primary route, the control route, and the percentage, then takes
|
|
38
|
+
that sha256 modulo 10000 against the percentage. The same subject and the same
|
|
39
|
+
evidence always land in the same arm, in any process; different subjects spread
|
|
40
|
+
across arms. Because the digest includes the evidence, changing the trials
|
|
41
|
+
reshuffles the split rather than silently carrying an old assignment forward.
|
|
42
|
+
|
|
43
|
+
Percent 0 assigns every subject to control and percent 100 assigns every subject to
|
|
44
|
+
the canary, which is what makes the rollback a real command rather than a promise.
|
|
45
|
+
|
|
46
|
+
## Rollback
|
|
47
|
+
|
|
48
|
+
A plan always carries `reversible: true` and a `rollback` object naming the control
|
|
49
|
+
route, its effect, and the exact command that assigns control to everyone:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
python3 tools/outcome-canary.py report.json --enable-canary \
|
|
53
|
+
--subject <key> --control-route <name> --canary-percent 0
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Output and exit codes
|
|
57
|
+
|
|
58
|
+
`--json` emits the whole plan for automation; the default is human-readable. Exit 0
|
|
59
|
+
means a plan exists, 3 means the evidence or policy cannot support one (including a
|
|
60
|
+
missing `--enable-canary`, and a source that drifted), 64 is an invocation error,
|
|
61
|
+
and 66 is a missing report file or a report whose evidence source is gone.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Outcome Router
|
|
2
|
+
|
|
3
|
+
`python3 tools/outcome-router.py trials.jsonl` recommends a route only from measured local trials. Each JSONL row records `route`, strict-boolean `accepted`, non-negative `cost_usd`, positive `duration_minutes`, `risk` from 0 to 1, and optional string `verifier`.
|
|
4
|
+
|
|
5
|
+
The advisor first rejects routes above the risk ceiling, with sparse evidence, or without an accepted outcome. It then ranks eligible routes by the harmonic mean of accepted outcomes per dollar and per minute, penalized by mean risk. Invalid input poisons eligibility instead of disappearing from the evidence basis. It never invokes or switches a provider and never weakens a gate.
|
|
6
|
+
|
|
7
|
+
Use `--json` for automation, `--min-trials` and `--max-risk` for evidence policy, and `--risk-weight` to tune the risk penalty. Exit 0 means a recommendation exists, 3 means the evidence cannot support one, 64 is invocation error, and 66 is a missing input file.
|
package/loki-ts/dist/loki.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.
|
|
2
|
+
var p_=Object.create;var{getPrototypeOf:d_,defineProperty:eK,getOwnPropertyNames:c_}=Object;var l_=Object.prototype.hasOwnProperty;function i_(Z){return this[Z]}var a_,s_,n_=(Z,X,Q)=>{var Y=Z!=null&&typeof Z==="object";if(Y){var J=X?a_??=new WeakMap:s_??=new WeakMap,z=J.get(Z);if(z)return z}Q=Z!=null?p_(d_(Z)):{};let K=X||!Z||!Z.__esModule?eK(Q,"default",{value:Z,enumerable:!0}):Q;for(let $ of c_(Z))if(!l_.call(K,$))eK(K,$,{get:i_.bind(Z,$),enumerable:!0});if(Y)J.set(Z,K);return K};var HQ=(Z,X)=>()=>(X||Z((X={exports:{}}).exports,X),X.exports);var o_=(Z)=>Z;function r_(Z,X){this[Z]=o_.bind(null,X)}var l0=(Z,X)=>{for(var Q in X)eK(Z,Q,{get:X[Q],enumerable:!0,configurable:!0,set:r_.bind(X,Q)})};var p=(Z,X)=>()=>(Z&&(X=Z(Z=0)),X);var t0=import.meta.require;var xO={};l0(xO,{lokiDir:()=>j0,homeLokiDir:()=>R4,findRepoRootForVersion:()=>X$,REPO_ROOT:()=>i0});import{resolve as n7,dirname as Z$}from"path";import{fileURLToPath as t_}from"url";import{existsSync as UQ}from"fs";import{homedir as e_}from"os";function Zf(){let Z=kO;for(let X=0;X<6;X++){if(UQ(n7(Z,"VERSION"))&&UQ(n7(Z,"autonomy/run.sh")))return Z;let Q=Z$(Z);if(Q===Z)break;Z=Q}return n7(kO,"..","..","..")}function X$(Z){let X=Z;for(let Q=0;Q<6;Q++){if(UQ(n7(X,"VERSION"))&&UQ(n7(X,"autonomy/run.sh")))return X;let Y=Z$(X);if(Y===X)break;X=Y}return n7(Z,"..","..","..")}function j0(){return process.env.LOKI_DIR??n7(process.cwd(),".loki")}function R4(){return n7(e_(),".loki")}var kO,i0;var H8=p(()=>{kO=Z$(t_(import.meta.url));i0=Zf()});import{readFileSync as Xf}from"fs";import{resolve as Qf,dirname as Yf}from"path";import{fileURLToPath as Jf}from"url";function h3(){if(h5!==null)return h5;let Z="9.22.0";if(typeof Z==="string"&&Z.length>0)return h5=Z,h5;try{let X=Yf(Jf(import.meta.url)),Q=X$(X);h5=Xf(Qf(Q,"VERSION"),"utf-8").trim()}catch{h5="unknown"}return h5}var h5=null;var BQ=p(()=>{H8()});var _O={};l0(_O,{runOrThrow:()=>Tf,run:()=>E0,readStreamCapped:()=>NQ,commandVersion:()=>Cf,commandExists:()=>X9,ShellError:()=>Q$,MAX_STDOUT_BYTES:()=>bO});async function NQ(Z,X=bO){let Q=Z.getReader(),Y=new TextDecoder,J="",z=0;try{while(z<X){let{done:K,value:$}=await Q.read();if(K)break;if(!$)continue;if(z+=$.byteLength,z>X){let V=$.byteLength-(z-X);J+=Y.decode($.subarray(0,V),{stream:!0});break}J+=Y.decode($,{stream:!0})}J+=Y.decode()}finally{try{await Q.cancel()}catch{}Q.releaseLock()}return J}async function E0(Z,X={}){let Q=Bun.spawn({cmd:[...Z],stdout:"pipe",stderr:"pipe",env:X.env?{...process.env,...X.env}:process.env,cwd:X.cwd}),Y,J;if(X.timeoutMs&&X.timeoutMs>0)Y=setTimeout(()=>{try{Q.kill("SIGTERM")}catch{}J=setTimeout(()=>{try{Q.kill("SIGKILL")}catch{}},2000)},X.timeoutMs);try{let[z,K,$]=await Promise.all([NQ(Q.stdout),new Response(Q.stderr).text(),Q.exited]);return{stdout:z,stderr:K,exitCode:$}}finally{if(Y)clearTimeout(Y);if(J)clearTimeout(J)}}async function Tf(Z,X={}){let Q=await E0(Z,X);if(Q.exitCode!==0)throw new Q$(`command failed (${Q.exitCode}): ${Z.join(" ")}`,Q.exitCode,Q.stdout,Q.stderr);return Q}async function X9(Z){let X=wf(Z),Q=await E0(["sh","-c",`command -v ${X}`],{timeoutMs:5000});if(Q.exitCode===0)return Q.stdout.trim()||null;return null}function wf(Z){if(!/^[A-Za-z0-9._/-]+$/.test(Z))throw Error(`refused to shell-escape suspect token: ${Z}`);return Z}async function Cf(Z,X="--version"){if(!await X9(Z))return null;let Y=await E0([Z,X],{timeoutMs:5000});if(Y.exitCode!==0)return null;return((Y.stdout||Y.stderr).split(/\r?\n/)[0]?.trim()??"")||null}var bO=16777216,Q$;var x9=p(()=>{Q$=class Q$ extends Error{message;exitCode;stdout;stderr;constructor(Z,X,Q,Y){super(Z);this.message=Z;this.exitCode=X;this.stdout=Q;this.stderr=Y;this.name="ShellError"}}});function o7(Z){return Ff?"":Z}var Ff,A0,F8,_0,$W0,a0,V8,Q9,h;var S6=p(()=>{Ff=(process.env.NO_COLOR??"").length>0;A0=o7("\x1B[0;31m"),F8=o7("\x1B[0;32m"),_0=o7("\x1B[1;33m"),$W0=o7("\x1B[0;34m"),a0=o7("\x1B[0;36m"),V8=o7("\x1B[1m"),Q9=o7("\x1B[2m"),h=o7("\x1B[0m")});import{existsSync as ff}from"fs";async function E7(){if(x4!==void 0)return x4;let Z="/opt/homebrew/bin/python3.12";if(ff(Z))return x4=Z,Z;let X=await X9("python3.12");if(X)return x4=X,X;let Q=await X9("python3");return x4=Q,Q}async function Y7(Z,X={}){let Q=await E7();if(!Q)return{stdout:"",stderr:"python3 not found",exitCode:127};return E0([Q,"-c",Z],X)}var x4;var r7=p(()=>{x9()});var XA={};l0(XA,{runStatus:()=>Vh});import{existsSync as Y9,readFileSync as g3,readdirSync as aO,statSync as sO}from"fs";import{resolve as h8,basename as ef}from"path";import{homedir as Zh}from"os";function nO(Z){let X=Math.trunc(Z);if(X>=1e6)return`${(Math.trunc(X/1e6*10)/10).toFixed(1)}M`;if(X>=1000)return`${(Math.trunc(X/1000*10)/10).toFixed(1)}K`;return String(X)}function oO(Z,X,Q){if(X===0)return null;let Y=Math.trunc(Z*100/X),J=Math.trunc(Z*AQ/X);if(J>AQ)J=AQ;let z=AQ-J,K=F8;if(Y>=80)K=A0;else if(Y>=50)K=_0;let $="=".repeat(Math.max(0,J))+" ".repeat(Math.max(0,z)),V=nO(Z),W=nO(X);return` ${V8}${Q}${h} ${K}[${$}]${h} ${Y}% (${V} / ${W})`}async function Qh(){if(await X9("jq"))return!0;return process.stdout.write(`${A0}Error: jq is required but not installed.${h}
|
|
3
3
|
`),process.stdout.write(`Install with:
|
|
4
4
|
`),process.stdout.write(` brew install jq (macOS)
|
|
5
5
|
`),process.stdout.write(` apt install jq (Debian/Ubuntu)
|
|
@@ -1236,4 +1236,4 @@ Set LOKI_LEGACY_BASH=1 to force the bash CLI for every command.
|
|
|
1236
1236
|
`),2}case"start":{let{runStart:Y}=await Promise.resolve().then(() => (g_(),v_));return Y(Q)}default:return process.stderr.write(`Unknown command: ${X}
|
|
1237
1237
|
`),process.stderr.write(m_),2}}lO();process.on("SIGINT",()=>process.exit(130));process.on("SIGTERM",()=>process.exit(143));var dV0=await pV0(Bun.argv.slice(2));process.exit(dV0);
|
|
1238
1238
|
|
|
1239
|
-
//# debugId=
|
|
1239
|
+
//# debugId=DE8569120D6E31F464756E2164756E21
|
package/mcp/__init__.py
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loki-mode",
|
|
3
3
|
"mcpName": "io.github.asklokesh/loki-mode",
|
|
4
|
-
"version": "9.
|
|
4
|
+
"version": "9.22.0",
|
|
5
5
|
"description": "Loki Mode by Autonomi. Autonomous spec-to-product system: takes a PRD, GitHub issue, OpenAPI/JSON/YAML, or one-line brief to a deployed app via the RARV-C closure loop with 8 quality gates. Provider-agnostic (Claude Code, OpenAI Codex, Cline, Aider).",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"agent",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
|
|
3
3
|
"name": "loki-mode",
|
|
4
4
|
"displayName": "Loki Mode",
|
|
5
|
-
"version": "9.
|
|
5
|
+
"version": "9.22.0",
|
|
6
6
|
"description": "Autonomous spec-to-product build system with a built-in trust layer (RARV-C closure loop, 8 quality gates, completion council). Ships Loki's spec-hardening, drift-detection, and deterministic PR verification commands plus the Loki MCP server.",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Autonomi",
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Evaluate source-bound canary observations without changing a route."""
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import importlib.util
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import pathlib
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
OK, REFUSED, USAGE, NO_INPUT = 0, 3, 64, 66
|
|
13
|
+
DOMAIN = "loki-outcome-canary-evaluation/v1"
|
|
14
|
+
OBSERVATIONS = "loki-outcome-canary-observations/v1"
|
|
15
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
16
|
+
MAX_ITEMS = 100_000
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Parser(argparse.ArgumentParser):
|
|
20
|
+
def error(self, message):
|
|
21
|
+
self.print_usage(sys.stderr)
|
|
22
|
+
self.exit(USAGE, f"{self.prog}: error: {message}\n")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DuplicateKey(ValueError):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _object(pairs):
|
|
30
|
+
out = {}
|
|
31
|
+
for key, value in pairs:
|
|
32
|
+
if key in out:
|
|
33
|
+
raise DuplicateKey(f"duplicate key: {key}")
|
|
34
|
+
out[key] = value
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _number(value, low, high):
|
|
39
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
40
|
+
return None
|
|
41
|
+
if not math.isfinite(value) or value < low or value > high:
|
|
42
|
+
return None
|
|
43
|
+
return float(value)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _integer(value, low, high):
|
|
47
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < low or value > high:
|
|
48
|
+
return None
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _sha256(data):
|
|
53
|
+
return hashlib.sha256(data).hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _read_json(path):
|
|
57
|
+
with open(path, "rb") as handle:
|
|
58
|
+
data = handle.read(MAX_BYTES + 1)
|
|
59
|
+
if len(data) > MAX_BYTES:
|
|
60
|
+
raise ValueError(f"input exceeds {MAX_BYTES} bytes")
|
|
61
|
+
return json.loads(data.decode("utf-8"), object_pairs_hook=_object), _sha256(data)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _load_planner():
|
|
65
|
+
path = pathlib.Path(__file__).with_name("outcome-canary.py")
|
|
66
|
+
spec = importlib.util.spec_from_file_location("outcome_canary", path)
|
|
67
|
+
if spec is None or spec.loader is None:
|
|
68
|
+
raise RuntimeError("cannot load outcome canary planner")
|
|
69
|
+
module = importlib.util.module_from_spec(spec)
|
|
70
|
+
spec.loader.exec_module(module)
|
|
71
|
+
return module
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _arm(items):
|
|
75
|
+
trials = len(items)
|
|
76
|
+
accepted = sum(1 for item in items if item["accepted"])
|
|
77
|
+
return {
|
|
78
|
+
"trials": trials,
|
|
79
|
+
"accepted": accepted,
|
|
80
|
+
"accepted_bps": (accepted * 10_000) // trials,
|
|
81
|
+
"mean_risk": sum(item["risk"] for item in items) / trials,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def evaluate(report_path, observations_path, control_route, canary_percent=10.0,
|
|
86
|
+
max_risk=.25, min_samples=5, min_lift_bps=1, enable_evaluation=False):
|
|
87
|
+
"""Return a deterministic aggregate verdict or an explained refusal."""
|
|
88
|
+
reasons = []
|
|
89
|
+
percent = _number(canary_percent, 0, 100)
|
|
90
|
+
ceiling = _number(max_risk, 0, 1)
|
|
91
|
+
samples = _integer(min_samples, 1, MAX_ITEMS)
|
|
92
|
+
lift = _integer(min_lift_bps, 0, 10_000)
|
|
93
|
+
if not enable_evaluation:
|
|
94
|
+
reasons.append("evaluation is opt-in; pass --enable-evaluation")
|
|
95
|
+
if not isinstance(control_route, str) or not control_route.strip():
|
|
96
|
+
reasons.append("control route must be non-empty")
|
|
97
|
+
if percent is None:
|
|
98
|
+
reasons.append("canary percent must be a finite number between 0 and 100")
|
|
99
|
+
if ceiling is None:
|
|
100
|
+
reasons.append("max risk must be a finite number between 0 and 1")
|
|
101
|
+
if samples is None:
|
|
102
|
+
reasons.append(f"min samples must be an integer between 1 and {MAX_ITEMS}")
|
|
103
|
+
if lift is None:
|
|
104
|
+
reasons.append("min lift bps must be an integer between 0 and 10000")
|
|
105
|
+
|
|
106
|
+
out = {
|
|
107
|
+
"evaluation": DOMAIN,
|
|
108
|
+
"report": os.path.abspath(report_path),
|
|
109
|
+
"report_sha256": None,
|
|
110
|
+
"source_sha256": None,
|
|
111
|
+
"observations": os.path.abspath(observations_path),
|
|
112
|
+
"observations_sha256": None,
|
|
113
|
+
"control_route": control_route,
|
|
114
|
+
"canary_route": None,
|
|
115
|
+
"canary_percent": percent,
|
|
116
|
+
"max_risk": ceiling,
|
|
117
|
+
"min_samples": samples,
|
|
118
|
+
"min_lift_bps": lift,
|
|
119
|
+
"control": None,
|
|
120
|
+
"canary": None,
|
|
121
|
+
"verdict": None,
|
|
122
|
+
"refusal_reasons": reasons,
|
|
123
|
+
}
|
|
124
|
+
try:
|
|
125
|
+
observations, observations_sha256 = _read_json(observations_path)
|
|
126
|
+
out["observations_sha256"] = observations_sha256
|
|
127
|
+
except Exception as exc:
|
|
128
|
+
out["refusal_reasons"].append(f"observations are malformed: {exc}")
|
|
129
|
+
return out
|
|
130
|
+
if not isinstance(observations, dict):
|
|
131
|
+
out["refusal_reasons"].append("observations are not a JSON object")
|
|
132
|
+
return out
|
|
133
|
+
if observations.get("observations") != OBSERVATIONS:
|
|
134
|
+
out["refusal_reasons"].append(f"observations version is not {OBSERVATIONS}")
|
|
135
|
+
items = observations.get("items")
|
|
136
|
+
if not isinstance(items, list):
|
|
137
|
+
out["refusal_reasons"].append("observations have no items list")
|
|
138
|
+
return out
|
|
139
|
+
if len(items) > MAX_ITEMS:
|
|
140
|
+
out["refusal_reasons"].append(f"observations exceed {MAX_ITEMS} items")
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
planner = _load_planner()
|
|
145
|
+
report, report_sha256, report_reasons = planner.load_report(report_path)
|
|
146
|
+
except Exception as exc:
|
|
147
|
+
out["refusal_reasons"].append(f"report is malformed: {exc}")
|
|
148
|
+
return out
|
|
149
|
+
out["report_sha256"] = report_sha256
|
|
150
|
+
out["refusal_reasons"].extend(report_reasons)
|
|
151
|
+
if report is None:
|
|
152
|
+
return out
|
|
153
|
+
out["source_sha256"] = report.get("source_sha256")
|
|
154
|
+
out["canary_route"] = report.get("selected_route")
|
|
155
|
+
if observations.get("report_sha256") != report_sha256:
|
|
156
|
+
out["refusal_reasons"].append("observations are not bound to the exact router report")
|
|
157
|
+
if observations.get("source_sha256") != report.get("source_sha256"):
|
|
158
|
+
out["refusal_reasons"].append("observations are not bound to the exact evidence source")
|
|
159
|
+
|
|
160
|
+
seen = set()
|
|
161
|
+
arms = {"control": [], "canary": []}
|
|
162
|
+
for index, item in enumerate(items):
|
|
163
|
+
label = f"item {index}"
|
|
164
|
+
if not isinstance(item, dict):
|
|
165
|
+
out["refusal_reasons"].append(f"{label} is not an object")
|
|
166
|
+
continue
|
|
167
|
+
if set(item) != {"subject", "assignment", "route", "accepted", "risk"}:
|
|
168
|
+
out["refusal_reasons"].append(f"{label} has a non-canonical shape")
|
|
169
|
+
continue
|
|
170
|
+
subject = item.get("subject")
|
|
171
|
+
assignment = item.get("assignment")
|
|
172
|
+
route = item.get("route")
|
|
173
|
+
accepted = item.get("accepted")
|
|
174
|
+
risk = _number(item.get("risk"), 0, 1)
|
|
175
|
+
if not isinstance(subject, str) or not subject.strip():
|
|
176
|
+
out["refusal_reasons"].append(f"{label} has an invalid subject")
|
|
177
|
+
continue
|
|
178
|
+
if subject in seen:
|
|
179
|
+
out["refusal_reasons"].append(f"{label} repeats a subject")
|
|
180
|
+
continue
|
|
181
|
+
seen.add(subject)
|
|
182
|
+
if assignment not in arms or not isinstance(route, str) or not isinstance(accepted, bool) or risk is None:
|
|
183
|
+
out["refusal_reasons"].append(f"{label} has invalid measured values")
|
|
184
|
+
continue
|
|
185
|
+
plan = planner.plan(report_path, subject, control_route, percent, ceiling, True)
|
|
186
|
+
if plan.get("refusal_reasons"):
|
|
187
|
+
out["refusal_reasons"].append(f"{label} cannot be rebound to a valid canary plan")
|
|
188
|
+
continue
|
|
189
|
+
if assignment != plan.get("assignment") or route != plan.get("route"):
|
|
190
|
+
out["refusal_reasons"].append(f"{label} does not match its deterministic assignment")
|
|
191
|
+
continue
|
|
192
|
+
arms[assignment].append({"accepted": accepted, "risk": risk})
|
|
193
|
+
|
|
194
|
+
if out["refusal_reasons"]:
|
|
195
|
+
return out
|
|
196
|
+
for name in ("control", "canary"):
|
|
197
|
+
if len(arms[name]) < samples:
|
|
198
|
+
out["refusal_reasons"].append(
|
|
199
|
+
f"{name} arm has {len(arms[name])} samples; requires at least {samples}"
|
|
200
|
+
)
|
|
201
|
+
if out["refusal_reasons"]:
|
|
202
|
+
return out
|
|
203
|
+
|
|
204
|
+
control = _arm(arms["control"])
|
|
205
|
+
canary = _arm(arms["canary"])
|
|
206
|
+
out["control"], out["canary"] = control, canary
|
|
207
|
+
accepted_delta = canary["accepted_bps"] - control["accepted_bps"]
|
|
208
|
+
out["accepted_delta_bps"] = accepted_delta
|
|
209
|
+
if canary["mean_risk"] > ceiling or accepted_delta < 0:
|
|
210
|
+
out["verdict"] = "ROLLBACK"
|
|
211
|
+
elif accepted_delta >= lift and canary["mean_risk"] <= control["mean_risk"]:
|
|
212
|
+
out["verdict"] = "PROMOTE"
|
|
213
|
+
else:
|
|
214
|
+
out["verdict"] = "HOLD"
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def main(argv=None):
|
|
219
|
+
parser = Parser(prog="outcome-canary-evaluate")
|
|
220
|
+
parser.add_argument("report")
|
|
221
|
+
parser.add_argument("observations")
|
|
222
|
+
parser.add_argument("--enable-evaluation", action="store_true")
|
|
223
|
+
parser.add_argument("--control-route", required=True)
|
|
224
|
+
parser.add_argument("--canary-percent", type=float, default=10.0)
|
|
225
|
+
parser.add_argument("--max-risk", type=float, default=.25)
|
|
226
|
+
parser.add_argument("--min-samples", type=int, default=5)
|
|
227
|
+
parser.add_argument("--min-lift-bps", type=int, default=1)
|
|
228
|
+
parser.add_argument("--json", action="store_true")
|
|
229
|
+
args = parser.parse_args(argv)
|
|
230
|
+
for path in (args.report, args.observations):
|
|
231
|
+
if not os.path.isfile(path):
|
|
232
|
+
print(f"outcome-canary-evaluate: no such file: {path}", file=sys.stderr)
|
|
233
|
+
return NO_INPUT
|
|
234
|
+
result = evaluate(
|
|
235
|
+
args.report, args.observations, args.control_route, args.canary_percent,
|
|
236
|
+
args.max_risk, args.min_samples, args.min_lift_bps, args.enable_evaluation,
|
|
237
|
+
)
|
|
238
|
+
if args.json:
|
|
239
|
+
print(json.dumps(result, sort_keys=True))
|
|
240
|
+
elif result["refusal_reasons"]:
|
|
241
|
+
print("Canary evaluation: REFUSED")
|
|
242
|
+
for reason in result["refusal_reasons"]:
|
|
243
|
+
print(f" {reason}")
|
|
244
|
+
else:
|
|
245
|
+
print(f"Canary evaluation: {result['verdict']}")
|
|
246
|
+
print(
|
|
247
|
+
f" control={result['control']['accepted']}/{result['control']['trials']} "
|
|
248
|
+
f"({result['control']['accepted_bps']} bps)"
|
|
249
|
+
)
|
|
250
|
+
print(
|
|
251
|
+
f" canary={result['canary']['accepted']}/{result['canary']['trials']} "
|
|
252
|
+
f"({result['canary']['accepted_bps']} bps)"
|
|
253
|
+
)
|
|
254
|
+
print(f" accepted delta: {result['accepted_delta_bps']} bps")
|
|
255
|
+
print(f" report sha256: {result['report_sha256']}")
|
|
256
|
+
print(f" observations sha256: {result['observations_sha256']}")
|
|
257
|
+
return REFUSED if result["refusal_reasons"] else OK
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
if __name__ == "__main__":
|
|
261
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Plan a reversible canary split between two measured routes, or refuse."""
|
|
3
|
+
import argparse, hashlib, json, math, os, sys
|
|
4
|
+
|
|
5
|
+
OK, REFUSED, USAGE, NO_INPUT = 0, 3, 64, 66
|
|
6
|
+
REPORT = "loki-outcome-router/v1"
|
|
7
|
+
DOMAIN = "loki-outcome-canary/v1"
|
|
8
|
+
|
|
9
|
+
class Parser(argparse.ArgumentParser):
|
|
10
|
+
def error(self, message):
|
|
11
|
+
self.print_usage(sys.stderr)
|
|
12
|
+
self.exit(USAGE, f"{self.prog}: error: {message}\n")
|
|
13
|
+
|
|
14
|
+
def _number(value, low, high=None):
|
|
15
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
16
|
+
return None
|
|
17
|
+
if value < low or (high is not None and value > high):
|
|
18
|
+
return None
|
|
19
|
+
return float(value)
|
|
20
|
+
|
|
21
|
+
def _count(value):
|
|
22
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
23
|
+
return None
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
def sha256_file(path):
|
|
27
|
+
digest = hashlib.sha256()
|
|
28
|
+
with open(path, "rb") as handle:
|
|
29
|
+
for chunk in iter(lambda: handle.read(65536), b""):
|
|
30
|
+
digest.update(chunk)
|
|
31
|
+
return digest.hexdigest()
|
|
32
|
+
|
|
33
|
+
def evidence_digest(subject, report_sha256, source_sha256, primary, control, percent):
|
|
34
|
+
# NUL-joined so a route name containing a separator cannot collide with another split.
|
|
35
|
+
parts = [DOMAIN, subject, report_sha256, source_sha256, primary, control, f"{percent:.6f}"]
|
|
36
|
+
return hashlib.sha256("\x00".join(parts).encode("utf-8")).hexdigest()
|
|
37
|
+
|
|
38
|
+
def assign(digest, percent):
|
|
39
|
+
return "canary" if int(digest, 16) % 10000 < round(percent * 100) else "control"
|
|
40
|
+
|
|
41
|
+
def load_report(path):
|
|
42
|
+
"""Return (report, report_sha256, refusal_reasons) from one exact byte read."""
|
|
43
|
+
with open(path, "rb") as handle:
|
|
44
|
+
report_bytes = handle.read()
|
|
45
|
+
report_sha256 = hashlib.sha256(report_bytes).hexdigest()
|
|
46
|
+
report = json.loads(report_bytes.decode("utf-8"))
|
|
47
|
+
reasons = []
|
|
48
|
+
if not isinstance(report, dict):
|
|
49
|
+
return None, report_sha256, ["report is not a JSON object"]
|
|
50
|
+
if report.get("report") != REPORT:
|
|
51
|
+
reasons.append(f"report version is not {REPORT}")
|
|
52
|
+
source = report.get("source")
|
|
53
|
+
if not isinstance(source, str) or not source.strip():
|
|
54
|
+
reasons.append("report has no source path")
|
|
55
|
+
source_sha256 = report.get("source_sha256")
|
|
56
|
+
if not isinstance(source_sha256, str) or len(source_sha256) != 64 or any(c not in "0123456789abcdef" for c in source_sha256):
|
|
57
|
+
reasons.append("report has no valid source_sha256")
|
|
58
|
+
if not isinstance(report.get("candidates"), list):
|
|
59
|
+
reasons.append("report has no candidates")
|
|
60
|
+
return report, report_sha256, reasons
|
|
61
|
+
|
|
62
|
+
def candidate(report, name):
|
|
63
|
+
for row in report.get("candidates") or []:
|
|
64
|
+
if isinstance(row, dict) and row.get("route") == name:
|
|
65
|
+
return row
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
def plan(report_path, subject, control_route, canary_percent=10.0, max_risk=.25, enable_canary=False):
|
|
69
|
+
"""Build a canary plan or an explained refusal. Reads only; writes nothing."""
|
|
70
|
+
reasons = []
|
|
71
|
+
if not isinstance(subject, str) or not subject.strip():
|
|
72
|
+
reasons.append("subject must be non-empty")
|
|
73
|
+
if not isinstance(control_route, str) or not control_route.strip():
|
|
74
|
+
reasons.append("control route must be non-empty")
|
|
75
|
+
if not enable_canary:
|
|
76
|
+
reasons.append("canary is opt-in; pass --enable-canary")
|
|
77
|
+
try:
|
|
78
|
+
report, report_sha256, schema_reasons = load_report(report_path)
|
|
79
|
+
except Exception as exc:
|
|
80
|
+
return {"plan": DOMAIN, "report": os.path.abspath(report_path), "report_sha256": None,
|
|
81
|
+
"assignment": None,
|
|
82
|
+
"refusal_reasons": reasons + [f"report is malformed: {exc}"]}
|
|
83
|
+
reasons += schema_reasons
|
|
84
|
+
if report is None:
|
|
85
|
+
return {"plan": DOMAIN, "report": os.path.abspath(report_path),
|
|
86
|
+
"report_sha256": report_sha256, "assignment": None,
|
|
87
|
+
"refusal_reasons": reasons}
|
|
88
|
+
|
|
89
|
+
source = report.get("source") if isinstance(report.get("source"), str) else None
|
|
90
|
+
source_sha256 = report.get("source_sha256")
|
|
91
|
+
source_missing = False
|
|
92
|
+
if source and not schema_reasons:
|
|
93
|
+
if not os.path.isfile(source):
|
|
94
|
+
source_missing = True
|
|
95
|
+
reasons.append(f"evidence source is absent: {source}")
|
|
96
|
+
elif sha256_file(source) != source_sha256:
|
|
97
|
+
reasons.append("evidence source has drifted from the report digest")
|
|
98
|
+
|
|
99
|
+
primary = report.get("selected_route")
|
|
100
|
+
if not isinstance(primary, str) or not primary.strip():
|
|
101
|
+
reasons.append("report selected no primary route")
|
|
102
|
+
primary = None
|
|
103
|
+
rows = {"primary": candidate(report, primary) if primary else None,
|
|
104
|
+
"control": candidate(report, control_route)}
|
|
105
|
+
for role, row in rows.items():
|
|
106
|
+
name = primary if role == "primary" else control_route
|
|
107
|
+
if row is None:
|
|
108
|
+
reasons.append(f"{role} route {name!r} is absent from the report")
|
|
109
|
+
elif row.get("eligible") is not True:
|
|
110
|
+
reasons.append(f"{role} route {name!r} is not eligible on measured evidence")
|
|
111
|
+
if primary is not None and primary == control_route:
|
|
112
|
+
reasons.append("primary and control must differ")
|
|
113
|
+
|
|
114
|
+
percent = _number(canary_percent, 0, 100)
|
|
115
|
+
if percent is None:
|
|
116
|
+
reasons.append("canary percent must be a finite number between 0 and 100")
|
|
117
|
+
ceiling = _number(max_risk, 0, 1)
|
|
118
|
+
if ceiling is None:
|
|
119
|
+
reasons.append("max risk must be a finite number between 0 and 1")
|
|
120
|
+
|
|
121
|
+
trials = {}
|
|
122
|
+
for role, row in rows.items():
|
|
123
|
+
if row is None:
|
|
124
|
+
continue
|
|
125
|
+
name = row.get("route")
|
|
126
|
+
risk = _number(row.get("mean_risk"), 0, 1)
|
|
127
|
+
count = _count(row.get("trials"))
|
|
128
|
+
if risk is None or count is None:
|
|
129
|
+
reasons.append(f"{role} route {name!r} has invalid measured values")
|
|
130
|
+
continue
|
|
131
|
+
trials[role] = count
|
|
132
|
+
if ceiling is not None and risk > ceiling:
|
|
133
|
+
reasons.append(f"{role} route {name!r} mean risk {risk:.6f} exceeds {ceiling:.6f}")
|
|
134
|
+
if len(trials) == 2 and trials["primary"] != trials["control"]:
|
|
135
|
+
reasons.append(f"evidence is unmatched: {trials['primary']} primary trials vs {trials['control']} control trials")
|
|
136
|
+
|
|
137
|
+
out = {"plan": DOMAIN, "report": os.path.abspath(report_path),
|
|
138
|
+
"report_sha256": report_sha256, "report_version": report.get("report"),
|
|
139
|
+
"source": source, "source_sha256": source_sha256, "source_missing": source_missing,
|
|
140
|
+
"primary_route": primary, "control_route": control_route,
|
|
141
|
+
"canary_percent": percent, "max_risk": ceiling, "subject": subject,
|
|
142
|
+
"assignment": None, "evidence_digest": None, "reversible": True,
|
|
143
|
+
"rollback": None, "refusal_reasons": reasons}
|
|
144
|
+
if reasons:
|
|
145
|
+
return out
|
|
146
|
+
digest = evidence_digest(subject, report_sha256, source_sha256, primary, control_route, percent)
|
|
147
|
+
out["evidence_digest"] = digest
|
|
148
|
+
out["assignment"] = assign(digest, percent)
|
|
149
|
+
out["route"] = primary if out["assignment"] == "canary" else control_route
|
|
150
|
+
out["rollback"] = {
|
|
151
|
+
"assignment": "control", "route": control_route, "reversible": True,
|
|
152
|
+
"effect": "every subject is assigned the control route",
|
|
153
|
+
"command": f"python3 tools/outcome-canary.py {report_path} --enable-canary "
|
|
154
|
+
f"--subject {subject} --control-route {control_route} --canary-percent 0"}
|
|
155
|
+
return out
|
|
156
|
+
|
|
157
|
+
def main(argv=None):
|
|
158
|
+
p = Parser(prog="outcome-canary")
|
|
159
|
+
p.add_argument("report")
|
|
160
|
+
p.add_argument("--enable-canary", action="store_true")
|
|
161
|
+
p.add_argument("--subject", required=True)
|
|
162
|
+
p.add_argument("--control-route", required=True)
|
|
163
|
+
p.add_argument("--canary-percent", type=float, default=10.0)
|
|
164
|
+
p.add_argument("--max-risk", type=float, default=.25)
|
|
165
|
+
p.add_argument("--json", action="store_true")
|
|
166
|
+
a = p.parse_args(argv)
|
|
167
|
+
if not a.subject.strip() or not a.control_route.strip():
|
|
168
|
+
p.error("subject and control route must be non-empty")
|
|
169
|
+
if _number(a.canary_percent, 0, 100) is None or _number(a.max_risk, 0, 1) is None:
|
|
170
|
+
p.error("invalid policy threshold")
|
|
171
|
+
if not os.path.isfile(a.report):
|
|
172
|
+
print(f"outcome-canary: no such report: {a.report}", file=sys.stderr)
|
|
173
|
+
return NO_INPUT
|
|
174
|
+
result = plan(a.report, a.subject, a.control_route, a.canary_percent, a.max_risk, a.enable_canary)
|
|
175
|
+
if result.get("source_missing"):
|
|
176
|
+
if a.json: print(json.dumps(result, sort_keys=True))
|
|
177
|
+
else: print(f"outcome-canary: evidence source is absent: {result['source']}", file=sys.stderr)
|
|
178
|
+
return NO_INPUT
|
|
179
|
+
if a.json:
|
|
180
|
+
print(json.dumps(result, sort_keys=True))
|
|
181
|
+
elif result["refusal_reasons"]:
|
|
182
|
+
print("Canary plan: REFUSED")
|
|
183
|
+
for reason in result["refusal_reasons"]:
|
|
184
|
+
print(f" {reason}")
|
|
185
|
+
else:
|
|
186
|
+
print(f"Canary plan: {result['assignment']} -> {result['route']}")
|
|
187
|
+
print(f" primary={result['primary_route']} control={result['control_route']} percent={result['canary_percent']:.6f}")
|
|
188
|
+
print(f" report sha256: {result['report_sha256']}")
|
|
189
|
+
print(f" evidence digest: {result['evidence_digest']}")
|
|
190
|
+
print(f" reversible: {result['reversible']}; rollback assigns {result['rollback']['assignment']}")
|
|
191
|
+
print(f" rollback: {result['rollback']['command']}")
|
|
192
|
+
return REFUSED if result["refusal_reasons"] else OK
|
|
193
|
+
|
|
194
|
+
if __name__ == "__main__": raise SystemExit(main())
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Recommend a route from measured accepted-outcome efficiency, or refuse."""
|
|
3
|
+
import argparse, hashlib, json, math, os, sys
|
|
4
|
+
|
|
5
|
+
OK, NO_ELIGIBLE, USAGE, NO_INPUT = 0, 3, 64, 66
|
|
6
|
+
|
|
7
|
+
class Parser(argparse.ArgumentParser):
|
|
8
|
+
def error(self, message):
|
|
9
|
+
self.print_usage(sys.stderr)
|
|
10
|
+
self.exit(USAGE, f"{self.prog}: error: {message}\n")
|
|
11
|
+
|
|
12
|
+
def _number(value, low, high=None, strict=False):
|
|
13
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
14
|
+
return None
|
|
15
|
+
if value < low or (strict and value == low) or (high is not None and value > high):
|
|
16
|
+
return None
|
|
17
|
+
return float(value)
|
|
18
|
+
|
|
19
|
+
def route(path, min_trials=3, max_risk=.25, risk_weight=1.0):
|
|
20
|
+
groups, invalid = {}, []
|
|
21
|
+
with open(path, encoding="utf-8") as handle:
|
|
22
|
+
for line_no, line in enumerate(handle, 1):
|
|
23
|
+
try:
|
|
24
|
+
row = json.loads(line)
|
|
25
|
+
name = row.get("route") if isinstance(row, dict) else None
|
|
26
|
+
cost = _number(row.get("cost_usd"), 0) if isinstance(row, dict) else None
|
|
27
|
+
mins = _number(row.get("duration_minutes"), 0, strict=True) if isinstance(row, dict) else None
|
|
28
|
+
risk = _number(row.get("risk"), 0, 1) if isinstance(row, dict) else None
|
|
29
|
+
accepted = row.get("accepted") if isinstance(row, dict) else None
|
|
30
|
+
verifier = row.get("verifier", "") if isinstance(row, dict) else ""
|
|
31
|
+
if not isinstance(name, str) or not name.strip() or type(accepted) is not bool or cost is None or mins is None or risk is None or not isinstance(verifier, str):
|
|
32
|
+
raise ValueError("invalid route observation schema")
|
|
33
|
+
groups.setdefault(name, []).append((accepted, cost, mins, risk))
|
|
34
|
+
except Exception as exc:
|
|
35
|
+
invalid.append({"line": line_no, "reason": str(exc)})
|
|
36
|
+
candidates = []
|
|
37
|
+
for name in sorted(groups):
|
|
38
|
+
rows = groups[name]; n = len(rows); accepted = sum(x[0] for x in rows)
|
|
39
|
+
cost = sum(x[1] for x in rows); mins = sum(x[2] for x in rows)
|
|
40
|
+
mean_risk = sum(x[3] for x in rows) / n
|
|
41
|
+
reasons = []
|
|
42
|
+
if invalid: reasons.append("input contains invalid observations")
|
|
43
|
+
if n < min_trials: reasons.append(f"needs {min_trials} trials; has {n}")
|
|
44
|
+
if accepted == 0: reasons.append("has no accepted outcome")
|
|
45
|
+
if mean_risk > max_risk: reasons.append(f"mean risk {mean_risk:.6f} exceeds {max_risk:.6f}")
|
|
46
|
+
per_dollar = accepted / max(cost, 1e-9)
|
|
47
|
+
per_minute = accepted / mins
|
|
48
|
+
harmonic = 2 * per_dollar * per_minute / (per_dollar + per_minute) if accepted else 0.0
|
|
49
|
+
score = harmonic * max(0.0, 1.0 - risk_weight * mean_risk)
|
|
50
|
+
candidates.append({"route": name, "trials": n, "accepted": accepted,
|
|
51
|
+
"cost_usd": cost, "duration_minutes": mins, "mean_risk": mean_risk,
|
|
52
|
+
"accepted_per_dollar": per_dollar, "accepted_per_minute": per_minute,
|
|
53
|
+
"score": score, "eligible": not reasons, "refusal_reasons": reasons})
|
|
54
|
+
eligible = sorted((c for c in candidates if c["eligible"]), key=lambda c: (-c["score"], c["route"]))
|
|
55
|
+
with open(path, "rb") as source_handle:
|
|
56
|
+
source_sha256 = hashlib.sha256(source_handle.read()).hexdigest()
|
|
57
|
+
return {"report": "loki-outcome-router/v1", "source": os.path.abspath(path),
|
|
58
|
+
"source_sha256": source_sha256,
|
|
59
|
+
"policy": {"min_trials": min_trials, "max_risk": max_risk,
|
|
60
|
+
"risk_weight": risk_weight},
|
|
61
|
+
"selected_route": eligible[0]["route"] if eligible else None,
|
|
62
|
+
"invalid_observations": invalid, "candidates": candidates,
|
|
63
|
+
"formula": "harmonic(accepted/USD, accepted/minute) * max(0, 1-risk_weight*mean_risk)"}
|
|
64
|
+
|
|
65
|
+
def main(argv=None):
|
|
66
|
+
p = Parser(); p.add_argument("path"); p.add_argument("--json", action="store_true")
|
|
67
|
+
p.add_argument("--min-trials", type=int, default=3); p.add_argument("--max-risk", type=float, default=.25)
|
|
68
|
+
p.add_argument("--risk-weight", type=float, default=1.0); a = p.parse_args(argv)
|
|
69
|
+
if a.min_trials < 1 or not math.isfinite(a.max_risk) or not 0 <= a.max_risk <= 1 or not math.isfinite(a.risk_weight) or a.risk_weight < 0: p.error("invalid threshold")
|
|
70
|
+
if not os.path.isfile(a.path): print(f"outcome-router: no such file: {a.path}", file=sys.stderr); return NO_INPUT
|
|
71
|
+
report = route(a.path, a.min_trials, a.max_risk, a.risk_weight)
|
|
72
|
+
if a.json: print(json.dumps(report, sort_keys=True))
|
|
73
|
+
else:
|
|
74
|
+
print("Outcome route: " + (report["selected_route"] or "REFUSED"))
|
|
75
|
+
for c in report["candidates"]:
|
|
76
|
+
print(f" {c['route']}: score={c['score']:.6f} trials={c['trials']} accepted={c['accepted']} " + ("eligible" if c['eligible'] else "; ".join(c['refusal_reasons'])))
|
|
77
|
+
if report["invalid_observations"]: print(f" invalid observations: {len(report['invalid_observations'])}")
|
|
78
|
+
return OK if report["selected_route"] else NO_ELIGIBLE
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__": raise SystemExit(main())
|