loki-mode 9.2.0 → 9.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/dashboard/__init__.py +1 -1
- package/loki-ts/dist/loki.js +2 -2
- package/mcp/__init__.py +1 -1
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/tools/cost-attribute.py +394 -0
- package/tools/gate-explain.py +290 -0
- package/tools/gate-log.py +422 -0
- package/tools/policy-diff.py +211 -0
- package/tools/receipt-export.py +348 -0
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""One gate verdict is a fact. A hundred of them is a pattern. Keep them.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. tools/ci-gate.py decides one run correctly and exits 0/1/2,
|
|
5
|
+
and tools/gate-report.py renders that one run for a human. Both are amnesiac by
|
|
6
|
+
design: they answer "is THIS merge safe" and then forget. But the questions an
|
|
7
|
+
engineer actually asks about a merge gate are historical ones --
|
|
8
|
+
|
|
9
|
+
which policy blocks us most often, and is it getting better or worse?
|
|
10
|
+
how long has the receipt axis been blind?
|
|
11
|
+
|
|
12
|
+
-- and nothing in this repo could answer them, because no verdict was ever
|
|
13
|
+
written down. So a durable append-only JSONL log, and a report over it.
|
|
14
|
+
|
|
15
|
+
THE RULE THIS FILE INHERITS, and the reason it is not just a counter:
|
|
16
|
+
|
|
17
|
+
A POLICY THAT COULD NOT BE EVALUATED HAS NOT PASSED.
|
|
18
|
+
|
|
19
|
+
An aggregator is where that rule dies, and it dies by arithmetic rather than by
|
|
20
|
+
argument. Two lines of code make it die:
|
|
21
|
+
|
|
22
|
+
passes = total - failures # folds UNEVALUABLE into pass
|
|
23
|
+
rate = passes / total # a "94% pass rate" built on blindness
|
|
24
|
+
|
|
25
|
+
Both read as reasonable. Both convert "we were blind on the receipt axis for
|
|
26
|
+
three weeks" into a healthy-looking green number, and the longer the blindness
|
|
27
|
+
lasts the healthier the number looks, because a blind axis never fails. So
|
|
28
|
+
UNEVALUABLE is its OWN category here, counted and printed beside PASS and FAIL,
|
|
29
|
+
and there is no derived pass-rate anywhere in this file. Three counts, stated.
|
|
30
|
+
|
|
31
|
+
A CORRUPT LINE IS DATA. The obvious loop skips lines that do not parse, and a
|
|
32
|
+
skipped line is a verdict that silently left the record. A log half-eaten by a
|
|
33
|
+
crashed writer would then report a clean history of whatever survived. Corrupt
|
|
34
|
+
lines are counted and reported as their own category, so "12 records, 4 of them
|
|
35
|
+
unreadable" can never render as "12 records".
|
|
36
|
+
|
|
37
|
+
AN EMPTY LOG IS NOT A CLEAN HISTORY. `report` on zero records exits non-zero
|
|
38
|
+
and says UNKNOWN. "No blocks recorded" and "never blocked" are opposite facts
|
|
39
|
+
about the world, and the first one is what an empty file means. Absent is not
|
|
40
|
+
zero -- this repo has paid for that inversion on more than a dozen surfaces.
|
|
41
|
+
|
|
42
|
+
MALFORMED STDIN IS AN ERROR, NEVER AN ASSUMED PASS. `record` reading garbage
|
|
43
|
+
writes nothing and exits 66. The one thing it must never do is invent a PASS
|
|
44
|
+
row, because the log is the evidence every later report is built from, and a
|
|
45
|
+
fabricated row is indistinguishable from a real one forever after.
|
|
46
|
+
|
|
47
|
+
But note the SPLIT, because over-strictness here is its own dishonesty: input
|
|
48
|
+
that is not JSON at all is an error and is refused, while a well-formed verdict
|
|
49
|
+
carrying a state word we do not recognise is RECORDED as UNEVALUABLE. The gate
|
|
50
|
+
did run and did report something; we simply cannot read its verdict, and that
|
|
51
|
+
is exactly what UNEVALUABLE means. Dropping it would delete the evidence that
|
|
52
|
+
our own vocabulary drifted.
|
|
53
|
+
|
|
54
|
+
WHY APPEND-ONLY, SINGLE WRITE. One `open(..., "a")` and one `write()` of one
|
|
55
|
+
line that already ends in a newline. No read-modify-write, so two CI jobs
|
|
56
|
+
finishing together cannot lose each other's verdict, and nothing this tool does
|
|
57
|
+
can edit or remove a verdict already recorded. A log a tool can rewrite is not
|
|
58
|
+
evidence.
|
|
59
|
+
|
|
60
|
+
WHAT IS DELIBERATELY NOT HERE. No cost figure. ci-gate's JSON carries no
|
|
61
|
+
measured cost, and recovering one by parsing the dollar amount out of
|
|
62
|
+
cost-guard's human-readable reason string would restate a predicate that
|
|
63
|
+
autonomy/lib/efficiency_cost.py owns -- the exact drift this repo fixed across
|
|
64
|
+
four surfaces. No cost surface is better than a re-derived one.
|
|
65
|
+
|
|
66
|
+
Usage:
|
|
67
|
+
tools/ci-gate.py <ws> --max-usd 5 --json | tools/gate-log.py record
|
|
68
|
+
tools/gate-log.py report [--json]
|
|
69
|
+
|
|
70
|
+
Exit: record 0 on write, 66 on unusable stdin. report 0 all-pass, 1 a FAIL was
|
|
71
|
+
recorded, 2 an UNEVALUABLE or corrupt record (blind outranks failed), 3 the log
|
|
72
|
+
exists but holds no records, 66 the log file does not exist, 64 usage error.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
import argparse
|
|
76
|
+
import datetime
|
|
77
|
+
import json
|
|
78
|
+
import os
|
|
79
|
+
import sys
|
|
80
|
+
|
|
81
|
+
PASSED, FAILED, COULD_NOT_CHECK, NOTHING, USAGE, MISSING = 0, 1, 2, 3, 64, 66
|
|
82
|
+
|
|
83
|
+
DEFAULT_LOG = os.path.join(".loki", "gate-log.jsonl")
|
|
84
|
+
|
|
85
|
+
# THE ONE MAPPING from a gate's verdict word to a bucket in the report. Three
|
|
86
|
+
# buckets, never two. Folding "UNEVALUABLE" in with "PASS" here is the entire
|
|
87
|
+
# defect this file exists to prevent, and it is a one-word edit, which is why
|
|
88
|
+
# tests/test_gate_log.py mutates precisely this line.
|
|
89
|
+
_CATEGORY = {"PASS": "pass", "FAIL": "fail", "UNEVALUABLE": "unevaluable"}
|
|
90
|
+
|
|
91
|
+
# Corrupt is not a verdict a gate can emit; it is what a damaged log line
|
|
92
|
+
# becomes. Kept out of _CATEGORY so no state word can ever map into it.
|
|
93
|
+
_CORRUPT = "corrupt"
|
|
94
|
+
|
|
95
|
+
_ORDER = ["pass", "fail", "unevaluable", _CORRUPT]
|
|
96
|
+
|
|
97
|
+
# Weakest link, same precedence ci-gate uses: blind outranks failed. An
|
|
98
|
+
# operator who sees 1, fixes the cost and re-runs is still blind on the dead
|
|
99
|
+
# axis, so the exit code must surface the blindness first.
|
|
100
|
+
_EXIT_FOR = {"pass": PASSED, "fail": FAILED,
|
|
101
|
+
"unevaluable": COULD_NOT_CHECK, _CORRUPT: COULD_NOT_CHECK}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class _Parser(argparse.ArgumentParser):
|
|
105
|
+
"""argparse exits 2 on a usage error. Here 2 means "could not check".
|
|
106
|
+
|
|
107
|
+
A mistyped flag would otherwise be indistinguishable from a gate reporting
|
|
108
|
+
that it was blind, and a CI job branching on the code would treat an
|
|
109
|
+
operator's typo as a real finding about the merge. 64 is the convention.
|
|
110
|
+
Subparsers inherit this class from the top-level parser, so `record
|
|
111
|
+
--bogus` lands on 64 too.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def error(self, message):
|
|
115
|
+
self.print_usage(sys.stderr)
|
|
116
|
+
sys.stderr.write("gate-log: %s\n" % message)
|
|
117
|
+
raise SystemExit(USAGE)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def classify(verdict):
|
|
121
|
+
"""Bucket ONE gate verdict. Anything unrecognised is unevaluable.
|
|
122
|
+
|
|
123
|
+
Never returns "pass" by default. A default of pass is how an aggregator
|
|
124
|
+
launders every shape it did not anticipate into green, and the shapes it
|
|
125
|
+
did not anticipate are precisely the broken ones.
|
|
126
|
+
"""
|
|
127
|
+
if not isinstance(verdict, dict):
|
|
128
|
+
return _CATEGORY["UNEVALUABLE"]
|
|
129
|
+
state = verdict.get("state")
|
|
130
|
+
if isinstance(state, str) and state.upper() in _CATEGORY:
|
|
131
|
+
return _CATEGORY[state.upper()]
|
|
132
|
+
return _CATEGORY["UNEVALUABLE"]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def failing_policies(verdict):
|
|
136
|
+
"""Policy names this verdict recorded as FAIL. Only FAIL, never blind ones.
|
|
137
|
+
|
|
138
|
+
An unevaluable policy is not a failing policy: naming it in the
|
|
139
|
+
most-failing tally would send an engineer to fix a rule that never fired,
|
|
140
|
+
while the real problem is that its instrumentation is dead. The unevaluable
|
|
141
|
+
tally is reported separately for that reason.
|
|
142
|
+
"""
|
|
143
|
+
rows = verdict.get("policies") if isinstance(verdict, dict) else None
|
|
144
|
+
out = []
|
|
145
|
+
if isinstance(rows, list):
|
|
146
|
+
for row in rows:
|
|
147
|
+
if not isinstance(row, dict):
|
|
148
|
+
continue
|
|
149
|
+
state = row.get("state")
|
|
150
|
+
if isinstance(state, str) and state.upper() == "FAIL":
|
|
151
|
+
out.append(str(row.get("policy") or "?"))
|
|
152
|
+
return out
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _entry(verdict):
|
|
156
|
+
"""The line that gets appended. Timestamp ours, verdict theirs, verbatim.
|
|
157
|
+
|
|
158
|
+
The raw verdict is embedded whole rather than summarised, so a later reader
|
|
159
|
+
who needs a field this version never thought about can still recover it.
|
|
160
|
+
Summarising at write time is a one-way loss.
|
|
161
|
+
"""
|
|
162
|
+
return {
|
|
163
|
+
"recorded_at": datetime.datetime.now(
|
|
164
|
+
datetime.timezone.utc).replace(microsecond=0).isoformat(),
|
|
165
|
+
"category": classify(verdict),
|
|
166
|
+
"failing_policies": failing_policies(verdict),
|
|
167
|
+
"verdict": verdict,
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def append_record(path, verdict):
|
|
172
|
+
"""One open, one write, one line. No read-modify-write, ever."""
|
|
173
|
+
parent = os.path.dirname(os.path.abspath(path))
|
|
174
|
+
if parent and not os.path.isdir(parent):
|
|
175
|
+
os.makedirs(parent, exist_ok=True)
|
|
176
|
+
line = json.dumps(_entry(verdict), sort_keys=True) + "\n"
|
|
177
|
+
with open(path, "a", encoding="utf-8") as fh:
|
|
178
|
+
fh.write(line)
|
|
179
|
+
return line
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _io_error(action, path, exc):
|
|
183
|
+
"""An unreachable log is "could not check", never "checked and failed".
|
|
184
|
+
|
|
185
|
+
Without this an OSError escapes as a traceback and Python exits 1, which in
|
|
186
|
+
this convention claims the gate was evaluated and FAILED. A permission
|
|
187
|
+
error is not a finding about a merge.
|
|
188
|
+
"""
|
|
189
|
+
sys.stderr.write("gate-log: could not %s %s: %s\n" % (action, path, exc))
|
|
190
|
+
return COULD_NOT_CHECK
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def read_log(path):
|
|
194
|
+
"""Every line, corrupt ones INCLUDED as their own category.
|
|
195
|
+
|
|
196
|
+
Returns (entries, corrupt_count). A corrupt line that is merely skipped is
|
|
197
|
+
a verdict deleted from the record by the reader, and the resulting report
|
|
198
|
+
describes a history that never happened.
|
|
199
|
+
"""
|
|
200
|
+
entries, corrupt = [], 0
|
|
201
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
202
|
+
for raw in fh:
|
|
203
|
+
if not raw.strip():
|
|
204
|
+
continue # a trailing newline is not a damaged record
|
|
205
|
+
try:
|
|
206
|
+
entry = json.loads(raw)
|
|
207
|
+
except ValueError:
|
|
208
|
+
corrupt += 1
|
|
209
|
+
continue
|
|
210
|
+
if not isinstance(entry, dict):
|
|
211
|
+
corrupt += 1
|
|
212
|
+
continue
|
|
213
|
+
entries.append(entry)
|
|
214
|
+
return entries, corrupt
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _category_of(entry):
|
|
218
|
+
"""An entry's bucket, re-derived from its verdict if it lacks one.
|
|
219
|
+
|
|
220
|
+
Trusting a stored "category" blindly would let a hand-edited log assert
|
|
221
|
+
anything; falling back to the embedded verdict keeps the raw evidence
|
|
222
|
+
authoritative. An entry with neither is unevaluable, not a pass.
|
|
223
|
+
"""
|
|
224
|
+
stored = entry.get("category")
|
|
225
|
+
# Deliberately _CATEGORY.values() and not _ORDER: "corrupt" is in _ORDER
|
|
226
|
+
# but is not a verdict any gate can emit. Honouring a stored "corrupt"
|
|
227
|
+
# would add to the count of lines this reader actually failed to parse,
|
|
228
|
+
# and the buckets would then no longer sum to the record total.
|
|
229
|
+
if isinstance(stored, str) and stored in _CATEGORY.values():
|
|
230
|
+
return stored
|
|
231
|
+
return classify(entry.get("verdict"))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def summarize(entries, corrupt):
|
|
235
|
+
"""Counts, the most frequently failing policy, and the trend.
|
|
236
|
+
|
|
237
|
+
Every bucket is pre-seeded to 0, so a MEASURED zero survives as 0 and is
|
|
238
|
+
reported as 0. That is not cosmetic: "fail: 0" over 40 records is a real
|
|
239
|
+
finding, and it must not be confused with the UNKNOWN this returns when
|
|
240
|
+
there are no records at all.
|
|
241
|
+
"""
|
|
242
|
+
counts = dict((name, 0) for name in _ORDER)
|
|
243
|
+
counts[_CORRUPT] = corrupt
|
|
244
|
+
policy_hits = {}
|
|
245
|
+
sequence = []
|
|
246
|
+
for entry in entries:
|
|
247
|
+
category = _category_of(entry)
|
|
248
|
+
counts[category] = counts.get(category, 0) + 1
|
|
249
|
+
sequence.append(category)
|
|
250
|
+
for name in failing_policies(entry.get("verdict")):
|
|
251
|
+
policy_hits[name] = policy_hits.get(name, 0) + 1
|
|
252
|
+
|
|
253
|
+
total = len(entries) + corrupt
|
|
254
|
+
return {
|
|
255
|
+
"records": total,
|
|
256
|
+
"readable": len(entries),
|
|
257
|
+
"counts": counts,
|
|
258
|
+
"top_failing_policy": _top_policy(policy_hits),
|
|
259
|
+
"policy_failures": policy_hits,
|
|
260
|
+
"trend": _trend(sequence),
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _top_policy(policy_hits):
|
|
265
|
+
"""The most frequently failing policy, or a MEASURED "none".
|
|
266
|
+
|
|
267
|
+
None means unknown; the string "none" with zero failures is a measurement
|
|
268
|
+
-- we read the records and no policy failed. Collapsing those two would
|
|
269
|
+
make an empty log and a clean history read identically.
|
|
270
|
+
"""
|
|
271
|
+
if not policy_hits:
|
|
272
|
+
return None
|
|
273
|
+
best = max(sorted(policy_hits), key=lambda name: policy_hits[name])
|
|
274
|
+
return {"policy": best, "failures": policy_hits[best]}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _trend(sequence):
|
|
278
|
+
"""Healthier, worse, or steady over the two halves. UNKNOWN under 2.
|
|
279
|
+
|
|
280
|
+
A single record has no trend. Reporting "steady" from one data point is an
|
|
281
|
+
invented measurement, so this returns None and the report prints UNKNOWN.
|
|
282
|
+
"Not blocked" counts pass only: an unevaluable half is not an improving
|
|
283
|
+
half, which is the same folding error the counts refuse to make.
|
|
284
|
+
"""
|
|
285
|
+
if len(sequence) < 2:
|
|
286
|
+
return None
|
|
287
|
+
half = len(sequence) // 2
|
|
288
|
+
older, newer = sequence[:half], sequence[half:]
|
|
289
|
+
before = sum(1 for c in older if c == "pass") / float(len(older))
|
|
290
|
+
after = sum(1 for c in newer if c == "pass") / float(len(newer))
|
|
291
|
+
if after > before:
|
|
292
|
+
direction = "improving"
|
|
293
|
+
elif after < before:
|
|
294
|
+
direction = "worsening"
|
|
295
|
+
else:
|
|
296
|
+
direction = "steady"
|
|
297
|
+
return {"direction": direction, "older_pass_rate": round(before, 3),
|
|
298
|
+
"newer_pass_rate": round(after, 3),
|
|
299
|
+
"window": [len(older), len(newer)]}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def render(summary):
|
|
303
|
+
counts = summary["counts"]
|
|
304
|
+
lines = ["gate-log: %d record(s)" % summary["records"], ""]
|
|
305
|
+
for name in _ORDER:
|
|
306
|
+
label = name if name != "unevaluable" \
|
|
307
|
+
else "unevaluable (NOT a pass)"
|
|
308
|
+
lines.append(" %-26s %d" % (label, counts.get(name, 0)))
|
|
309
|
+
lines.append("")
|
|
310
|
+
top = summary["top_failing_policy"]
|
|
311
|
+
if summary["readable"] == 0:
|
|
312
|
+
# Every line was unreadable. "no policy failed" would be a measurement
|
|
313
|
+
# we never took: we read nothing, so we know nothing about failures.
|
|
314
|
+
lines.append("most failing policy: UNKNOWN -- no readable record")
|
|
315
|
+
elif top is None:
|
|
316
|
+
lines.append("most failing policy: none -- no FAIL in %d readable "
|
|
317
|
+
"record(s)" % summary["readable"])
|
|
318
|
+
else:
|
|
319
|
+
lines.append("most failing policy: %s (%d failure(s))"
|
|
320
|
+
% (top["policy"], top["failures"]))
|
|
321
|
+
trend = summary["trend"]
|
|
322
|
+
if trend is None:
|
|
323
|
+
lines.append("trend: UNKNOWN -- fewer than 2 readable records (%d)"
|
|
324
|
+
% summary["readable"])
|
|
325
|
+
else:
|
|
326
|
+
lines.append("trend: %s (pass rate %.0f%% -> %.0f%% over %d then %d "
|
|
327
|
+
"record(s))"
|
|
328
|
+
% (trend["direction"], trend["older_pass_rate"] * 100,
|
|
329
|
+
trend["newer_pass_rate"] * 100,
|
|
330
|
+
trend["window"][0], trend["window"][1]))
|
|
331
|
+
return "\n".join(lines)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def exit_code(summary):
|
|
335
|
+
"""Weakest link over the whole history. Blind and corrupt outrank failed."""
|
|
336
|
+
worst = PASSED
|
|
337
|
+
counts = summary["counts"]
|
|
338
|
+
for name in _ORDER:
|
|
339
|
+
if counts.get(name, 0):
|
|
340
|
+
worst = max(worst, _EXIT_FOR[name])
|
|
341
|
+
return worst
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _cmd_record(args):
|
|
345
|
+
raw = sys.stdin.read()
|
|
346
|
+
if not raw.strip():
|
|
347
|
+
sys.stderr.write(
|
|
348
|
+
"gate-log: empty stdin -- expected a ci-gate --json verdict. "
|
|
349
|
+
"Nothing to record is not a pass, so nothing was written.\n")
|
|
350
|
+
return MISSING
|
|
351
|
+
try:
|
|
352
|
+
verdict = json.loads(raw)
|
|
353
|
+
except ValueError as exc:
|
|
354
|
+
sys.stderr.write(
|
|
355
|
+
"gate-log: could not parse the gate verdict (%s). Refusing to "
|
|
356
|
+
"record: an invented row is indistinguishable from a real one.\n"
|
|
357
|
+
% exc)
|
|
358
|
+
return MISSING
|
|
359
|
+
if not isinstance(verdict, dict):
|
|
360
|
+
sys.stderr.write(
|
|
361
|
+
"gate-log: expected a JSON object from ci-gate, got %s. Nothing "
|
|
362
|
+
"was written.\n" % type(verdict).__name__)
|
|
363
|
+
return MISSING
|
|
364
|
+
try:
|
|
365
|
+
line = append_record(args.file, verdict)
|
|
366
|
+
except OSError as exc:
|
|
367
|
+
return _io_error("write", args.file, exc)
|
|
368
|
+
entry = json.loads(line)
|
|
369
|
+
sys.stderr.write("gate-log: recorded %s to %s\n"
|
|
370
|
+
% (entry["category"], args.file))
|
|
371
|
+
return PASSED
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _cmd_report(args):
|
|
375
|
+
if not os.path.exists(args.file):
|
|
376
|
+
sys.stderr.write(
|
|
377
|
+
"gate-log: no log at %s -- UNKNOWN, not a clean history. A gate "
|
|
378
|
+
"that was never recorded is not a gate that never blocked.\n"
|
|
379
|
+
% args.file)
|
|
380
|
+
return MISSING
|
|
381
|
+
try:
|
|
382
|
+
entries, corrupt = read_log(args.file)
|
|
383
|
+
except OSError as exc:
|
|
384
|
+
return _io_error("read", args.file, exc)
|
|
385
|
+
if not entries and not corrupt:
|
|
386
|
+
sys.stderr.write(
|
|
387
|
+
"gate-log: %s holds no records -- UNKNOWN. An empty log must "
|
|
388
|
+
"never read as 'never blocked'.\n" % args.file)
|
|
389
|
+
return NOTHING
|
|
390
|
+
summary = summarize(entries, corrupt)
|
|
391
|
+
if args.as_json:
|
|
392
|
+
print(json.dumps(summary, indent=2, sort_keys=True))
|
|
393
|
+
else:
|
|
394
|
+
print(render(summary))
|
|
395
|
+
return exit_code(summary)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def main(argv=None):
|
|
399
|
+
ap = _Parser(description="Append ci-gate verdicts to a durable log and "
|
|
400
|
+
"report the pattern.")
|
|
401
|
+
subs = ap.add_subparsers(dest="command")
|
|
402
|
+
for name, help_text in (("record", "append a ci-gate --json verdict read "
|
|
403
|
+
"from stdin"),
|
|
404
|
+
("report", "summarise the recorded verdicts")):
|
|
405
|
+
sub = subs.add_parser(name, help=help_text)
|
|
406
|
+
sub.add_argument("--file", default=DEFAULT_LOG,
|
|
407
|
+
help="JSONL log path (default: %s)" % DEFAULT_LOG)
|
|
408
|
+
if name == "report":
|
|
409
|
+
sub.add_argument("--json", action="store_true", dest="as_json",
|
|
410
|
+
help="emit the summary as JSON")
|
|
411
|
+
args = ap.parse_args(argv)
|
|
412
|
+
if args.command == "record":
|
|
413
|
+
return _cmd_record(args)
|
|
414
|
+
if args.command == "report":
|
|
415
|
+
return _cmd_report(args)
|
|
416
|
+
# No subcommand. Not a verdict about anything, so it is a usage error --
|
|
417
|
+
# exiting 0 here would let `gate-log.py` alone read as a clean gate.
|
|
418
|
+
ap.error("a subcommand is required: record or report")
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
if __name__ == "__main__":
|
|
422
|
+
sys.exit(main())
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Diff two policy files by SAFETY DIRECTION, because a loosening looks like any other line.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. tools/policy-load.py put the merge policy in a version-
|
|
5
|
+
controlled file so lowering a ceiling would look like a code change. It does.
|
|
6
|
+
So does RAISING one. In a 400-line PR diff, this:
|
|
7
|
+
|
|
8
|
+
- "max_usd": 5,
|
|
9
|
+
+ "max_usd": 50,
|
|
10
|
+
|
|
11
|
+
is one line of JSON, the same visual weight as a renamed variable, and it has
|
|
12
|
+
just given every merge a 10x cost ceiling. `git diff` is direction-blind by
|
|
13
|
+
construction: it reports that a value changed, and leaves the safety reasoning
|
|
14
|
+
to a reviewer who is 300 lines from the end and has no schema in their head.
|
|
15
|
+
|
|
16
|
+
Doing that reasoning FOR the reviewer is the entire product. Not "max_usd:
|
|
17
|
+
5 -> 50" -- that is what git already said. "WEAKENS".
|
|
18
|
+
|
|
19
|
+
Three rules follow, each one a way a loosening gets waved through:
|
|
20
|
+
|
|
21
|
+
A CHANGE WE CANNOT CLASSIFY READS "UNKNOWN DIRECTION", NEVER "unchanged" and
|
|
22
|
+
never assumed safe. policy-load.KNOWN_KEYS will grow. A key added there but
|
|
23
|
+
not here is a fully valid policy edit that diffs cleanly -- exactly the shape
|
|
24
|
+
a silent loosening hides in. Falling through to "changed" makes the new axis
|
|
25
|
+
invisible on the one surface built to see it. So the fallthrough is loud, and
|
|
26
|
+
a test asserts DIRECTIONS covers every KNOWN_KEYS entry -- UNKNOWN is the
|
|
27
|
+
runtime safety net for whatever still slips past, not the accepted resting
|
|
28
|
+
state for a key we ship.
|
|
29
|
+
|
|
30
|
+
REMOVING A KEY IS ALWAYS WEAKENING. Not "removed". Deleting `max_usd` does
|
|
31
|
+
not lower a ceiling, it stops enforcing cost AT ALL, which is strictly weaker
|
|
32
|
+
than any number you could have written. This holds for `require_receipt:
|
|
33
|
+
false -> absent` too, where the tempting reading is "it enforced nothing
|
|
34
|
+
either way, so it is neutral". A per-case exception here is the hole: the
|
|
35
|
+
reviewer needs one rule they can trust, not a rule with a footnote about
|
|
36
|
+
which removals are the safe kind.
|
|
37
|
+
|
|
38
|
+
BOTH FILES MUST BE VALID POLICIES FIRST. A diff of a file policy-load would
|
|
39
|
+
reject is a confident verdict about a document no gate would ever honour --
|
|
40
|
+
the vacuously-green shape one level over. Refuse, name the file, name the
|
|
41
|
+
reason.
|
|
42
|
+
|
|
43
|
+
Exit 0 means the diff was produced, INCLUDING when it found weakenings; that is
|
|
44
|
+
a successful answer to the question asked. --fail-on-weaken is what turns a
|
|
45
|
+
finding into a build failure, so CI can require a human ack on a loosening.
|
|
46
|
+
|
|
47
|
+
Usage:
|
|
48
|
+
tools/policy-diff.py <old.json> <new.json> [--json] [--fail-on-weaken]
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
import argparse
|
|
52
|
+
import importlib.util
|
|
53
|
+
import json
|
|
54
|
+
import os
|
|
55
|
+
import sys
|
|
56
|
+
|
|
57
|
+
sys.dont_write_bytecode = True
|
|
58
|
+
|
|
59
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
60
|
+
|
|
61
|
+
# policy-load.py is hyphenated, so it is not importable by name. Load it by
|
|
62
|
+
# path rather than re-implementing validation: a second copy of the schema is a
|
|
63
|
+
# second thing to forget to update, and this tool's whole claim is that it
|
|
64
|
+
# refuses exactly what the loader refuses.
|
|
65
|
+
_spec = importlib.util.spec_from_file_location(
|
|
66
|
+
"policy_load", os.path.join(_HERE, "policy-load.py"))
|
|
67
|
+
policy_load = importlib.util.module_from_spec(_spec)
|
|
68
|
+
_spec.loader.exec_module(policy_load)
|
|
69
|
+
|
|
70
|
+
WEAKENS = "WEAKENS"
|
|
71
|
+
TIGHTENS = "TIGHTENS"
|
|
72
|
+
UNKNOWN = "UNKNOWN DIRECTION"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Missing:
|
|
76
|
+
"""A key absent from a policy. Not None: `null` is a value a file can hold."""
|
|
77
|
+
|
|
78
|
+
def __repr__(self):
|
|
79
|
+
return "<absent>"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
MISSING = _Missing()
|
|
83
|
+
|
|
84
|
+
# Exit codes, per the convention tests/test_tool_exit_contract.py enforces.
|
|
85
|
+
EXIT_OK = 0
|
|
86
|
+
EXIT_FAILED = 1
|
|
87
|
+
EXIT_CANNOT_EVALUATE = 2
|
|
88
|
+
EXIT_USAGE = 64
|
|
89
|
+
EXIT_INPUT_MISSING = 66
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _direction_max_usd(old, new):
|
|
93
|
+
"""A higher ceiling admits runs the old policy would have blocked."""
|
|
94
|
+
return WEAKENS if new > old else TIGHTENS
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _direction_require_receipt(old, new):
|
|
98
|
+
# Only false -> true tightens; true -> false drops the requirement.
|
|
99
|
+
return TIGHTENS if new else WEAKENS
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# Keyed by policy key. The DEFAULT is UNKNOWN, never "changed": see the module
|
|
103
|
+
# docstring. Adding a key to policy-load.KNOWN_KEYS without adding it here is
|
|
104
|
+
# caught by a test, but until someone fixes it the output must still be loud.
|
|
105
|
+
DIRECTIONS = {
|
|
106
|
+
"max_usd": _direction_max_usd,
|
|
107
|
+
"require_receipt": _direction_require_receipt,
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def classify(key, old, new):
|
|
112
|
+
"""Return (direction, detail) for one key. `old`/`new` may be MISSING."""
|
|
113
|
+
if old == new:
|
|
114
|
+
return None, None
|
|
115
|
+
if new is MISSING:
|
|
116
|
+
# Never "removed". The gate stops enforcing this axis entirely, which
|
|
117
|
+
# is weaker than any value that could have been there.
|
|
118
|
+
return WEAKENS, "{} removed (was {}) -- this axis is no longer enforced at all".format(
|
|
119
|
+
key, json.dumps(old))
|
|
120
|
+
if old is MISSING:
|
|
121
|
+
return TIGHTENS, "{} added = {} -- a new axis is now enforced".format(
|
|
122
|
+
key, json.dumps(new))
|
|
123
|
+
decide = DIRECTIONS.get(key)
|
|
124
|
+
if decide is None:
|
|
125
|
+
return UNKNOWN, "{}: {} -> {} -- this tool does not know which direction is safer".format(
|
|
126
|
+
key, json.dumps(old), json.dumps(new))
|
|
127
|
+
return decide(old, new), "{}: {} -> {}".format(
|
|
128
|
+
key, json.dumps(old), json.dumps(new))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def diff(old_policy, new_policy):
|
|
132
|
+
"""Every classified change between two validated policies."""
|
|
133
|
+
changes = []
|
|
134
|
+
for key in sorted(set(old_policy) | set(new_policy)):
|
|
135
|
+
direction, detail = classify(
|
|
136
|
+
key, old_policy.get(key, MISSING), new_policy.get(key, MISSING))
|
|
137
|
+
if direction is not None:
|
|
138
|
+
changes.append({"key": key, "direction": direction, "detail": detail})
|
|
139
|
+
return changes
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _load_or_die(path, label):
|
|
143
|
+
"""Validated policy, or an exit code. Missing file and invalid file differ."""
|
|
144
|
+
if not os.path.exists(path):
|
|
145
|
+
print("policy-diff: {} file does not exist: {}".format(label, path),
|
|
146
|
+
file=sys.stderr)
|
|
147
|
+
return None, EXIT_INPUT_MISSING
|
|
148
|
+
try:
|
|
149
|
+
return policy_load.load(path), None
|
|
150
|
+
except policy_load.PolicyError as exc:
|
|
151
|
+
# Refusing to diff is "could not be checked", not "the check failed".
|
|
152
|
+
print("policy-diff: refusing to diff -- the {} file is not a valid "
|
|
153
|
+
"policy: {}".format(label, exc), file=sys.stderr)
|
|
154
|
+
return None, EXIT_CANNOT_EVALUATE
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class _Parser(argparse.ArgumentParser):
|
|
158
|
+
# argparse exits 2 on a usage error, and 2 means "could not be checked" in
|
|
159
|
+
# this repo. Overriding error() (not parse_args) leaves --help exiting 0.
|
|
160
|
+
def error(self, message):
|
|
161
|
+
self.print_usage(sys.stderr)
|
|
162
|
+
print("{}: error: {}".format(self.prog, message), file=sys.stderr)
|
|
163
|
+
raise SystemExit(EXIT_USAGE)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def main(argv=None):
|
|
167
|
+
ap = _Parser(description="Classify every policy change as WEAKENS or TIGHTENS.")
|
|
168
|
+
ap.add_argument("old", help="the policy file as it is today")
|
|
169
|
+
ap.add_argument("new", help="the policy file as proposed")
|
|
170
|
+
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
171
|
+
help="emit the classified changes as JSON")
|
|
172
|
+
ap.add_argument("--fail-on-weaken", action="store_true", dest="fail_on_weaken",
|
|
173
|
+
help="exit non-zero if any change weakens the gate")
|
|
174
|
+
args = ap.parse_args(argv)
|
|
175
|
+
|
|
176
|
+
old, rc = _load_or_die(args.old, "old")
|
|
177
|
+
if rc is not None:
|
|
178
|
+
return rc
|
|
179
|
+
new, rc = _load_or_die(args.new, "new")
|
|
180
|
+
if rc is not None:
|
|
181
|
+
return rc
|
|
182
|
+
|
|
183
|
+
changes = diff(old, new)
|
|
184
|
+
weakenings = [c for c in changes if c["direction"] == WEAKENS]
|
|
185
|
+
unknowns = [c for c in changes if c["direction"] == UNKNOWN]
|
|
186
|
+
|
|
187
|
+
if args.as_json:
|
|
188
|
+
print(json.dumps({
|
|
189
|
+
"changes": changes,
|
|
190
|
+
"weakens": len(weakenings),
|
|
191
|
+
"unknown_direction": len(unknowns),
|
|
192
|
+
}, indent=2, sort_keys=True))
|
|
193
|
+
elif not changes:
|
|
194
|
+
print("no change: the two policies enforce the same gate")
|
|
195
|
+
else:
|
|
196
|
+
for c in changes:
|
|
197
|
+
print("{}: {}".format(c["direction"], c["detail"]))
|
|
198
|
+
# UNKNOWN is called out beside the weakenings, not buried in the list:
|
|
199
|
+
# it is the count a reviewer must resolve by hand.
|
|
200
|
+
print("{} weakening(s), {} unknown direction, {} change(s) total".format(
|
|
201
|
+
len(weakenings), len(unknowns), len(changes)))
|
|
202
|
+
|
|
203
|
+
if weakenings and args.fail_on_weaken:
|
|
204
|
+
print("policy-diff: {} weakening(s) require an explicit human ack".format(
|
|
205
|
+
len(weakenings)), file=sys.stderr)
|
|
206
|
+
return EXIT_FAILED
|
|
207
|
+
return EXIT_OK
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
sys.exit(main())
|