loki-mode 8.0.3 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -10
- package/SKILL.md +2 -2
- package/VERSION +1 -1
- package/autonomy/completion-council.sh +45 -4
- package/autonomy/council-v2.sh +114 -16
- package/autonomy/grill.sh +32 -0
- package/autonomy/lib/done-recognition.sh +16 -4
- package/autonomy/lib/fast_verify.py +346 -0
- package/autonomy/lib/no_mock_scan.py +21 -1
- package/autonomy/lib/prd-enrich.sh +12 -1
- package/autonomy/lib/proof-generator.py +173 -6
- package/autonomy/lib/proof-template.html +77 -12
- package/autonomy/lib/proof-verify.py +52 -0
- package/autonomy/loki +246 -7
- package/autonomy/run.sh +14 -0
- package/autonomy/verify.sh +12 -6
- package/dashboard/__init__.py +1 -1
- package/events/emit.sh +105 -18
- package/loki-ts/dist/loki.js +319 -316
- package/mcp/__init__.py +30 -12
- package/mcp/server.py +150 -0
- package/package.json +1 -1
- package/plugins/loki-mode/.claude-plugin/plugin.json +1 -1
- package/providers/claude.sh +72 -0
- package/providers/codex.sh +13 -0
- package/providers/loader.sh +10 -4
- package/providers/model_catalog.json +114 -17
- package/providers/models.sh +26 -3
- package/providers/opencode.sh +145 -0
- package/references/design-archetypes.md +85 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""fast_verify -- millisecond-scale deterministic verification.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS
|
|
5
|
+
Verification that takes minutes does not scale and cannot be embedded. To be
|
|
6
|
+
pluggable -- into an IDE, an MCP tool, a CI step, or another vendor's agent --
|
|
7
|
+
a verdict has to come back in the time a human would wait for a keystroke,
|
|
8
|
+
not a coffee break.
|
|
9
|
+
|
|
10
|
+
MEASURED BASELINE on this repo before this module:
|
|
11
|
+
tests/detect-mock-problems.sh 11,040 ms
|
|
12
|
+
tests/detect-test-mutations.sh 12,361 ms
|
|
13
|
+
|
|
14
|
+
The work itself is not slow. The ARCHITECTURE was:
|
|
15
|
+
1. FOUR separate full-tree `find` walks in one detector (one walk = 110 ms).
|
|
16
|
+
2. ~25 subprocess spawns at ~24 ms each of pure interpreter startup.
|
|
17
|
+
3. Every detector re-discovering the same file set independently.
|
|
18
|
+
|
|
19
|
+
`git ls-files` returns the same set in 37 ms, already deduplicated and
|
|
20
|
+
already gitignore-aware. So the fix is not "optimize the shell" -- it is to
|
|
21
|
+
stop paying discovery and process tax N times.
|
|
22
|
+
|
|
23
|
+
THE FIVE DESIGN RULES
|
|
24
|
+
A. SINGLE PASS - walk once, classify once, hand each detector its slice.
|
|
25
|
+
B. ZERO SUBPROCESS - detectors are pure functions in ONE process. Startup is
|
|
26
|
+
paid once, not 25 times.
|
|
27
|
+
C. CONTENT-ADDRESSED- cache each file's findings by content hash. An unchanged
|
|
28
|
+
file is never re-read. Warm runs approach cache-read cost.
|
|
29
|
+
D. DIFF-SCOPED - verifying a change needs the changed files, not the repo.
|
|
30
|
+
E. EXOGENOUS ONLY - no LLM on this path, ever. Only checks the agent cannot
|
|
31
|
+
author or influence: deterministic, reproducible, and
|
|
32
|
+
therefore trustworthy. That constraint is what makes the
|
|
33
|
+
fast path both fast AND the thing worth trusting.
|
|
34
|
+
|
|
35
|
+
WHAT THIS DELIBERATELY DOES NOT DO
|
|
36
|
+
It does not run the test suite, call a model, or render judgment. Those are
|
|
37
|
+
slower and, in the case of model judgment, not exogenous. This module answers
|
|
38
|
+
only the question a machine can answer deterministically, which is exactly the
|
|
39
|
+
question worth answering in milliseconds.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import hashlib
|
|
45
|
+
import json
|
|
46
|
+
import os
|
|
47
|
+
import re
|
|
48
|
+
import subprocess
|
|
49
|
+
import sys
|
|
50
|
+
import time
|
|
51
|
+
from dataclasses import dataclass, field, asdict
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
from typing import Iterable
|
|
54
|
+
|
|
55
|
+
SCHEMA_VERSION = 1
|
|
56
|
+
|
|
57
|
+
# Source extensions worth scanning. Anything else is discovery noise.
|
|
58
|
+
CODE_EXT = {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".rb", ".java", ".sh"}
|
|
59
|
+
TEST_RE = re.compile(r"(^|/)(test_[^/]+\.py|[^/]+\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs))$")
|
|
60
|
+
|
|
61
|
+
# Excluded even when git tracks them: vendored or generated trees produce findings
|
|
62
|
+
# nobody can act on, and scanning them is pure latency.
|
|
63
|
+
EXCLUDE_RE = re.compile(r"(^|/)(node_modules|dist|build|vendor|\.venv|coverage|__pycache__)(/|$)")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# --- Findings ---------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class Finding:
|
|
70
|
+
rule: str
|
|
71
|
+
path: str
|
|
72
|
+
line: int
|
|
73
|
+
message: str
|
|
74
|
+
severity: str = "high"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class Result:
|
|
79
|
+
verdict: str # PASS | FAIL | INCONCLUSIVE
|
|
80
|
+
findings: list = field(default_factory=list)
|
|
81
|
+
files_scanned: int = 0
|
|
82
|
+
files_from_cache: int = 0
|
|
83
|
+
elapsed_ms: float = 0.0
|
|
84
|
+
schema_version: int = SCHEMA_VERSION
|
|
85
|
+
exogenous: bool = True # never set False; this path admits no LLM
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# --- Rule A: mock data rendered as if it were real ---------------------------
|
|
89
|
+
# A UI that maps over a hardcoded array and renders it is the "Potemkin
|
|
90
|
+
# interface" failure -- it looks finished and is not wired to anything.
|
|
91
|
+
|
|
92
|
+
_INLINE_COLLECTION = re.compile(
|
|
93
|
+
r"""(?:const|let|var)\s+(\w+)\s*=\s*\[\s*\{""", re.M)
|
|
94
|
+
_RENDER_MAP = re.compile(r"""\{?\s*(\w+)\s*(?:\?\.)?\.map\s*\(""")
|
|
95
|
+
_FETCHY = re.compile(r"\b(fetch|axios|useQuery|useSWR|supabase|prisma|createClient)\b")
|
|
96
|
+
|
|
97
|
+
# Names that are legitimately static content, not stand-ins for a backend.
|
|
98
|
+
_STATIC_OK = re.compile(
|
|
99
|
+
r"^(features?|benefits?|faqs?|steps?|tabs?|nav|navigation|menu|links?|routes?|"
|
|
100
|
+
r"columns?|options?|plans?|pricing|tiers?|testimonials?|stats?|socials?|icons?)$",
|
|
101
|
+
re.I)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _rule_mock_render(path: str, text: str) -> list:
|
|
105
|
+
out = []
|
|
106
|
+
if not text or "[" not in text:
|
|
107
|
+
return out
|
|
108
|
+
declared = {}
|
|
109
|
+
for m in _INLINE_COLLECTION.finditer(text):
|
|
110
|
+
declared[m.group(1)] = text[:m.start()].count("\n") + 1
|
|
111
|
+
if not declared:
|
|
112
|
+
return out
|
|
113
|
+
# A real data source in the same file means the array is plausibly a fallback,
|
|
114
|
+
# not the product. Stay quiet rather than cry wolf: a false BLOCK on a correct
|
|
115
|
+
# build is worse than a missed warning, because it trains users to ignore us.
|
|
116
|
+
if _FETCHY.search(text):
|
|
117
|
+
return out
|
|
118
|
+
for m in _RENDER_MAP.finditer(text):
|
|
119
|
+
name = m.group(1)
|
|
120
|
+
if name in declared and not _STATIC_OK.match(name):
|
|
121
|
+
out.append(Finding(
|
|
122
|
+
rule="mock_render",
|
|
123
|
+
path=path,
|
|
124
|
+
line=declared[name],
|
|
125
|
+
message=(f"'{name}' is a hardcoded collection rendered directly; "
|
|
126
|
+
"no data source found in this file"),
|
|
127
|
+
))
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# --- Rule B: tests that cannot fail ------------------------------------------
|
|
132
|
+
# A test with no assertion is worse than no test: it turns a green suite into a
|
|
133
|
+
# false claim, which is precisely the lie this product exists to prevent.
|
|
134
|
+
|
|
135
|
+
_SKIPPED = re.compile(r"\b(it|test|describe)\.(skip|todo)\s*\(|@(unittest\.)?skip\b")
|
|
136
|
+
_HAS_ASSERT = re.compile(r"\b(expect|assert|should|chai|sinon\.assert)\b")
|
|
137
|
+
_TEST_BLOCK = re.compile(r"""\b(?:it|test)\s*\(\s*['"`]([^'"`]{1,120})['"`]""")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _rule_toothless_test(path: str, text: str) -> list:
|
|
141
|
+
out = []
|
|
142
|
+
if not text:
|
|
143
|
+
return out
|
|
144
|
+
for m in _SKIPPED.finditer(text):
|
|
145
|
+
out.append(Finding(
|
|
146
|
+
rule="skipped_test", path=path,
|
|
147
|
+
line=text[:m.start()].count("\n") + 1,
|
|
148
|
+
message="test is skipped; it cannot fail and cannot verify anything",
|
|
149
|
+
severity="medium",
|
|
150
|
+
))
|
|
151
|
+
blocks = list(_TEST_BLOCK.finditer(text))
|
|
152
|
+
for i, m in enumerate(blocks):
|
|
153
|
+
end = blocks[i + 1].start() if i + 1 < len(blocks) else len(text)
|
|
154
|
+
body = text[m.end():end]
|
|
155
|
+
if not _HAS_ASSERT.search(body):
|
|
156
|
+
out.append(Finding(
|
|
157
|
+
rule="assertionless_test", path=path,
|
|
158
|
+
line=text[:m.start()].count("\n") + 1,
|
|
159
|
+
message=f"test '{m.group(1)[:60]}' contains no assertion; it always passes",
|
|
160
|
+
))
|
|
161
|
+
return out
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
RULES = (_rule_mock_render, _rule_toothless_test)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# --- Discovery: one pass, from the git index ---------------------------------
|
|
168
|
+
|
|
169
|
+
def _git(args: list, cwd: str) -> str:
|
|
170
|
+
try:
|
|
171
|
+
p = subprocess.run(["git"] + args, cwd=cwd, capture_output=True,
|
|
172
|
+
text=True, timeout=20)
|
|
173
|
+
return p.stdout if p.returncode == 0 else ""
|
|
174
|
+
except (OSError, subprocess.SubprocessError):
|
|
175
|
+
return ""
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def discover(root: str, diff_base: str = "") -> list:
|
|
179
|
+
"""Return candidate files. ONE listing, no per-detector walk.
|
|
180
|
+
|
|
181
|
+
`git ls-files` beats `find` on every axis that matters here: it reads the
|
|
182
|
+
index instead of stat-ing the tree (37 ms vs 110 ms measured), it is already
|
|
183
|
+
gitignore-aware, and it never descends into node_modules. When a diff base is
|
|
184
|
+
given we narrow further -- verifying a change does not require reading the
|
|
185
|
+
repository.
|
|
186
|
+
"""
|
|
187
|
+
if diff_base:
|
|
188
|
+
raw = _git(["diff", "--name-only", diff_base + "...HEAD"], root)
|
|
189
|
+
if not raw.strip():
|
|
190
|
+
raw = _git(["diff", "--name-only", diff_base], root)
|
|
191
|
+
else:
|
|
192
|
+
raw = _git(["ls-files"], root)
|
|
193
|
+
|
|
194
|
+
if not raw:
|
|
195
|
+
# Not a git repo (or an empty one): fall back to a single os.walk. Still
|
|
196
|
+
# one pass -- the guarantee holds even off the happy path.
|
|
197
|
+
files = []
|
|
198
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
199
|
+
dirnames[:] = [d for d in dirnames
|
|
200
|
+
if not EXCLUDE_RE.search(d) and not d.startswith(".")]
|
|
201
|
+
for fn in filenames:
|
|
202
|
+
rel = os.path.relpath(os.path.join(dirpath, fn), root)
|
|
203
|
+
if Path(fn).suffix in CODE_EXT and not EXCLUDE_RE.search(rel):
|
|
204
|
+
files.append(rel)
|
|
205
|
+
return files
|
|
206
|
+
|
|
207
|
+
return [ln for ln in raw.splitlines()
|
|
208
|
+
if ln and Path(ln).suffix in CODE_EXT and not EXCLUDE_RE.search(ln)]
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
# --- Content-addressed cache -------------------------------------------------
|
|
212
|
+
|
|
213
|
+
class Cache:
|
|
214
|
+
"""Findings keyed by content hash, so unchanged files are never re-read.
|
|
215
|
+
|
|
216
|
+
Correctness note: the key is the file's CONTENT, not its path or mtime. A
|
|
217
|
+
file that moves keeps its result; a file that changes gets a new key. There
|
|
218
|
+
is no staleness window to reason about, which is what makes it safe to trust
|
|
219
|
+
a cache hit as if the scan had just run.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
def __init__(self, path: str):
|
|
223
|
+
self.path = path
|
|
224
|
+
self.data = {}
|
|
225
|
+
self.hits = 0
|
|
226
|
+
try:
|
|
227
|
+
with open(path, encoding="utf-8") as fh:
|
|
228
|
+
blob = json.load(fh)
|
|
229
|
+
if isinstance(blob, dict) and blob.get("schema") == SCHEMA_VERSION:
|
|
230
|
+
self.data = blob.get("entries", {})
|
|
231
|
+
except (OSError, ValueError):
|
|
232
|
+
self.data = {}
|
|
233
|
+
|
|
234
|
+
def get(self, key: str):
|
|
235
|
+
v = self.data.get(key)
|
|
236
|
+
if v is not None:
|
|
237
|
+
self.hits += 1
|
|
238
|
+
return v
|
|
239
|
+
|
|
240
|
+
def put(self, key: str, findings: list) -> None:
|
|
241
|
+
self.data[key] = findings
|
|
242
|
+
|
|
243
|
+
def save(self) -> None:
|
|
244
|
+
try:
|
|
245
|
+
os.makedirs(os.path.dirname(self.path), exist_ok=True)
|
|
246
|
+
tmp = self.path + ".tmp"
|
|
247
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
248
|
+
json.dump({"schema": SCHEMA_VERSION, "entries": self.data}, fh)
|
|
249
|
+
os.replace(tmp, self.path)
|
|
250
|
+
except OSError:
|
|
251
|
+
pass # a cache that cannot be written must never break a verdict
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
# --- The entry point ---------------------------------------------------------
|
|
255
|
+
|
|
256
|
+
def verify(root: str = ".", diff_base: str = "", use_cache: bool = True) -> Result:
|
|
257
|
+
t0 = time.perf_counter()
|
|
258
|
+
root = os.path.abspath(root)
|
|
259
|
+
cache = Cache(os.path.join(root, ".loki", "cache", "fast-verify.json")) if use_cache else None
|
|
260
|
+
|
|
261
|
+
findings = []
|
|
262
|
+
scanned = 0
|
|
263
|
+
|
|
264
|
+
for rel in discover(root, diff_base):
|
|
265
|
+
full = os.path.join(root, rel)
|
|
266
|
+
try:
|
|
267
|
+
with open(full, "rb") as fh:
|
|
268
|
+
raw = fh.read()
|
|
269
|
+
except OSError:
|
|
270
|
+
continue
|
|
271
|
+
scanned += 1
|
|
272
|
+
|
|
273
|
+
key = hashlib.blake2b(raw, digest_size=16).hexdigest()
|
|
274
|
+
if cache is not None:
|
|
275
|
+
hit = cache.get(key)
|
|
276
|
+
if hit is not None:
|
|
277
|
+
for d in hit:
|
|
278
|
+
findings.append(Finding(**{**d, "path": rel}))
|
|
279
|
+
continue
|
|
280
|
+
|
|
281
|
+
try:
|
|
282
|
+
text = raw.decode("utf-8", errors="replace")
|
|
283
|
+
except Exception:
|
|
284
|
+
continue
|
|
285
|
+
|
|
286
|
+
is_test = bool(TEST_RE.search(rel))
|
|
287
|
+
got = []
|
|
288
|
+
for rule in RULES:
|
|
289
|
+
# Rule B is about tests; rule A is about product code. Running each
|
|
290
|
+
# only where it applies is not just faster, it removes a whole class
|
|
291
|
+
# of false positive.
|
|
292
|
+
if rule is _rule_toothless_test and not is_test:
|
|
293
|
+
continue
|
|
294
|
+
if rule is _rule_mock_render and is_test:
|
|
295
|
+
continue
|
|
296
|
+
got.extend(rule(rel, text))
|
|
297
|
+
|
|
298
|
+
if cache is not None:
|
|
299
|
+
cache.put(key, [{k: v for k, v in asdict(f).items() if k != "path"} for f in got])
|
|
300
|
+
findings.extend(got)
|
|
301
|
+
|
|
302
|
+
if cache is not None:
|
|
303
|
+
cache.save()
|
|
304
|
+
|
|
305
|
+
blocking = [f for f in findings if f.severity == "high"]
|
|
306
|
+
verdict = "FAIL" if blocking else ("PASS" if scanned else "INCONCLUSIVE")
|
|
307
|
+
|
|
308
|
+
return Result(
|
|
309
|
+
verdict=verdict,
|
|
310
|
+
findings=[asdict(f) for f in findings],
|
|
311
|
+
files_scanned=scanned,
|
|
312
|
+
files_from_cache=(cache.hits if cache else 0),
|
|
313
|
+
elapsed_ms=round((time.perf_counter() - t0) * 1000, 2),
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def main(argv: list) -> int:
|
|
318
|
+
root, base, use_cache, as_json = ".", "", True, False
|
|
319
|
+
i = 0
|
|
320
|
+
while i < len(argv):
|
|
321
|
+
a = argv[i]
|
|
322
|
+
if a == "--diff-base" and i + 1 < len(argv):
|
|
323
|
+
base = argv[i + 1]; i += 1
|
|
324
|
+
elif a == "--no-cache":
|
|
325
|
+
use_cache = False
|
|
326
|
+
elif a == "--json":
|
|
327
|
+
as_json = True
|
|
328
|
+
elif not a.startswith("-"):
|
|
329
|
+
root = a
|
|
330
|
+
i += 1
|
|
331
|
+
|
|
332
|
+
r = verify(root, base, use_cache)
|
|
333
|
+
if as_json:
|
|
334
|
+
print(json.dumps(asdict(r), indent=2))
|
|
335
|
+
else:
|
|
336
|
+
print(f"{r.verdict} in {r.elapsed_ms}ms "
|
|
337
|
+
f"({r.files_scanned} files, {r.files_from_cache} cached)")
|
|
338
|
+
for f in r.findings[:20]:
|
|
339
|
+
print(f" [{f['severity']}] {f['rule']} {f['path']}:{f['line']} - {f['message']}")
|
|
340
|
+
if len(r.findings) > 20:
|
|
341
|
+
print(f" ... and {len(r.findings) - 20} more")
|
|
342
|
+
return 1 if r.verdict == "FAIL" else 0
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
if __name__ == "__main__":
|
|
346
|
+
sys.exit(main(sys.argv[1:]))
|
|
@@ -7,6 +7,7 @@ import hashlib
|
|
|
7
7
|
import json
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
|
+
import sys
|
|
10
11
|
from dataclasses import dataclass
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
from typing import Iterable
|
|
@@ -753,7 +754,26 @@ def scan(files: list[str], tree: Path) -> tuple[int, dict[str, object] | None]:
|
|
|
753
754
|
|
|
754
755
|
|
|
755
756
|
def main() -> int:
|
|
756
|
-
|
|
757
|
+
# Transport: stdin first, _NM_FILES as fallback. An env string is capped at
|
|
758
|
+
# MAX_ARG_STRLEN (131071 bytes) on Linux, so a large changed-file list makes
|
|
759
|
+
# execve fail with E2BIG -- the caller then captures its own
|
|
760
|
+
# "INCONCLUSIVE:detector_error" fallback and the gate silently passes
|
|
761
|
+
# through. macOS has no per-string cap, which is why that failure was
|
|
762
|
+
# Linux-only. stdin has no such limit.
|
|
763
|
+
# Read defensively: a caller may close stdin (sys.stdin is then None) or
|
|
764
|
+
# hand over a tty. Either way fall back to the env var rather than raising,
|
|
765
|
+
# because an uncaught error here reads to the caller as
|
|
766
|
+
# "INCONCLUSIVE:detector_error" -- the exact silent pass-through this
|
|
767
|
+
# transport change exists to remove.
|
|
768
|
+
raw = ""
|
|
769
|
+
try:
|
|
770
|
+
if sys.stdin is not None and not sys.stdin.isatty():
|
|
771
|
+
raw = sys.stdin.read()
|
|
772
|
+
except (OSError, ValueError):
|
|
773
|
+
raw = ""
|
|
774
|
+
if not raw.strip():
|
|
775
|
+
raw = os.environ.get("_NM_FILES", "")
|
|
776
|
+
files = [line.strip() for line in raw.splitlines() if line.strip()]
|
|
757
777
|
tree = Path(os.environ.get("_NM_TREE", ".")).resolve()
|
|
758
778
|
output = os.environ.get("_NM_OUT", "")
|
|
759
779
|
scanned, hit = scan(files, tree)
|
|
@@ -97,8 +97,14 @@ _loki_prd_enrich_invoke() {
|
|
|
97
97
|
# Decide whether enrichment should even be attempted. Returns 0 (attempt)
|
|
98
98
|
# only when the active provider is claude and not in degraded mode.
|
|
99
99
|
_loki_prd_enrich_provider_ok() {
|
|
100
|
-
[ "${LOKI_PROVIDER:-claude}" = "claude" ] || return 1
|
|
101
100
|
[ "${PROVIDER_DEGRADED:-false}" != "true" ] || return 1
|
|
101
|
+
# v8.2.0: capability, not identity. A provider exposing the timeout-able
|
|
102
|
+
# argv seam can run enrichment regardless of which CLI it is. Mirrors
|
|
103
|
+
# _loki_done_recog_provider_ok (autonomy/lib/done-recognition.sh).
|
|
104
|
+
if type provider_invoke_argv >/dev/null 2>&1; then
|
|
105
|
+
return 0
|
|
106
|
+
fi
|
|
107
|
+
[ "${LOKI_PROVIDER:-claude}" = "claude" ] || return 1
|
|
102
108
|
# v8: the raw-SDK enrich path (LOKI_SDK_PRD_ENRICH=1) needs no claude binary,
|
|
103
109
|
# so attempt is viable when that path is usable (bridge + bun). The invoke fn
|
|
104
110
|
# still fails closed to claude on an SDK miss.
|
|
@@ -224,6 +230,11 @@ loki_prd_enrich() {
|
|
|
224
230
|
# improve tasks deterministically (content-derived user_story) so even
|
|
225
231
|
# offline users get informative tasks, then return without a model call.
|
|
226
232
|
if ! _loki_prd_enrich_provider_ok; then
|
|
233
|
+
# HONESTY: say so. This used to degrade to the deterministic path with no
|
|
234
|
+
# log line at all, so a user on a non-Claude provider believed their PRD
|
|
235
|
+
# had been model-enriched when it had not. A capability the run did not
|
|
236
|
+
# get must never be indistinguishable from one it did.
|
|
237
|
+
log_warn "PRD enrichment: model-assisted pass SKIPPED (provider='${LOKI_PROVIDER:-claude}'), using the deterministic pass only."
|
|
227
238
|
_loki_prd_enrich_deterministic "$pending_path"
|
|
228
239
|
return 0
|
|
229
240
|
fi
|
|
@@ -243,6 +243,70 @@ def _norm_gate_status(raw):
|
|
|
243
243
|
return "inconclusive"
|
|
244
244
|
|
|
245
245
|
|
|
246
|
+
# TRUST-4: gate provenance. The discriminating property of a verification signal
|
|
247
|
+
# is NOT how many checks ran -- it is whether the checker sits OUTSIDE the agent's
|
|
248
|
+
# control. arXiv 2606.28438 shows AI-self-gates "look strong early but later lose
|
|
249
|
+
# their filtering effect", drifting into "a rubber-stamp regime where acceptance
|
|
250
|
+
# scores rise while benchmark correctness falls". arXiv 2607.05904 shows a judge
|
|
251
|
+
# conditioned on a candidate "scores plausibility, not correctness": self-play
|
|
252
|
+
# drove judge pass rate 0.72 -> 0.94 while TRUE accuracy stayed 0.20, and "a
|
|
253
|
+
# strict three-judge ensemble still accepts 55% of them". So a model-coupled gate
|
|
254
|
+
# is REPORTED but may never lift the headline.
|
|
255
|
+
#
|
|
256
|
+
# ADVISORY = the agent (or a model it prompts) authored the verdict. Everything
|
|
257
|
+
# else is EXOGENOUS: a deterministic script whose output the agent cannot write.
|
|
258
|
+
#
|
|
259
|
+
# Membership is keyed on the ADVISORY side only, and an UNKNOWN gate defaults to
|
|
260
|
+
# EXOGENOUS. That direction is load-bearing and must never be inverted: an
|
|
261
|
+
# unrecognized gate then still counts against VERIFIED (fail-closed). Defaulting
|
|
262
|
+
# unknown gates to advisory would let any newly-added or renamed gate silently
|
|
263
|
+
# lose its power to block -- the exact fake-green vector this split exists to
|
|
264
|
+
# close. Names are matched on a normalized key because run.sh emits BOTH
|
|
265
|
+
# spellings for the same gate (static-analysis / static_analysis, test-mutation /
|
|
266
|
+
# mutation_integrity), verified against run.sh's gate_failures writers.
|
|
267
|
+
_ADVISORY_GATES = frozenset((
|
|
268
|
+
# The agent writes both the test and the fix, so a green suite is a claim
|
|
269
|
+
# about its own work, not an independent measurement.
|
|
270
|
+
"test_coverage", "unit_tests", "test_suite", "semantic_tests", "tests",
|
|
271
|
+
# LLM-judgment gates: blind council, devil's advocate, magic-module debate.
|
|
272
|
+
"code_review", "devils_advocate", "devil_advocate", "magic_debate",
|
|
273
|
+
"council", "anti_sycophancy",
|
|
274
|
+
))
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _gate_key(name):
|
|
278
|
+
"""Normalize a gate name for provenance lookup.
|
|
279
|
+
|
|
280
|
+
run.sh emits the same gate under multiple spellings (`static-analysis` vs
|
|
281
|
+
`static_analysis`), and track_gate_failure appends `_PAUSED`/`_ESCALATED`
|
|
282
|
+
/`_not_run` suffixes. Fold all of them onto one key so classification cannot
|
|
283
|
+
be defeated by a cosmetic rename.
|
|
284
|
+
"""
|
|
285
|
+
s = str(name or "").strip().lower().replace("-", "_")
|
|
286
|
+
s = re.sub(r"_(paused|escalated|not_run|blocked)$", "", s)
|
|
287
|
+
return s
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _gate_provenance(name):
|
|
291
|
+
"""'advisory' for a model-authored gate, else 'exogenous' (fail-closed)."""
|
|
292
|
+
return "advisory" if _gate_key(name) in _ADVISORY_GATES else "exogenous"
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _is_exogenous(gate):
|
|
296
|
+
"""Provenance of a collected gate dict, honoring the stamped value.
|
|
297
|
+
|
|
298
|
+
Reads the `provenance` key stamped by _collect_quality_gates so the
|
|
299
|
+
`unresolved` override (a gate that HALTED the run counts as an execution
|
|
300
|
+
fact) is respected. Falls back to name lookup for a gate dict that never
|
|
301
|
+
passed through the collector. Fail-closed: anything not positively
|
|
302
|
+
identified as advisory counts as exogenous.
|
|
303
|
+
"""
|
|
304
|
+
stamped = gate.get("provenance")
|
|
305
|
+
if stamped:
|
|
306
|
+
return stamped == "exogenous"
|
|
307
|
+
return _gate_provenance(gate.get("name")) == "exogenous"
|
|
308
|
+
|
|
309
|
+
|
|
246
310
|
def _collect_quality_gates(loki_dir):
|
|
247
311
|
gates_raw = _read_json(
|
|
248
312
|
os.path.join(loki_dir, "state", "quality-gates.json"), default=None
|
|
@@ -354,14 +418,48 @@ def _collect_quality_gates(loki_dir):
|
|
|
354
418
|
for name in failed_names:
|
|
355
419
|
if name in by_name:
|
|
356
420
|
by_name[name]["status"] = "failed"
|
|
421
|
+
by_name[name]["unresolved"] = True
|
|
357
422
|
else:
|
|
358
|
-
gate = {"name": name, "status": "failed"}
|
|
423
|
+
gate = {"name": name, "status": "failed", "unresolved": True}
|
|
359
424
|
gates.append(gate)
|
|
360
425
|
by_name[name] = gate
|
|
361
426
|
|
|
427
|
+
# TRUST-4: stamp provenance on every gate at the single point the list is
|
|
428
|
+
# finalized, so every downstream reader (headline, template, verifier) sees
|
|
429
|
+
# the same classification and none can drift.
|
|
430
|
+
# An UNRESOLVED gate (listed in gate-failures.txt) is classified EXOGENOUS
|
|
431
|
+
# even when the gate itself is model-coupled. The fact being recorded is not
|
|
432
|
+
# "a judge disliked the code" -- it is "the run halted here and never
|
|
433
|
+
# cleared this blocker", which is an execution outcome the agent did not
|
|
434
|
+
# author. Without this, a run stopped dead by an unresolved code_review
|
|
435
|
+
# would emit a green receipt: the "cryptographically valid but semantically
|
|
436
|
+
# false green" the gate-failures merge above exists to prevent.
|
|
437
|
+
for gate in gates:
|
|
438
|
+
gate["provenance"] = (
|
|
439
|
+
"exogenous" if gate.get("unresolved")
|
|
440
|
+
else _gate_provenance(gate.get("name"))
|
|
441
|
+
)
|
|
442
|
+
|
|
362
443
|
total = len(gates)
|
|
363
444
|
passed = sum(1 for gate in gates if gate.get("status") == "passed")
|
|
364
|
-
|
|
445
|
+
exo = [g for g in gates if g.get("provenance") == "exogenous"]
|
|
446
|
+
adv = [g for g in gates if g.get("provenance") == "advisory"]
|
|
447
|
+
return {
|
|
448
|
+
"passed": passed,
|
|
449
|
+
"total": total,
|
|
450
|
+
"gates": gates,
|
|
451
|
+
# Pre-split counts so the renderer never has to re-derive provenance.
|
|
452
|
+
"exogenous": {
|
|
453
|
+
"passed": sum(1 for g in exo if g.get("status") == "passed"),
|
|
454
|
+
"total": len(exo),
|
|
455
|
+
"gates": exo,
|
|
456
|
+
},
|
|
457
|
+
"advisory": {
|
|
458
|
+
"passed": sum(1 for g in adv if g.get("status") == "passed"),
|
|
459
|
+
"total": len(adv),
|
|
460
|
+
"gates": adv,
|
|
461
|
+
},
|
|
462
|
+
}
|
|
365
463
|
|
|
366
464
|
|
|
367
465
|
def _collect_build(loki_dir):
|
|
@@ -429,6 +527,19 @@ def _collect_termination(loki_dir, session_exit_code=None):
|
|
|
429
527
|
"exit_code": None,
|
|
430
528
|
"outcome": "",
|
|
431
529
|
"run_status": "",
|
|
530
|
+
# Which gate stopped the run, if one did. The receipt previously named
|
|
531
|
+
# only a bare outcome ("intervention"), so a user reading the artifact
|
|
532
|
+
# could not tell WHICH gate blocked them or how close it came to its
|
|
533
|
+
# threshold -- the single most actionable fact about a blocked run. The
|
|
534
|
+
# engine already writes it to .loki/signals/GATE_ESCALATION.json and
|
|
535
|
+
# already surfaces it in COMPLETION.txt and PAUSED.md; the signed
|
|
536
|
+
# receipt was the one surface that stayed silent.
|
|
537
|
+
#
|
|
538
|
+
# These are deterministic FACTS read from a file the engine wrote, not
|
|
539
|
+
# an AI assessment, so they belong in the facts block.
|
|
540
|
+
"blocking_gate": "",
|
|
541
|
+
"blocking_gate_failures": None,
|
|
542
|
+
"blocking_gate_threshold": None,
|
|
432
543
|
}
|
|
433
544
|
state_paths = [os.path.join(loki_dir, "autonomy-state.json")]
|
|
434
545
|
sessions_dir = os.path.join(loki_dir, "sessions")
|
|
@@ -455,6 +566,20 @@ def _collect_termination(loki_dir, session_exit_code=None):
|
|
|
455
566
|
out["status"] = outcome
|
|
456
567
|
if outcome not in ("complete", "completed", "success"):
|
|
457
568
|
out["reason"] = outcome
|
|
569
|
+
# Gate escalation: name the gate that stopped the run. Best-effort and
|
|
570
|
+
# non-fatal -- a missing or corrupt signal simply leaves the fields empty,
|
|
571
|
+
# which reads as "no gate escalation recorded", never as a false claim.
|
|
572
|
+
gate = _read_json(
|
|
573
|
+
os.path.join(loki_dir, "signals", "GATE_ESCALATION.json"), default=None
|
|
574
|
+
)
|
|
575
|
+
if isinstance(gate, dict):
|
|
576
|
+
gate_name = str(gate.get("gate") or "").strip()
|
|
577
|
+
if gate_name:
|
|
578
|
+
out["blocking_gate"] = gate_name
|
|
579
|
+
count = gate.get("count")
|
|
580
|
+
thr = gate.get("threshold")
|
|
581
|
+
out["blocking_gate_failures"] = _to_int(count, None)
|
|
582
|
+
out["blocking_gate_threshold"] = _to_int(thr, None)
|
|
458
583
|
if session_exit_code is not None:
|
|
459
584
|
out["exit_code"] = session_exit_code
|
|
460
585
|
if session_exit_code != 0 and not out["reason"]:
|
|
@@ -1005,8 +1130,18 @@ def _build_proof(args, loki_dir, target_dir, repo_root):
|
|
|
1005
1130
|
"execution": termination,
|
|
1006
1131
|
"build": build,
|
|
1007
1132
|
"tests": tests,
|
|
1133
|
+
# TRUST-4: carry `provenance` into the facts projection. _compute_headline
|
|
1134
|
+
# and _compute_degraded read THIS list, so dropping the field silently
|
|
1135
|
+
# sent them back to name-only lookup -- which mis-classified an
|
|
1136
|
+
# UNRESOLVED code_review (a run-halting execution fact) as advisory and
|
|
1137
|
+
# green-washed a blocked run.
|
|
1008
1138
|
"quality_gates": [
|
|
1009
|
-
{
|
|
1139
|
+
{
|
|
1140
|
+
"name": g.get("name", ""),
|
|
1141
|
+
"status": g.get("status", "not_run"),
|
|
1142
|
+
"provenance": g.get("provenance")
|
|
1143
|
+
or _gate_provenance(g.get("name")),
|
|
1144
|
+
}
|
|
1010
1145
|
for g in (quality_gates.get("gates") or [])
|
|
1011
1146
|
],
|
|
1012
1147
|
"security": security,
|
|
@@ -1170,8 +1305,14 @@ def _compute_degraded(facts):
|
|
|
1170
1305
|
else ("exit_code=%s" % build.get("exit_code"))
|
|
1171
1306
|
out.append({"item": "build", "status": build.get("status"),
|
|
1172
1307
|
"reason": reason})
|
|
1308
|
+
# TRUST-4: only EXOGENOUS gates enter the degraded ledger, because degraded[]
|
|
1309
|
+
# is an INPUT to the headline (a non-empty ledger blocks VERIFIED). Letting an
|
|
1310
|
+
# advisory gate in here would give a model-authored verdict the power to
|
|
1311
|
+
# downgrade the headline through the back door, which is exactly what this
|
|
1312
|
+
# split forbids. Advisory outcomes are still reported in full -- they render
|
|
1313
|
+
# from quality_gates.advisory, which the template shows verbatim.
|
|
1173
1314
|
for g in facts.get("quality_gates") or []:
|
|
1174
|
-
if g.get("status") in weak:
|
|
1315
|
+
if g.get("status") in weak and _is_exogenous(g):
|
|
1175
1316
|
out.append({"item": "quality_gate:%s" % g.get("name", ""),
|
|
1176
1317
|
"status": g.get("status"),
|
|
1177
1318
|
"reason": "gate %s" % g.get("status")})
|
|
@@ -1262,12 +1403,21 @@ def _compute_headline(facts, degraded):
|
|
|
1262
1403
|
and execution_outcome not in ("complete", "completed", "success")
|
|
1263
1404
|
)
|
|
1264
1405
|
)
|
|
1406
|
+
# TRUST-4: only an EXOGENOUS gate failure forces NOT VERIFIED. An advisory
|
|
1407
|
+
# (model-authored) gate is reported but cannot move the verdict in EITHER
|
|
1408
|
+
# direction -- see _ADVISORY_GATES for the research basis. Note the
|
|
1409
|
+
# asymmetry is deliberate and one-way: advisory results are barred from
|
|
1410
|
+
# UPGRADING a verdict (below, in any_verified), and barred from downgrading
|
|
1411
|
+
# one here, because a judge that scores plausibility is not a measurement.
|
|
1412
|
+
# tests.status keeps its own hard-fail check: it is the recorded suite
|
|
1413
|
+
# outcome (an exit code), not the model's opinion of the suite.
|
|
1265
1414
|
any_failed = (
|
|
1266
1415
|
execution_failed
|
|
1267
1416
|
or tests.get("status") == "failed"
|
|
1268
1417
|
or build.get("status") == "failed"
|
|
1269
1418
|
or any(g.get("status") == "failed"
|
|
1270
|
-
for g in (facts.get("quality_gates") or [])
|
|
1419
|
+
for g in (facts.get("quality_gates") or [])
|
|
1420
|
+
if _is_exogenous(g))
|
|
1271
1421
|
or sec_high
|
|
1272
1422
|
or fn_failed
|
|
1273
1423
|
)
|
|
@@ -1288,11 +1438,15 @@ def _compute_headline(facts, degraded):
|
|
|
1288
1438
|
# produced code emit "VERIFIED WITH GAPS" - a fake-green at the receipt. Only
|
|
1289
1439
|
# a fact that actually ran and passed (tests/build verified, or a passed gate)
|
|
1290
1440
|
# may qualify; otherwise the honest headline is NOT VERIFIED.
|
|
1441
|
+
# TRUST-4: an advisory PASS is not positive evidence. A run whose ONLY green
|
|
1442
|
+
# signals are a council vote and a devil's-advocate nod has proven nothing
|
|
1443
|
+
# deterministically, so it must not reach "VERIFIED WITH GAPS" on that basis.
|
|
1291
1444
|
any_verified = (
|
|
1292
1445
|
tests.get("status") == "verified"
|
|
1293
1446
|
or build.get("status") == "verified"
|
|
1294
1447
|
or any(g.get("status") == "passed"
|
|
1295
|
-
for g in (facts.get("quality_gates") or [])
|
|
1448
|
+
for g in (facts.get("quality_gates") or [])
|
|
1449
|
+
if _is_exogenous(g))
|
|
1296
1450
|
)
|
|
1297
1451
|
if any_verified and degraded:
|
|
1298
1452
|
return "VERIFIED WITH GAPS"
|
|
@@ -1454,6 +1608,19 @@ def _render_fallback_html(proof):
|
|
|
1454
1608
|
ver = proof.get("verification", {})
|
|
1455
1609
|
rows.append('<p class="hash">Integrity hash (%s): %s</p>' % (
|
|
1456
1610
|
esc(ver.get("algo", "sha256")), esc(ver.get("hash", ""))))
|
|
1611
|
+
# Signing state, stated plainly. Mirrors renderProvenance in
|
|
1612
|
+
# proof-template.html (the primary renderer); this fallback path must not
|
|
1613
|
+
# be quieter about provenance than the page it stands in for.
|
|
1614
|
+
if ver.get("gpg_signature"):
|
|
1615
|
+
rows.append("<p>Signature: SIGNED (detached GPG over the canonical "
|
|
1616
|
+
"bytes). A verifier holding the signer public key can "
|
|
1617
|
+
"confirm provenance offline: loki proof verify <id></p>")
|
|
1618
|
+
else:
|
|
1619
|
+
rows.append("<p>Signature: UNSIGNED. The integrity hash proves the "
|
|
1620
|
+
"bytes were not edited after hashing; it does NOT prove "
|
|
1621
|
+
"who produced them, so this receipt trusts its generator. "
|
|
1622
|
+
"To sign future receipts, set LOKI_PROOF_GPG_KEY to a gpg "
|
|
1623
|
+
"key id (see docs/SIGNED-RECEIPTS.md).</p>")
|
|
1457
1624
|
red = proof.get("redaction", {})
|
|
1458
1625
|
rows.append("<p>Redaction applied: %s (%s redactions, rules v%s)</p>" % (
|
|
1459
1626
|
esc(red.get("applied")), esc(red.get("redactions_count")),
|