devlyn-cli 2.9.3 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -3
- package/CLAUDE.md +5 -5
- package/README.md +5 -5
- package/benchmark/auto-resolve/fixtures/SCHEMA.md +1 -1
- package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +1 -0
- package/benchmark/auto-resolve/scripts/run-fixture.sh +11 -1
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +83 -1
- package/bin/devlyn.js +108 -15
- package/config/skills/_shared/adapters/README.md +2 -2
- package/config/skills/_shared/adapters/grok.md +99 -0
- package/config/skills/_shared/archive_run.py +35 -1
- package/config/skills/_shared/collect-codex-findings.py +231 -50
- package/config/skills/_shared/engine-doctor.sh +6 -3
- package/config/skills/_shared/engine-preflight.md +12 -5
- package/config/skills/_shared/expected.schema.json +7 -0
- package/config/skills/_shared/grok-anchor-guard.py +61 -0
- package/config/skills/_shared/phase-prompt-render.py +181 -0
- package/config/skills/_shared/resolve-bootstrap.py +630 -0
- package/config/skills/_shared/resolve-stop-hook.py +325 -0
- package/config/skills/_shared/spec-verify-check.py +308 -31
- package/config/skills/_shared/state-phase-write.py +2966 -27
- package/config/skills/_shared/terminal-claim-check.py +537 -0
- package/config/skills/_shared/verify-merge-findings.py +533 -16
- package/config/skills/devlyn:engines/SKILL.md +2 -2
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:queue/SKILL.md +12 -0
- package/config/skills/devlyn:resolve/SKILL.md +60 -78
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +2 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/plan.md +1 -5
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +4 -2
- package/config/skills/devlyn:resolve/references/phases/surface-close.md +34 -0
- package/config/skills/devlyn:resolve/references/phases/verify.md +70 -97
- package/config/skills/devlyn:resolve/references/state-schema.md +26 -19
- package/optional-skills/devlyn:reap/SKILL.md +1 -1
- package/optional-skills/devlyn:reap/scripts/reap.sh +25 -13
- package/optional-skills/devlyn:reap/scripts/scan.sh +15 -1
- package/package.json +1 -1
- package/scripts/lint-fixtures.sh +4 -1
- package/scripts/lint-skills.sh +269 -77
|
@@ -27,7 +27,19 @@ PER_RUN_PATTERNS = (
|
|
|
27
27
|
"*.findings.jsonl",
|
|
28
28
|
"*.log.md",
|
|
29
29
|
"fix-batch.round-*.json",
|
|
30
|
+
"closure-durability.round-*.json",
|
|
31
|
+
"resolve-stop-hook.*.json",
|
|
30
32
|
"criteria.generated.md",
|
|
33
|
+
"goal.raw.txt",
|
|
34
|
+
"surface-close.input.patch",
|
|
35
|
+
"surface-close.prompt",
|
|
36
|
+
"surface-close.output.json",
|
|
37
|
+
"surface-close.stdout",
|
|
38
|
+
# Mutation workers retain the exact session JSONL identified by their
|
|
39
|
+
# dispatch receipt. Round-scoped root files avoid engine-global scans.
|
|
40
|
+
"implement.worker-session.*.jsonl",
|
|
41
|
+
"surface-close.worker-session.*.jsonl",
|
|
42
|
+
"cleanup.worker-session.*.jsonl",
|
|
31
43
|
"risk-probes.jsonl",
|
|
32
44
|
# Probe scripts referenced by risk-probes.jsonl are archived by
|
|
33
45
|
# move_probe_scripts() to preserve probes/<file> layout.
|
|
@@ -159,6 +171,15 @@ def self_test() -> int:
|
|
|
159
171
|
)
|
|
160
172
|
for name in (
|
|
161
173
|
"risk-probes.jsonl",
|
|
174
|
+
"goal.raw.txt",
|
|
175
|
+
"surface-close.input.patch",
|
|
176
|
+
"surface-close.prompt",
|
|
177
|
+
"surface-close.output.json",
|
|
178
|
+
"surface-close.stdout",
|
|
179
|
+
"implement.worker-session.0.jsonl",
|
|
180
|
+
"surface-close.worker-session.0.jsonl",
|
|
181
|
+
"cleanup.worker-session.1.jsonl",
|
|
182
|
+
"resolve-stop-hook.run-1.123.456.json",
|
|
162
183
|
"verify.pair.findings.jsonl",
|
|
163
184
|
"verify-merge.summary.json",
|
|
164
185
|
"codex-judge.stdout",
|
|
@@ -169,13 +190,24 @@ def self_test() -> int:
|
|
|
169
190
|
(devlyn / name).write_text("{}\n", encoding="utf-8")
|
|
170
191
|
(devlyn / "probes").mkdir()
|
|
171
192
|
(devlyn / "probes" / "P1.py").write_text("print('probe')\n", encoding="utf-8")
|
|
193
|
+
global_rollout = devlyn / "engine-sessions" / "rollout-global.jsonl"
|
|
194
|
+
global_rollout.parent.mkdir()
|
|
195
|
+
global_rollout.write_text("{}\n", encoding="utf-8")
|
|
172
196
|
run_id = read_run_id(devlyn)
|
|
173
197
|
assert run_id == "run-1", run_id
|
|
174
198
|
moved = move_artifacts(devlyn, devlyn / "runs" / run_id)
|
|
175
|
-
assert moved >=
|
|
199
|
+
assert moved >= 12, moved
|
|
176
200
|
for name in (
|
|
177
201
|
"pipeline.state.json",
|
|
178
202
|
"risk-probes.jsonl",
|
|
203
|
+
"goal.raw.txt",
|
|
204
|
+
"surface-close.input.patch",
|
|
205
|
+
"surface-close.prompt",
|
|
206
|
+
"surface-close.output.json",
|
|
207
|
+
"surface-close.stdout",
|
|
208
|
+
"implement.worker-session.0.jsonl",
|
|
209
|
+
"surface-close.worker-session.0.jsonl",
|
|
210
|
+
"cleanup.worker-session.1.jsonl",
|
|
179
211
|
"probes/P1.py",
|
|
180
212
|
"verify.pair.findings.jsonl",
|
|
181
213
|
"verify-merge.summary.json",
|
|
@@ -185,6 +217,8 @@ def self_test() -> int:
|
|
|
185
217
|
"claude-judge.stderr",
|
|
186
218
|
):
|
|
187
219
|
assert (devlyn / "runs" / run_id / name).is_file(), name
|
|
220
|
+
assert global_rollout.is_file(), "engine-global session files must stay untouched"
|
|
221
|
+
assert not (devlyn / "runs" / run_id / global_rollout.name).exists()
|
|
188
222
|
|
|
189
223
|
bad = pathlib.Path(tmp) / "bad"
|
|
190
224
|
bad.mkdir()
|
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""Normalize raw
|
|
2
|
+
"""Normalize raw pair-JUDGE stdout into canonical VERIFY JSONL."""
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import argparse
|
|
7
7
|
import json
|
|
8
8
|
import pathlib
|
|
9
|
+
import runpy
|
|
9
10
|
import sys
|
|
10
11
|
import tempfile
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
FINDING_SEVERITIES = {"CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"}
|
|
16
|
+
ENVELOPE_KEYS = {"text", "stopReason", "sessionId", "requestId"}
|
|
17
|
+
IGNORABLE_FENCES = {"```", "```json", "```jsonl"}
|
|
18
|
+
MERGE_CONTRACT = runpy.run_path(pathlib.Path(__file__).with_name("verify-merge-findings.py"))
|
|
19
|
+
VERDICT_RANK = MERGE_CONTRACT["VERDICT_RANK"]
|
|
20
|
+
finding_rank = MERGE_CONTRACT["finding_rank"]
|
|
21
|
+
NARRATIVE_PREAMBLE_BYTES = frozenset(
|
|
22
|
+
b"\t\n\r !\"$%&'()*+,-./0123456789:;<=>?@"
|
|
23
|
+
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz|}~"
|
|
24
|
+
) | frozenset(range(0x80, 0x100))
|
|
15
25
|
|
|
16
26
|
|
|
17
27
|
def reject_json_constant(token: str) -> None:
|
|
@@ -32,71 +42,242 @@ def atomic_write(path: pathlib.Path, text: str) -> None:
|
|
|
32
42
|
pathlib.Path(tmp_name).replace(path)
|
|
33
43
|
|
|
34
44
|
|
|
35
|
-
def
|
|
45
|
+
def collect_text(
|
|
46
|
+
text: str, source: pathlib.Path
|
|
47
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
36
48
|
findings: list[dict[str, Any]] = []
|
|
37
49
|
summary: dict[str, Any] | None = None
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if
|
|
44
|
-
|
|
45
|
-
item = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
|
|
46
|
-
except ValueError as exc:
|
|
47
|
-
raise SystemExit(f"error: invalid SUMMARY JSON at {stdout_path}:{line_no}: {exc}")
|
|
48
|
-
if not isinstance(item, dict):
|
|
49
|
-
raise SystemExit(f"error: SUMMARY is not an object at {stdout_path}:{line_no}")
|
|
50
|
-
summary = item
|
|
51
|
-
continue
|
|
52
|
-
if raw.startswith("#"):
|
|
53
|
-
continue
|
|
50
|
+
for line_no, line in enumerate(text.splitlines(), 1):
|
|
51
|
+
raw = line.strip()
|
|
52
|
+
if not raw or raw in IGNORABLE_FENCES:
|
|
53
|
+
continue
|
|
54
|
+
if raw.startswith("# SUMMARY "):
|
|
55
|
+
if summary is not None:
|
|
56
|
+
raise SystemExit(f"error: record after terminal SUMMARY at {source}:{line_no}")
|
|
54
57
|
try:
|
|
55
|
-
item = loads_strict_json(raw)
|
|
58
|
+
item = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
|
|
56
59
|
except ValueError as exc:
|
|
57
|
-
raise SystemExit(f"error: invalid
|
|
60
|
+
raise SystemExit(f"error: invalid SUMMARY JSON at {source}:{line_no}: {exc}")
|
|
58
61
|
if not isinstance(item, dict):
|
|
59
|
-
raise SystemExit(f"error:
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
62
|
+
raise SystemExit(f"error: SUMMARY is not an object at {source}:{line_no}")
|
|
63
|
+
if item.get("verdict") not in VERDICT_RANK:
|
|
64
|
+
raise SystemExit(f"error: SUMMARY has unknown verdict at {source}:{line_no}")
|
|
65
|
+
summary = item
|
|
66
|
+
continue
|
|
67
|
+
if raw.startswith("#"):
|
|
68
|
+
continue
|
|
69
|
+
if summary is not None:
|
|
70
|
+
raise SystemExit(f"error: record after terminal SUMMARY at {source}:{line_no}")
|
|
71
|
+
try:
|
|
72
|
+
item = loads_strict_json(raw)
|
|
73
|
+
except ValueError as exc:
|
|
74
|
+
raise SystemExit(f"error: invalid JSONL at {source}:{line_no}: {exc}")
|
|
75
|
+
if not isinstance(item, dict):
|
|
76
|
+
raise SystemExit(f"error: JSONL item is not an object at {source}:{line_no}")
|
|
77
|
+
if set(item) == {"verdict"}:
|
|
78
|
+
if item["verdict"] not in VERDICT_RANK:
|
|
79
|
+
raise SystemExit(f"error: SUMMARY has unknown verdict at {source}:{line_no}")
|
|
80
|
+
summary = item
|
|
81
|
+
continue
|
|
82
|
+
severity = str(item.get("severity") or "").upper()
|
|
83
|
+
if severity not in FINDING_SEVERITIES:
|
|
84
|
+
raise SystemExit(f"error: finding missing valid severity at {source}:{line_no}")
|
|
85
|
+
findings.append(item)
|
|
86
|
+
if summary is not None and summary["verdict"] == "PASS" and any(
|
|
87
|
+
finding_rank(finding) == 2 for finding in findings
|
|
88
|
+
):
|
|
89
|
+
raise SystemExit("error: verdict-binding finding cannot have a PASS SUMMARY")
|
|
90
|
+
if not findings and (summary is None or summary.get("verdict") != "PASS"):
|
|
67
91
|
raise SystemExit("error: non-PASS SUMMARY without JSONL findings")
|
|
68
92
|
return findings, summary
|
|
69
93
|
|
|
70
94
|
|
|
95
|
+
def recover_envelope_text(
|
|
96
|
+
text: str, source: pathlib.Path
|
|
97
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
98
|
+
encoded = text.encode("utf-8")
|
|
99
|
+
for cut, byte in enumerate(encoded[:-1], 1):
|
|
100
|
+
if byte not in NARRATIVE_PREAMBLE_BYTES:
|
|
101
|
+
break
|
|
102
|
+
try:
|
|
103
|
+
candidate = encoded[cut:].decode("utf-8")
|
|
104
|
+
except UnicodeDecodeError:
|
|
105
|
+
continue
|
|
106
|
+
try:
|
|
107
|
+
findings, summary = collect_text(candidate, source)
|
|
108
|
+
except SystemExit:
|
|
109
|
+
continue
|
|
110
|
+
if (
|
|
111
|
+
summary is not None
|
|
112
|
+
and summary["verdict"] == "NEEDS_WORK"
|
|
113
|
+
and any(finding_rank(finding) == 2 for finding in findings)
|
|
114
|
+
):
|
|
115
|
+
return findings, summary
|
|
116
|
+
raise SystemExit("error: no admissible envelope recovery")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def collect_stdout(stdout_path: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
|
|
120
|
+
stdout_text = stdout_path.read_text(encoding="utf-8")
|
|
121
|
+
try:
|
|
122
|
+
candidate = loads_strict_json(stdout_text)
|
|
123
|
+
except ValueError:
|
|
124
|
+
candidate = None
|
|
125
|
+
if (
|
|
126
|
+
not isinstance(candidate, dict)
|
|
127
|
+
or not ENVELOPE_KEYS.issubset(candidate)
|
|
128
|
+
or "severity" in candidate
|
|
129
|
+
):
|
|
130
|
+
return collect_text(stdout_text, stdout_path)
|
|
131
|
+
|
|
132
|
+
stop_reason = candidate["stopReason"]
|
|
133
|
+
if stop_reason != "EndTurn":
|
|
134
|
+
raise SystemExit(
|
|
135
|
+
f"error: envelope stopReason must be 'EndTurn' at {stdout_path}; "
|
|
136
|
+
f"got {stop_reason!r}"
|
|
137
|
+
)
|
|
138
|
+
envelope_text = candidate["text"]
|
|
139
|
+
if not isinstance(envelope_text, str):
|
|
140
|
+
raise SystemExit(f"error: envelope text must be a string at {stdout_path}")
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
return collect_text(envelope_text, stdout_path)
|
|
144
|
+
except SystemExit as exc:
|
|
145
|
+
try:
|
|
146
|
+
return recover_envelope_text(envelope_text, stdout_path)
|
|
147
|
+
except SystemExit:
|
|
148
|
+
raise SystemExit(f"error: envelope text rejected: {exc}") from None
|
|
149
|
+
|
|
150
|
+
|
|
71
151
|
def self_test() -> int:
|
|
72
152
|
with tempfile.TemporaryDirectory() as tmp:
|
|
73
153
|
root = pathlib.Path(tmp)
|
|
74
|
-
stdout_path = root / "
|
|
154
|
+
stdout_path = root / "pair-judge.stdout"
|
|
75
155
|
out_path = root / "verify.pair.findings.jsonl"
|
|
76
|
-
summary_path = root / "
|
|
156
|
+
summary_path = root / "pair-judge.summary.json"
|
|
157
|
+
|
|
158
|
+
def assert_rejected(text: str, label: str, *message_parts: str) -> None:
|
|
159
|
+
stdout_path.write_text(text, encoding="utf-8")
|
|
160
|
+
try:
|
|
161
|
+
collect_stdout(stdout_path)
|
|
162
|
+
except SystemExit as exc:
|
|
163
|
+
message = str(exc)
|
|
164
|
+
assert all(part in message for part in message_parts)
|
|
165
|
+
else:
|
|
166
|
+
raise AssertionError(f"{label} must be rejected")
|
|
167
|
+
|
|
168
|
+
plain_finding = {"id": "a", "severity": "HIGH"}
|
|
169
|
+
plain_summary = {"verdict": "NEEDS_WORK"}
|
|
77
170
|
stdout_path.write_text(
|
|
78
|
-
json.dumps(
|
|
79
|
-
+
|
|
171
|
+
json.dumps(plain_finding) + "\n"
|
|
172
|
+
+ "# SUMMARY " + json.dumps(plain_summary) + "\n",
|
|
80
173
|
encoding="utf-8",
|
|
81
174
|
)
|
|
82
|
-
findings, summary =
|
|
175
|
+
findings, summary = collect_stdout(stdout_path)
|
|
176
|
+
assert findings == [plain_finding]
|
|
177
|
+
assert summary == plain_summary
|
|
83
178
|
write_outputs(findings, summary, out_path, summary_path)
|
|
84
179
|
assert out_path.read_text(encoding="utf-8").count("\n") == 1
|
|
85
180
|
assert loads_strict_json(summary_path.read_text(encoding="utf-8"))["verdict"] == "NEEDS_WORK"
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
181
|
+
assert_rejected(
|
|
182
|
+
'{"id":"nan","severity":NaN}\n',
|
|
183
|
+
"NaN pair-JUDGE stdout finding",
|
|
184
|
+
"invalid JSON numeric constant: NaN",
|
|
185
|
+
)
|
|
186
|
+
rejection_cases = (
|
|
187
|
+
("", "no SUMMARY line", "non-PASS SUMMARY without JSONL findings"),
|
|
188
|
+
('# SUMMARY {}\n', "empty SUMMARY object", "SUMMARY has unknown verdict"),
|
|
189
|
+
('# SUMMARY {"verdict":"pass"}\n', "lowercase pass", "SUMMARY has unknown verdict"),
|
|
190
|
+
('# SUMMARY {"verdict":"UNKNOWN"}\n', "unknown verdict", "SUMMARY has unknown verdict"),
|
|
191
|
+
)
|
|
192
|
+
for text, label, message in rejection_cases:
|
|
193
|
+
assert_rejected(
|
|
194
|
+
text,
|
|
195
|
+
label,
|
|
196
|
+
message,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def envelope(text: Any, stop_reason: str = "EndTurn") -> str:
|
|
200
|
+
return json.dumps(
|
|
201
|
+
{
|
|
202
|
+
"text": text,
|
|
203
|
+
"stopReason": stop_reason,
|
|
204
|
+
"sessionId": "session",
|
|
205
|
+
"requestId": "request",
|
|
206
|
+
}
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
envelope_finding = {
|
|
210
|
+
"id": "envelope-finding",
|
|
211
|
+
"severity": "HIGH",
|
|
212
|
+
"detail": {"preserved": True},
|
|
213
|
+
}
|
|
214
|
+
envelope_summary = {"verdict": "NEEDS_WORK"}
|
|
215
|
+
stdout_path.write_text(
|
|
216
|
+
envelope(
|
|
217
|
+
json.dumps(envelope_finding)
|
|
218
|
+
+ "\n# SUMMARY "
|
|
219
|
+
+ json.dumps(envelope_summary)
|
|
220
|
+
+ "\n"
|
|
221
|
+
),
|
|
222
|
+
encoding="utf-8",
|
|
223
|
+
)
|
|
224
|
+
findings, summary = collect_stdout(stdout_path)
|
|
225
|
+
assert findings == [envelope_finding]
|
|
226
|
+
assert summary == envelope_summary
|
|
227
|
+
|
|
228
|
+
assert_rejected(
|
|
229
|
+
envelope("review completed without findings"),
|
|
230
|
+
"findings-less envelope text",
|
|
231
|
+
"error: envelope text rejected: error: invalid JSONL",
|
|
232
|
+
)
|
|
233
|
+
assert_rejected(
|
|
234
|
+
envelope('{"findings":[],"verdict":"PASS"}', "Cancelled"),
|
|
235
|
+
"Cancelled envelope",
|
|
236
|
+
"envelope stopReason must be 'EndTurn'",
|
|
237
|
+
"'Cancelled'",
|
|
238
|
+
)
|
|
239
|
+
assert_rejected(
|
|
240
|
+
envelope("", "UnknownStop"),
|
|
241
|
+
"unknown envelope stopReason",
|
|
242
|
+
"envelope stopReason must be 'EndTurn'",
|
|
243
|
+
"'UnknownStop'",
|
|
244
|
+
)
|
|
245
|
+
assert_rejected(
|
|
246
|
+
envelope(None),
|
|
247
|
+
"non-string envelope text",
|
|
248
|
+
"error: envelope text must be a string",
|
|
249
|
+
)
|
|
250
|
+
welded = (
|
|
251
|
+
"I will inspect the result."
|
|
252
|
+
+ json.dumps(envelope_finding)
|
|
253
|
+
+ "\n# SUMMARY "
|
|
254
|
+
+ json.dumps(envelope_summary)
|
|
255
|
+
+ "\n"
|
|
256
|
+
)
|
|
257
|
+
stdout_path.write_text(envelope(welded), encoding="utf-8")
|
|
258
|
+
findings, summary = collect_stdout(stdout_path)
|
|
259
|
+
assert findings == [envelope_finding]
|
|
260
|
+
assert summary == envelope_summary
|
|
261
|
+
dual_document = (
|
|
262
|
+
json.dumps(envelope_finding)
|
|
263
|
+
+ json.dumps({"id": "second", "severity": "LOW"})
|
|
264
|
+
+ "\n# SUMMARY "
|
|
265
|
+
+ json.dumps(envelope_summary)
|
|
266
|
+
+ "\n"
|
|
267
|
+
)
|
|
268
|
+
assert_rejected(
|
|
269
|
+
envelope(dual_document),
|
|
270
|
+
"concatenated envelope documents",
|
|
271
|
+
"error: envelope text rejected: error: invalid JSONL",
|
|
272
|
+
)
|
|
273
|
+
clean_summary = {"verdict": "PASS"}
|
|
274
|
+
stdout_path.write_text(
|
|
275
|
+
envelope("# SUMMARY " + json.dumps(clean_summary) + "\n"),
|
|
276
|
+
encoding="utf-8",
|
|
277
|
+
)
|
|
278
|
+
findings, summary = collect_stdout(stdout_path)
|
|
279
|
+
assert findings == []
|
|
280
|
+
assert summary == clean_summary
|
|
100
281
|
return 0
|
|
101
282
|
|
|
102
283
|
|
|
@@ -117,9 +298,9 @@ def write_outputs(
|
|
|
117
298
|
def main() -> int:
|
|
118
299
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
119
300
|
parser.add_argument("--devlyn-dir", default=".devlyn")
|
|
120
|
-
parser.add_argument("--stdout-file", default="
|
|
301
|
+
parser.add_argument("--stdout-file", default="pair-judge.stdout")
|
|
121
302
|
parser.add_argument("--out", default="verify.pair.findings.jsonl")
|
|
122
|
-
parser.add_argument("--summary-out", default="
|
|
303
|
+
parser.add_argument("--summary-out", default="pair-judge.summary.json")
|
|
123
304
|
parser.add_argument("--self-test", action="store_true")
|
|
124
305
|
args = parser.parse_args()
|
|
125
306
|
if args.self_test:
|
|
@@ -130,7 +311,7 @@ def main() -> int:
|
|
|
130
311
|
if not stdout_path.is_file():
|
|
131
312
|
sys.stderr.write(f"error: {stdout_path} not found\n")
|
|
132
313
|
return 1
|
|
133
|
-
findings, summary =
|
|
314
|
+
findings, summary = collect_stdout(stdout_path)
|
|
134
315
|
write_outputs(findings, summary, devlyn / args.out, devlyn / args.summary_out)
|
|
135
316
|
print(json.dumps({"findings_count": len(findings), "summary": summary}, sort_keys=True))
|
|
136
317
|
return 0
|
|
@@ -17,12 +17,13 @@ set -uo pipefail
|
|
|
17
17
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
|
|
18
18
|
ADAPTERS_DIR="$SCRIPT_DIR/adapters"
|
|
19
19
|
|
|
20
|
-
TARGETS=(claude codex omp pi)
|
|
21
|
-
KINDS=(cli-engine cli-engine cli-engine orchestrator-only)
|
|
22
|
-
BINARIES=(claude codex omp "")
|
|
20
|
+
TARGETS=(claude codex grok omp pi)
|
|
21
|
+
KINDS=(cli-engine cli-engine cli-engine cli-engine orchestrator-only)
|
|
22
|
+
BINARIES=(claude codex grok omp "")
|
|
23
23
|
INSTALL_HINTS=(
|
|
24
24
|
"see https://docs.anthropic.com/en/docs/claude-code"
|
|
25
25
|
"npm install -g @openai/codex"
|
|
26
|
+
"see https://docs.x.ai/build/overview (install + browser auth)"
|
|
26
27
|
"brew install can1357/tap/omp"
|
|
27
28
|
""
|
|
28
29
|
)
|
|
@@ -88,6 +89,8 @@ for i in "${!TARGETS[@]}"; do
|
|
|
88
89
|
;;
|
|
89
90
|
esac
|
|
90
91
|
|
|
92
|
+
[ "$target" = 'grok' ] && note='not emission-certified; BLOCKED pair source'
|
|
93
|
+
|
|
91
94
|
printf '%-8s %-17s %-8s %-8s %-14s %-12s %s\n' \
|
|
92
95
|
"$target" "$kind" "$binary" "$adapter" "$role" "$pin_eligible" "$note"
|
|
93
96
|
done
|
|
@@ -4,12 +4,12 @@ Used by `/devlyn:resolve` and `/devlyn:ideate`. One shared availability rule so
|
|
|
4
4
|
|
|
5
5
|
## Rule
|
|
6
6
|
|
|
7
|
-
Each skill resolves the effective engine from its own SKILL.md default plus any explicit `--engine` flag passed by the user. `/devlyn:resolve` also computes conditional
|
|
7
|
+
Each skill resolves the effective engine from its own SKILL.md default plus any explicit `--engine` flag passed by the user. `/devlyn:resolve` also computes the default VERIFY-pair route and conditional risk-probe requirements before the phase that needs the OTHER engine.
|
|
8
8
|
|
|
9
9
|
Engine requirements have two classes:
|
|
10
10
|
|
|
11
11
|
- **Explicit routes** — `--engine`, `--risk-probes`, `--pair-verify`. These are promises. If the required engine is unavailable, fail closed with `BLOCKED:<engine>-unavailable` and never downgrade to solo.
|
|
12
|
-
- **Automatic
|
|
12
|
+
- **Automatic routes** — high-risk risk-probes remain conditional; VERIFY pair-JUDGE is selected by default unless `--no-pair` was passed. Both require an available OTHER engine. If it is absent, proceed solo and report the skipped route and reason. This is route selection, not a fallback.
|
|
13
13
|
|
|
14
14
|
## Role resolution
|
|
15
15
|
|
|
@@ -32,11 +32,11 @@ When a run or phase requires engine `<name>` (claude, codex, omp, or any adapter
|
|
|
32
32
|
|
|
33
33
|
Never prompt the user mid-pipeline. Missing engines for explicit routes are BLOCKED states, not silent fallbacks. Missing OTHER engines for automatic escalations are reported solo-skips, not fallbacks — the auto route was never selected.
|
|
34
34
|
|
|
35
|
-
Per-skill defaults: `/devlyn:resolve` uses Claude for PLAN/IMPLEMENT when the orchestrator has Claude Code's native Agent primitive; Codex CLI and oh-my-pi orchestrators use their own fresh worker as the default route (Codex child process, omp native `task`) so the pipeline still has structural phase isolation. VERIFY
|
|
35
|
+
Per-skill defaults: `/devlyn:resolve` uses Claude for PLAN/IMPLEMENT when the orchestrator has Claude Code's native Agent primitive; Codex CLI and oh-my-pi orchestrators use their own fresh worker as the default route (Codex child process, omp native `task`) so the pipeline still has structural phase isolation. VERIFY invokes the OTHER engine by default when available unless `--no-pair` was passed. `/devlyn:ideate` defaults to Claude; `--engine` selects the elicitation/normalization adapter, not an automatic cross-model challenge phase. Any future ideate read-only critique must follow `_shared/codex-config.md` isolation rules. Each SKILL.md flag block is source of truth for that skill's default.
|
|
36
36
|
|
|
37
37
|
## What a skill must report after a BLOCKED engine check
|
|
38
38
|
|
|
39
|
-
When an engine required by the selected route or
|
|
39
|
+
When an engine required by the selected route or default VERIFY pair route is absent, the final user-facing report/summary shows the requested route, the missing engine, and setup steps:
|
|
40
40
|
|
|
41
41
|
```
|
|
42
42
|
Engine: claude + codex pair required
|
|
@@ -48,4 +48,11 @@ Do not report a downgraded successful run when a required engine is missing.
|
|
|
48
48
|
|
|
49
49
|
## Canonical cross-engine invocations
|
|
50
50
|
|
|
51
|
-
After the availability check passes
|
|
51
|
+
After the availability check passes, Codex keeps the monitored `codex-monitored.sh`
|
|
52
|
+
path; every other resolved OTHER engine follows `_shared/adapters/<name>.md`
|
|
53
|
+
`## Invocation`. Capture `.devlyn/<name>-judge.stdout`, then normalize it with
|
|
54
|
+
`collect-codex-findings.py --devlyn-dir "<abs repo>/.devlyn" --stdout-file
|
|
55
|
+
<name>-judge.stdout`. A non-zero collector exit writes no canonical findings:
|
|
56
|
+
the pair source is `BLOCKED` for `verify.pair.emission-contract`, and unparsed
|
|
57
|
+
stdout remains diagnostic-only rather than being merged. On exit 124, the
|
|
58
|
+
timeout marker's `engine` field is the resolved `<name>`.
|
|
@@ -24,6 +24,13 @@
|
|
|
24
24
|
"description": "Required exit code. Default 0 if omitted.",
|
|
25
25
|
"default": 0
|
|
26
26
|
},
|
|
27
|
+
"timeout_sec": {
|
|
28
|
+
"type": "integer",
|
|
29
|
+
"minimum": 1,
|
|
30
|
+
"maximum": 600,
|
|
31
|
+
"description": "Per-command timeout in seconds. Default 60 if omitted; maximum 600.",
|
|
32
|
+
"default": 60
|
|
33
|
+
},
|
|
27
34
|
"stdout_contains": {
|
|
28
35
|
"type": "array",
|
|
29
36
|
"description": "Each substring must appear verbatim in (stdout + stderr) for pass.",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Deny pair-judge shell outside the probe anchor, with a model-readable reason.
|
|
3
|
+
|
|
4
|
+
Registered as a native grok `PreToolUse` hook. It fires at step 1 of the
|
|
5
|
+
authorization pipeline, before the permission rules and before the mode policy
|
|
6
|
+
whose auto-deny silently terminates the review at exit 0 (iter-0081).
|
|
7
|
+
|
|
8
|
+
The anchor is supplied via DEVLYN_PROBE_ANCHOR, derived by the orchestrator from
|
|
9
|
+
the same source as the --allow rule.
|
|
10
|
+
|
|
11
|
+
Measured (iter-0081 M-C6 v2, external isolation, admissible cell): Line A 6/6 —
|
|
12
|
+
the denial arrives as a model-visible tool_result, the judge re-routes to
|
|
13
|
+
read_file, the anchor still executes inside the grok process, EndTurn with zero
|
|
14
|
+
PermissionCancelled. Line A-chain 1/1 — `<anchor> && <write>` is denied
|
|
15
|
+
model-visibly, neither segment executes, and the judge then runs the bare anchor.
|
|
16
|
+
"""
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
REASON = (
|
|
22
|
+
"DEVLYN-0081-HOOK-DENY: this shell command is outside the permitted probe "
|
|
23
|
+
"anchor. Use the read_file tool for file reads. The only permitted shell "
|
|
24
|
+
"command is the anchor named in your prompt, unchained."
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
SHELL_TOOLS = {"run_terminal_command", "run_terminal_cmd", "Bash"}
|
|
28
|
+
|
|
29
|
+
# A conservative, quote-OBLIVIOUS lexical veto — deliberately not a shell parser.
|
|
30
|
+
# It rejects these characters even inside quotes, over-denying rather than ever
|
|
31
|
+
# under-denying a chain. Adding quote-, escape-, or operator-aware exceptions is
|
|
32
|
+
# how this becomes a parser; that is a registered falsifier, not a polish.
|
|
33
|
+
FORBIDDEN = set(";&|`$()<>") | {"\n", "\r"}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main():
|
|
37
|
+
anchor = os.environ.get("DEVLYN_PROBE_ANCHOR", "")
|
|
38
|
+
payload = json.load(sys.stdin)
|
|
39
|
+
|
|
40
|
+
if payload.get("toolName") not in SHELL_TOOLS or not anchor:
|
|
41
|
+
print(json.dumps({"decision": "allow"}))
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
# The anchor, or the anchor plus argv, and nothing that could start a second
|
|
45
|
+
# command. A bare prefix rule would allow `<anchor> && <other>` — the vendor's
|
|
46
|
+
# own documented footgun. Exact match would instead deny `<anchor> --n 5`,
|
|
47
|
+
# mechanically destroying the "bounded input variations" that verify.md
|
|
48
|
+
# requires of every engine. Both were rejected by both seats (iter-0081 v2).
|
|
49
|
+
command = (payload.get("toolInput") or {}).get("command", "").strip()
|
|
50
|
+
if command == anchor or (
|
|
51
|
+
command.startswith(anchor + " ")
|
|
52
|
+
and not any(ch in FORBIDDEN for ch in command)
|
|
53
|
+
):
|
|
54
|
+
print(json.dumps({"decision": "allow"}))
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
print(json.dumps({"decision": "deny", "reason": REASON}))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == "__main__":
|
|
61
|
+
main()
|