devlyn-cli 4.0.1 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/AGENTS.md +11 -95
  2. package/CLAUDE.md +7 -160
  3. package/LICENSE +21 -0
  4. package/README.md +83 -114
  5. package/bin/devlyn.js +330 -472
  6. package/bin/instruction-templates.json +120 -2
  7. package/bin/instructions.js +35 -8
  8. package/config/skills/_shared/adapters/claude.md +8 -85
  9. package/config/skills/_shared/adapters/codex.md +14 -34
  10. package/config/skills/_shared/adapters/omp.md +1 -11
  11. package/config/skills/_shared/codex-monitored.sh +4 -54
  12. package/config/skills/_shared/engine-doctor.sh +18 -58
  13. package/config/skills/_shared/engine-preflight.md +15 -47
  14. package/config/skills/_shared/expected-contract.py +242 -0
  15. package/config/skills/_shared/expected.schema.json +10 -95
  16. package/config/skills/_shared/invocation-receipt.py +26 -757
  17. package/config/skills/_shared/platform-support.py +37 -28
  18. package/config/skills/_shared/role-config.py +100 -421
  19. package/config/skills/_shared/runtime-principles.md +17 -93
  20. package/config/skills/_shared/task-complete.py +490 -285
  21. package/config/skills/_shared/task-completion.md +235 -0
  22. package/config/skills/devlyn-engines/SKILL.md +9 -11
  23. package/config/skills/devlyn-ideate/SKILL.md +57 -107
  24. package/config/skills/devlyn-ideate/references/elicitation.md +18 -99
  25. package/config/skills/devlyn-ideate/references/loop.md +110 -0
  26. package/config/skills/devlyn-ideate/references/package-format.md +128 -0
  27. package/config/skills/devlyn-ideate/scripts/acceptance.py +795 -0
  28. package/config/skills/devlyn-ideate/scripts/queue.py +1780 -0
  29. package/package.json +2 -3
  30. package/config/commit-conventions.md +0 -28
  31. package/config/skills/_shared/adapters/README.md +0 -74
  32. package/config/skills/_shared/adapters/grok.md +0 -100
  33. package/config/skills/_shared/archive_run.py +0 -969
  34. package/config/skills/_shared/codex-config.md +0 -81
  35. package/config/skills/_shared/collect-codex-findings.py +0 -505
  36. package/config/skills/_shared/finish-gate.py +0 -521
  37. package/config/skills/_shared/grok-anchor-guard.py +0 -78
  38. package/config/skills/_shared/judge-output-parser.py +0 -200
  39. package/config/skills/_shared/judge-role-evidence.py +0 -320
  40. package/config/skills/_shared/phase-prompt-render.py +0 -183
  41. package/config/skills/_shared/process-evidence.py +0 -926
  42. package/config/skills/_shared/resolve-bootstrap.py +0 -1410
  43. package/config/skills/_shared/resolve-stop-hook.py +0 -359
  44. package/config/skills/_shared/run-bounded.py +0 -58
  45. package/config/skills/_shared/spec-verify-check.py +0 -5083
  46. package/config/skills/_shared/state-phase-write.py +0 -5989
  47. package/config/skills/_shared/terminal-claim-check.py +0 -777
  48. package/config/skills/_shared/verify-merge-findings.py +0 -3528
  49. package/config/skills/devlyn-design-ui/SKILL.md +0 -650
  50. package/config/skills/devlyn-ideate/references/from-spec-mode.md +0 -61
  51. package/config/skills/devlyn-ideate/references/project-mode.md +0 -80
  52. package/config/skills/devlyn-ideate/references/spec-template.md +0 -121
  53. package/config/skills/devlyn-ideate/references/templates/decision.md +0 -48
  54. package/config/skills/devlyn-ideate/references/templates/roadmap.md +0 -50
  55. package/config/skills/devlyn-ideate/references/templates/vision.md +0 -44
  56. package/config/skills/devlyn-queue/SKILL.md +0 -66
  57. package/config/skills/devlyn-queue/scripts/append.py +0 -94
  58. package/config/skills/devlyn-resolve/SKILL.md +0 -379
  59. package/config/skills/devlyn-resolve/references/free-form-mode.md +0 -71
  60. package/config/skills/devlyn-resolve/references/outer-loop.md +0 -23
  61. package/config/skills/devlyn-resolve/references/phases/build-gate.md +0 -75
  62. package/config/skills/devlyn-resolve/references/phases/cleanup.md +0 -33
  63. package/config/skills/devlyn-resolve/references/phases/implement.md +0 -44
  64. package/config/skills/devlyn-resolve/references/phases/plan.md +0 -39
  65. package/config/skills/devlyn-resolve/references/phases/probe-derive.md +0 -261
  66. package/config/skills/devlyn-resolve/references/phases/surface-close.md +0 -34
  67. package/config/skills/devlyn-resolve/references/phases/verify.md +0 -310
  68. package/config/skills/devlyn-resolve/references/state-schema.md +0 -152
  69. package/config/skills/devlyn-resolve/references/task-completion.md +0 -160
  70. package/config/templates/prompt-templates.md +0 -71
  71. package/config/templates/template-feature.spec.md +0 -255
  72. package/config/templates/template-product-spec.md +0 -680
@@ -1,81 +0,0 @@
1
- # Shared — Codex Invocation
2
-
3
- Single source of truth for how every skill calls Codex. **MCP is not used.** Skills shell out via the wrapper at `$CODEX_MONITORED_PATH`, resolved from the invoked skill's sibling `_shared/codex-monitored.sh`, which fronts the local Codex CLI.
4
-
5
- ## Canonical invocations
6
-
7
- All long-running Codex calls go through `codex-monitored.sh`. It streams full stdout and emits a `[codex-monitored] heartbeat` every 30s on stderr. Write multiline prompts as exact UTF-8 bytes to a task-local file; set `DEVLYN_CODEX_PROMPT_FILE` and pass the sole prompt argument `-`. The wrapper snapshots those bytes before dispatch and seals actual argv, prompt digest and completion in `<prompt-file>.transport.json`. Missing/unreadable files, competing prompt arguments and receipt/prompt mismatches fail before launch. Without file transport, stdin remains DEVNULL (avoiding the open-stdin plus argument-prompt hang).
8
-
9
- Use the DEVLYN_SHARED_DIR established by the invoking skill; if it was
10
- lost, re-establish it by that skill's rules. When this document is
11
- opened directly, bind DEVLYN_SHARED_DIR to its reader-supplied
12
- containing directory, resolving directory symlinks. Never take it from
13
- an environment variable, cwd or another installation; without either
14
- source, stop with BLOCKED:skill-source-unresolved.
15
-
16
- Bind CODEX_MONITORED_PATH to DEVLYN_SHARED_DIR/codex-monitored.sh and
17
- verify the file before calling it; failure is
18
- BLOCKED:shared-dir-unresolved with the failed path.
19
-
20
- After establishing DEVLYN_SHARED_DIR in a Bash invocation:
21
- ```bash
22
- CODEX_MONITORED_PATH="$DEVLYN_SHARED_DIR/codex-monitored.sh"
23
- ```
24
-
25
- **Read-only critique / adversarial review / debate** (`/devlyn-resolve` VERIFY pair-mode, plus any future ideate read-only critique). Codex returns findings on stdout; the orchestrator writes files.
26
-
27
- ```bash
28
- DEVLYN_CODEX_PROMPT_FILE="<prompt-file>" CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600 bash "$CODEX_MONITORED_PATH" \
29
- -C <project-root> \
30
- -s read-only \
31
- -c model_reasoning_effort=xhigh \
32
- -
33
- ```
34
-
35
- **Workspace-write implementation** (`/devlyn-resolve` IMPLEMENT phase when `--engine codex` or `--engine auto` routes to Codex, plus codex-routed `/devlyn-ideate` phases):
36
-
37
- ```bash
38
- DEVLYN_CODEX_PROMPT_FILE="<prompt-file>" bash "$CODEX_MONITORED_PATH" \
39
- -C <project-root> \
40
- -s workspace-write \
41
- -c sandbox_workspace_write.network_access=false \
42
- -c model_reasoning_effort=xhigh \
43
- -
44
- ```
45
-
46
- PLAN uses owner reasoning; BUILD_GATE/CLEANUP run owner commands, without separate Codex invocations. Code/doc cleanup runs inside the selected IMPLEMENT invocation. An already-open historical BUILD span without `execution_kind` keeps its receipt-bound workspace-write/network-true worker route; do not reinterpret or rewrite that evidence.
47
-
48
- Notes:
49
- - `DEVLYN_CODEX_PROMPT_FILE` — use the same file as `DEVLYN_INVOCATION_PROMPT_FILE` for receipt-bound calls. Retain the generated transport carrier with the prompt/session/argv evidence; a path or claimed environment value alone does not prove delivery. Do not use shell command substitution for prompt bytes.
50
- - `-C` — project root so Codex's working directory matches.
51
- - `-s read-only` / `-s workspace-write` — sandbox policy. Use workspace-write for implementation/probe phases that write tracked files or `.devlyn` artifacts.
52
- - `-c sandbox_workspace_write.network_access=<true|false>` — required and receipt-bound for mutation phases: `true` only for historical receipt-bound BUILD_GATE and, per call, PROBE-DERIVE when the probe's visible Verification command requires a localhost service; `false` for IMPLEMENT, historical worker PLAN/CLEANUP, and PROBE-DERIVE by default. This allows CI-equivalent loopback/network tests without widening to `danger-full-access` and prevents user configuration from silently changing other phases.
53
- - `-c model_reasoning_effort=xhigh` — config override for reasoning depth. Required for deep critique; skills may choose `high` or `medium` when thoroughness doesn't warrant xhigh.
54
- - **For every receipt-bound Codex call, add `--json -m <model_requested>` to the recipes above**, matching the model recorded at spawn; keep the skill’s existing model-selection rules. Other routes keep their omission rules unless an explicit role profile supplies a model: normal workers may inherit user config, while unconfigured isolated VERIFY uses its CLI default. Explicit profiles use role-config.py options and the separate judge-role-evidence.py contract; they do not relax mutation receipts. Omission proves neither model identity nor role fitness. Re-certify seats after model/version changes before re-pinning them.
55
- - `CODEX_MONITORED_ISOLATED=1` — required for bounded read-only critique/probe/judge calls. The wrapper adds `--ignore-user-config --ignore-rules --ephemeral --disable codex_hooks --disable hooks` so user config, AGENTS.md, hooks, and project rules cannot add hidden context, tool calls, or transcript side effects. Do not set it for workspace-write implementation phases.
56
- - Wrapper calls are **foreground-blocking**. Never launch them via a backgrounded shell (`run_in_background`, `&`, `nohup`) and never end the orchestrator message while one runs: a headless print-mode session kills backgrounded children at wind-down (observed 2026-07-07: an FS1 A-arm IMPLEMENT codex call was killed at turn end → 0-byte delivery). The heartbeat stream is the observability channel; block on the call. Interactive Claude Code caps a foreground Bash call at `BASH_MAX_TIMEOUT_MS` (the installer sets at least 3600000 ms, IMPLEMENT's effective outer ceiling), so every foreground Codex wrapper call must pass the Bash `timeout` explicitly at that ceiling or the phase's own budget (for example, 600s judges) — never rely on the 120 s default.
57
- - Raw `codex exec ...` invocations are **forbidden** in skill prompts. The benchmark variant arm runs a PATH shim (`scripts/codex-shim/codex`) that transparently re-routes any raw `codex exec` to the wrapper as a safety net, but skills should always emit the wrapper form directly so the orchestrator's first-attempt has the right shape. Two prior iterations (iter-0006 universal foreground ban, iter-0008 prompt-level kill-shape contract) failed because the orchestrator picked starvation-prone shapes (`codex exec ... 2>&1 | tail -200`) from its own pattern prior — the wrapper plus the shim is the runtime binding layer those iters lacked. See `autoresearch/iterations/0009-wrapper-and-hook.md`.
58
-
59
- ## Constrained Windows judge reads
60
-
61
- Codex supports [native Windows sandboxing](https://learn.chatgpt.com/docs/windows/windows-sandbox). A policy-denied read selects a constrained-read route for that judge; it does not establish that Windows lacks sandbox support or denies every read. The orchestrator supplies the complete spec, sibling expected contract, accepted source SHA and cumulative diff, relevant source/tests with file:line locators, and validated sealed MECHANICAL results, manifests and required raw streams inline in the prompt file. Tell the fresh judge: "Judge only the supplied evidence; run no tools. Missing, truncated or unbound inputs require a verdict-binding BLOCKED finding." Keep `-s read-only`, isolation, freshness, selected model/effort, the 600s budget and normal findings/timeout handling. Never widen sandbox permissions or invent evidence to complete the packet.
62
-
63
- ## Availability check
64
-
65
- Before the first Codex call in a run, verify the CLI is on PATH:
66
-
67
- ```bash
68
- command -v codex >/dev/null 2>&1
69
- ```
70
-
71
- If the check fails while Codex is explicitly selected or conditionally required by pair/risk-probe VERIFY, follow `_shared/engine-preflight.md`: stop with `BLOCKED:codex-unavailable`, preserve run evidence, and print setup guidance. Do not convert the run to Claude. `--no-pair` and `--no-risk-probes` are explicit user opt-outs for reruns, not automatic fallbacks.
72
-
73
- ## Why CLI over other paths
74
-
75
- The local Codex CLI, fronted by `codex-monitored.sh`, is the integration boundary. CLI flags compose with the existing phase routes and keep invocation evidence inspectable. The shared preflight checks availability; a present binary does not establish model identity or role fitness.
76
-
77
- ## Invocation from inside a skill prompt
78
-
79
- Skills write the invocation as a Bash command the runtime executes. Example shape from `/devlyn-resolve` PHASE 2 IMPLEMENT when routed to Codex:
80
-
81
- > Run `DEVLYN_CODEX_PROMPT_FILE="<IMPLEMENT-prompt-file>" bash "$CODEX_MONITORED_PATH" --json -C <state.base_ref.repo_root> -s workspace-write -m <model_requested> -c sandbox_workspace_write.network_access=false -c model_reasoning_effort=xhigh -`. Capture stdout as the IMPLEMENT reply; non-zero exit → treat as subagent failure. The wrapper emits `[codex-monitored]` heartbeat and lifecycle lines on **stderr** — stdout stays clean for Codex output, so the orchestrator can parse the reply without filtering. Heartbeat-on-stderr keeps the orchestrator's combined-output stream non-silent (defeats the iter-0008 byte-watchdog kill) without polluting the codex-reply view of stdout. Do not pipe the wrapper; direct capture or file redirection preserves streaming and avoids the pipe-refusal exit.
@@ -1,505 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Normalize raw pair-JUDGE stdout into canonical VERIFY JSONL."""
3
-
4
- from __future__ import annotations
5
-
6
- import argparse
7
- import json
8
- import pathlib
9
- import runpy
10
- import sys
11
- import tempfile
12
- from typing import Any
13
-
14
-
15
- MERGE_CONTRACT = runpy.run_path(pathlib.Path(__file__).with_name("verify-merge-findings.py"))
16
- PARSER = runpy.run_path(pathlib.Path(__file__).with_name("judge-output-parser.py"))
17
- VERDICT_RANK = MERGE_CONTRACT["VERDICT_RANK"]
18
- finding_rank = MERGE_CONTRACT["finding_rank"]
19
-
20
-
21
- def reject_json_constant(token: str) -> None:
22
- raise ValueError(f"invalid JSON numeric constant: {token}")
23
-
24
-
25
- def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
26
- result = {}
27
- for key, value in pairs:
28
- if key in result:
29
- raise ValueError(f"duplicate JSON key: {key}")
30
- result[key] = value
31
- return result
32
-
33
-
34
- def loads_strict_json(text: str) -> Any:
35
- return json.loads(
36
- text,
37
- parse_constant=reject_json_constant,
38
- object_pairs_hook=reject_duplicate_keys,
39
- )
40
-
41
-
42
- def atomic_write(path: pathlib.Path, text: str) -> None:
43
- path.parent.mkdir(parents=True, exist_ok=True)
44
- with tempfile.NamedTemporaryFile(
45
- "w", encoding="utf-8", dir=path.parent, delete=False
46
- ) as handle:
47
- handle.write(text)
48
- tmp_name = handle.name
49
- pathlib.Path(tmp_name).replace(path)
50
-
51
-
52
- def collect_text(
53
- text: str, source: pathlib.Path
54
- ) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
55
- findings, summary = PARSER["collect_text"](text, source)
56
- if findings and summary is None:
57
- raise SystemExit("error: findings without terminal verdict")
58
- if summary is not None and summary["verdict"] == "PASS" and any(finding_rank(finding) == 2 for finding in findings):
59
- raise SystemExit("error: verdict-binding finding cannot have a PASS verdict")
60
- if not findings and (summary is None or summary.get("verdict") != "PASS"):
61
- raise SystemExit("error: non-PASS verdict without JSONL findings")
62
- return findings, summary
63
-
64
-
65
- def collect_stdout(stdout_path: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
66
- findings, summary = PARSER["collect_stdout"](stdout_path)
67
- if findings and summary is None:
68
- raise SystemExit("error: findings without terminal verdict")
69
- if summary is not None and summary["verdict"] == "PASS" and any(finding_rank(finding) == 2 for finding in findings):
70
- raise SystemExit("error: verdict-binding finding cannot have a PASS verdict")
71
- if not findings and (summary is None or summary.get("verdict") != "PASS"):
72
- raise SystemExit("error: non-PASS verdict without JSONL findings")
73
- return findings, summary
74
-
75
-
76
- def self_test() -> int:
77
- try:
78
- loads_strict_json('{"verdict":"PASS","verdict":"BLOCKED"}')
79
- except ValueError as exc:
80
- assert "duplicate JSON key" in str(exc)
81
- else:
82
- raise AssertionError("duplicate judge result key was accepted")
83
- with tempfile.TemporaryDirectory() as tmp:
84
- root = pathlib.Path(tmp)
85
- stdout_path = root / "pair-judge.stdout"
86
- out_path = root / "verify.pair.findings.jsonl"
87
- summary_path = root / "pair-judge.summary.json"
88
-
89
- def assert_rejected(text: str, label: str, *message_parts: str) -> None:
90
- stdout_path.write_text(text, encoding="utf-8")
91
- try:
92
- collect_stdout(stdout_path)
93
- except SystemExit as exc:
94
- message = str(exc)
95
- assert all(part in message for part in message_parts)
96
- else:
97
- raise AssertionError(f"{label} must be rejected")
98
-
99
- plain_finding = {"id": "a", "severity": "HIGH"}
100
- plain_summary = {"verdict": "NEEDS_WORK"}
101
- stdout_path.write_text(
102
- json.dumps(plain_finding) + "\n"
103
- + plain_summary["verdict"] + "\n",
104
- encoding="utf-8",
105
- )
106
- findings, summary = collect_stdout(stdout_path)
107
- assert findings == [plain_finding]
108
- assert summary == plain_summary
109
- write_outputs(findings, summary, out_path, summary_path)
110
- assert out_path.read_text(encoding="utf-8").count("\n") == 1
111
- assert loads_strict_json(summary_path.read_text(encoding="utf-8"))["verdict"] == "NEEDS_WORK"
112
- assert_rejected(
113
- '{"id":"nan","severity":NaN}\n',
114
- "NaN pair-JUDGE stdout finding",
115
- "invalid JSON numeric constant: NaN",
116
- )
117
- assert_rejected(
118
- '# SUMMARY {"verdict":"PASS","verdict":"BLOCKED"}\n',
119
- "duplicate pair-JUDGE summary verdict",
120
- "duplicate JSON key: verdict",
121
- )
122
- rejection_cases = (
123
- ("", "no verdict line", "non-PASS verdict without JSONL findings"),
124
- (json.dumps(plain_finding) + "\n", "finding without terminal verdict", "findings without terminal verdict"),
125
- ('{"verdict":"pass"}\n', "lowercase pass", "finding missing valid severity"),
126
- ('{"verdict":"UNKNOWN"}\n', "unknown verdict", "finding missing valid severity"),
127
- )
128
- for text, label, message in rejection_cases:
129
- assert_rejected(
130
- text,
131
- label,
132
- message,
133
- )
134
-
135
- def envelope(text: Any, stop_reason: str = "EndTurn") -> str:
136
- return json.dumps(
137
- {
138
- "text": text,
139
- "stopReason": stop_reason,
140
- "sessionId": "session",
141
- "requestId": "request",
142
- }
143
- )
144
-
145
- envelope_finding = {
146
- "id": "envelope-finding",
147
- "severity": "HIGH",
148
- "detail": {"preserved": True},
149
- }
150
- envelope_summary = {"verdict": "NEEDS_WORK"}
151
- stdout_path.write_text(
152
- envelope(
153
- json.dumps(envelope_finding)
154
- + "\n" + envelope_summary["verdict"] + "\n"
155
- ),
156
- encoding="utf-8",
157
- )
158
- findings, summary = collect_stdout(stdout_path)
159
- assert findings == [envelope_finding]
160
- assert summary == envelope_summary
161
-
162
- assert_rejected(
163
- envelope("review completed without findings"),
164
- "findings-less envelope text",
165
- "invalid JSONL",
166
- )
167
- assert_rejected(
168
- envelope('{"findings":[],"verdict":"PASS"}', "Cancelled"),
169
- "Cancelled envelope",
170
- "envelope stopReason must be 'EndTurn'",
171
- "'Cancelled'",
172
- )
173
- assert_rejected(
174
- envelope("", "UnknownStop"),
175
- "unknown envelope stopReason",
176
- "envelope stopReason must be 'EndTurn'",
177
- "'UnknownStop'",
178
- )
179
- assert_rejected(
180
- envelope(None),
181
- "non-string envelope text",
182
- "error: envelope text must be a string",
183
- )
184
- assert_rejected(
185
- envelope("I will inspect the result.\n" + envelope_summary["verdict"] + "\n"),
186
- "narrative envelope preamble",
187
- "invalid JSONL",
188
- )
189
- dual_document = (
190
- json.dumps(envelope_finding)
191
- + json.dumps({"id": "second", "severity": "LOW"})
192
- + "\n" + envelope_summary["verdict"] + "\n"
193
- )
194
- assert_rejected(
195
- envelope(dual_document),
196
- "concatenated envelope documents",
197
- "invalid JSONL",
198
- )
199
- clean_summary = {"verdict": "PASS"}
200
- stdout_path.write_text(
201
- envelope(clean_summary["verdict"] + "\n"),
202
- encoding="utf-8",
203
- )
204
- findings, summary = collect_stdout(stdout_path)
205
- assert findings == []
206
- assert summary == clean_summary
207
- stdout_path.write_text(
208
- json.dumps(
209
- {
210
- "type": "system",
211
- "text": clean_summary["verdict"] + "\n",
212
- "stopReason": "EndTurn",
213
- "sessionId": "session",
214
- "requestId": "request",
215
- }
216
- ),
217
- encoding="utf-8",
218
- )
219
- findings, summary = collect_stdout(stdout_path)
220
- assert findings == []
221
- assert summary == clean_summary
222
-
223
- # R4: a valid plain finding may carry an unrelated `type` field. A
224
- # first-line `type: system` finding is not a stream discriminator.
225
- system_finding = {"id": "system-finding", "severity": "LOW", "type": "system"}
226
- stdout_path.write_text(
227
- json.dumps(system_finding) + "\n" + clean_summary["verdict"] + "\n",
228
- encoding="utf-8",
229
- )
230
- findings, summary = collect_stdout(stdout_path)
231
- assert findings == [system_finding]
232
- assert summary == clean_summary
233
-
234
- # iter-0106 — Grok whole-message NDJSON carrier (R2/R3).
235
- def init_record(session: str = "s1") -> dict[str, Any]:
236
- return {"type": "system", "subtype": "init", "session_id": session, "model": "grok-build"}
237
-
238
- def assistant_record(
239
- blocks: list[Any], stop_reason: str = "end_turn", session: str = "s1"
240
- ) -> dict[str, Any]:
241
- return {
242
- "type": "assistant",
243
- "message": {"id": "msg", "role": "assistant", "content": blocks, "stop_reason": stop_reason},
244
- "parent_tool_use_id": None,
245
- "session_id": session,
246
- }
247
-
248
- def result_record(
249
- text: str,
250
- session: str = "s1",
251
- subtype: str = "success",
252
- is_error: Any = False,
253
- stop_reason: str = "end_turn",
254
- ) -> dict[str, Any]:
255
- return {
256
- "type": "result",
257
- "subtype": subtype,
258
- "is_error": is_error,
259
- "result": text,
260
- "stop_reason": stop_reason,
261
- "session_id": session,
262
- }
263
-
264
- def text_block(value: str) -> dict[str, Any]:
265
- return {"type": "text", "text": value}
266
-
267
- def stream(*records: Any) -> str:
268
- return "".join(json.dumps(record) + "\n" for record in records)
269
-
270
- pass_text = "PASS\n"
271
- stdout_path.write_text(
272
- stream(init_record(), assistant_record([text_block(pass_text)]), result_record(pass_text)),
273
- encoding="utf-8",
274
- )
275
- findings, summary = collect_stdout(stdout_path)
276
- assert findings == []
277
- assert summary == {"verdict": "PASS"}
278
-
279
- # A real tool turn precedes the contract, and the terminal message arrives
280
- # as two text blocks whose in-order concatenation is the result text.
281
- stream_finding = {"id": "stream-finding", "severity": "HIGH"}
282
- head = json.dumps(stream_finding) + "\n"
283
- tail = "# SUMMARY " + json.dumps({"verdict": "NEEDS_WORK"}) + "\n"
284
- contract = head + tail
285
- narration = assistant_record(
286
- [
287
- text_block("Let me read the file."),
288
- {"type": "tool_use", "id": "call_1", "name": "read_file", "input": {"path": "src/main.rs"}},
289
- ],
290
- stop_reason="tool_use",
291
- )
292
- tool_result = {
293
- "type": "user",
294
- "message": {
295
- "role": "user",
296
- "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "fn main() {}"}],
297
- },
298
- "parent_tool_use_id": None,
299
- "session_id": "s1",
300
- }
301
- stdout_path.write_text(
302
- stream(
303
- init_record(),
304
- narration,
305
- tool_result,
306
- assistant_record([text_block(head), text_block(tail)]),
307
- result_record(contract),
308
- ),
309
- encoding="utf-8",
310
- )
311
- findings, summary = collect_stdout(stdout_path)
312
- assert findings == [stream_finding]
313
- assert summary == {"verdict": "NEEDS_WORK"}
314
-
315
- # The welded body is exactly the shape envelope recovery accepts, so this
316
- # also proves the stream branch never falls through to that recovery.
317
- welded = "I reviewed the diff.\n" + contract
318
- stream_rejections = (
319
- (
320
- stream(init_record(), assistant_record([text_block(welded)]), result_record(welded)),
321
- "narration welded into the terminal message",
322
- "invalid JSONL",
323
- ),
324
- (
325
- stream(
326
- init_record(),
327
- assistant_record([text_block(contract)]),
328
- assistant_record([text_block(contract)]),
329
- result_record(contract),
330
- ),
331
- "two terminal end_turn messages",
332
- "exactly one final end_turn assistant message",
333
- ),
334
- (
335
- stream(
336
- init_record(),
337
- assistant_record([text_block(contract)]),
338
- assistant_record([text_block("more")], stop_reason="tool_use"),
339
- result_record(contract),
340
- ),
341
- "end_turn message that is not the final assistant message",
342
- "exactly one final end_turn assistant message",
343
- ),
344
- (
345
- # The malformed line is the only defect: skipping it leaves a valid stream.
346
- stream(init_record(), assistant_record([text_block(pass_text)]))
347
- + '{"type":"assistant"\n'
348
- + stream(result_record(pass_text)),
349
- "malformed stream NDJSON inside an otherwise valid stream",
350
- "invalid stream NDJSON",
351
- ),
352
- (
353
- stream(init_record(), assistant_record([text_block(pass_text)])),
354
- "stream without a terminal result",
355
- "exactly one terminal result",
356
- ),
357
- (
358
- stream(
359
- init_record(),
360
- assistant_record([text_block(pass_text)]),
361
- result_record(pass_text),
362
- tool_result,
363
- ),
364
- "data after the terminal result",
365
- "exactly one terminal result",
366
- ),
367
- (
368
- stream(
369
- init_record(),
370
- assistant_record([text_block(pass_text)]),
371
- tool_result,
372
- result_record(pass_text),
373
- ),
374
- "data between the terminal assistant and result",
375
- "exactly one final end_turn assistant message",
376
- ),
377
- (
378
- stream(
379
- init_record(),
380
- assistant_record([text_block(pass_text)]),
381
- result_record(pass_text, subtype="error_during_execution"),
382
- ),
383
- "error result subtype",
384
- "not a successful end_turn",
385
- ),
386
- (
387
- stream(
388
- init_record(),
389
- assistant_record([text_block(pass_text)]),
390
- result_record(pass_text, stop_reason="cancelled"),
391
- ),
392
- "cancelled result",
393
- "not a successful end_turn",
394
- ),
395
- (
396
- stream(
397
- init_record(),
398
- assistant_record([text_block(pass_text)]),
399
- result_record(pass_text, is_error=True),
400
- ),
401
- "error-flagged result",
402
- "not a successful end_turn",
403
- ),
404
- (
405
- stream(
406
- init_record(),
407
- assistant_record([text_block(pass_text)]),
408
- result_record("NEEDS_WORK\n"),
409
- ),
410
- "result text that disagrees with the terminal message",
411
- "result text does not match the terminal message",
412
- ),
413
- (
414
- stream(
415
- init_record(),
416
- assistant_record([text_block(pass_text)]),
417
- result_record(pass_text, session="s2"),
418
- ),
419
- "result from another session",
420
- "session identity does not agree",
421
- ),
422
- (
423
- stream(
424
- init_record(),
425
- {"type": "stream_event", "event": {"type": "content_block_delta"}, "session_id": "s1"},
426
- assistant_record([text_block(pass_text)]),
427
- result_record(pass_text),
428
- ),
429
- "partial-message frame",
430
- "unknown stream record",
431
- ),
432
- (
433
- stream(
434
- init_record(),
435
- {"type": "system", "subtype": "compact_boundary", "session_id": "s1"},
436
- assistant_record([text_block(pass_text)]),
437
- result_record(pass_text),
438
- ),
439
- "compact boundary",
440
- "second system record",
441
- ),
442
- (
443
- stream(
444
- init_record(),
445
- {"type": "unknown", "session_id": "s1"},
446
- assistant_record([text_block(pass_text)]),
447
- result_record(pass_text),
448
- ),
449
- "unknown record type",
450
- "unknown stream record",
451
- ),
452
- (
453
- stream(
454
- {"type": "system", "subtype": "compact_boundary", "session_id": "s1"},
455
- assistant_record([text_block(pass_text)]),
456
- result_record(pass_text),
457
- ),
458
- "stream that does not open with system/init",
459
- "must open with system/init",
460
- ),
461
- )
462
- for text, label, message in stream_rejections:
463
- assert_rejected(text, label, message)
464
- return 0
465
-
466
-
467
- def write_outputs(
468
- findings: list[dict[str, Any]],
469
- summary: dict[str, Any] | None,
470
- out_path: pathlib.Path,
471
- summary_path: pathlib.Path,
472
- ) -> None:
473
- atomic_write(
474
- out_path,
475
- "".join(json.dumps(item, sort_keys=True, separators=(",", ":")) + "\n" for item in findings),
476
- )
477
- if summary is not None:
478
- atomic_write(summary_path, json.dumps(summary, indent=2, sort_keys=True) + "\n")
479
-
480
-
481
- def main() -> int:
482
- parser = argparse.ArgumentParser(description=__doc__)
483
- parser.add_argument("--devlyn-dir", default=".devlyn")
484
- parser.add_argument("--stdout-file", default="pair-judge.stdout")
485
- parser.add_argument("--out", default="verify.pair.findings.jsonl")
486
- parser.add_argument("--summary-out", default="pair-judge.summary.json")
487
- parser.add_argument("--self-test", action="store_true")
488
- args = parser.parse_args()
489
- if args.self_test:
490
- return self_test()
491
-
492
- devlyn = pathlib.Path(args.devlyn_dir)
493
- stdout_path = devlyn / args.stdout_file
494
- if not stdout_path.is_file():
495
- sys.stderr.write(f"error: {stdout_path} not found\n")
496
- return 1
497
- findings, summary = collect_stdout(stdout_path)
498
- write_outputs(findings, summary, devlyn / args.out, devlyn / args.summary_out)
499
- print(json.dumps({"findings_count": len(findings), "summary": summary}, sort_keys=True))
500
- return 0
501
-
502
-
503
- if __name__ == "__main__":
504
- runpy.run_path(str(pathlib.Path(__file__).with_name("platform-support.py")))["configure_utf8"]()
505
- raise SystemExit(main())