devlyn-cli 2.10.0 → 2.10.1

package/config/skills/_shared/collect-codex-findings.py

@@ -1,17 +1,27 @@
 #!/usr/bin/env python3
-"""Normalize raw Codex pair-JUDGE stdout into canonical VERIFY JSONL."""
+"""Normalize raw pair-JUDGE stdout into canonical VERIFY JSONL."""
 
 from __future__ import annotations
 
 import argparse
 import json
 import pathlib
+import runpy
 import sys
 import tempfile
 from typing import Any
 
 
 FINDING_SEVERITIES = {"CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"}
+ENVELOPE_KEYS = {"text", "stopReason", "sessionId", "requestId"}
+IGNORABLE_FENCES = {"```", "```json", "```jsonl"}
+MERGE_CONTRACT = runpy.run_path(pathlib.Path(__file__).with_name("verify-merge-findings.py"))
+VERDICT_RANK = MERGE_CONTRACT["VERDICT_RANK"]
+finding_rank = MERGE_CONTRACT["finding_rank"]
+NARRATIVE_PREAMBLE_BYTES = frozenset(
+    b"\t\n\r !\"$%&'()*+,-./0123456789:;<=>?@"
+    b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz|}~"
+) | frozenset(range(0x80, 0x100))
 
 
 def reject_json_constant(token: str) -> None:
@@ -32,76 +42,242 @@ def atomic_write(path: pathlib.Path, text: str) -> None:
     pathlib.Path(tmp_name).replace(path)
 
 
-def collect(stdout_path: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
+def collect_text(
+    text: str, source: pathlib.Path
+) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
     findings: list[dict[str, Any]] = []
     summary: dict[str, Any] | None = None
-    with stdout_path.open(encoding="utf-8") as handle:
-        for line_no, line in enumerate(handle, 1):
-            raw = line.strip()
-            if not raw:
-                continue
-            if raw.startswith("# SUMMARY "):
-                try:
-                    item = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
-                except ValueError as exc:
-                    raise SystemExit(f"error: invalid SUMMARY JSON at {stdout_path}:{line_no}: {exc}")
-                if not isinstance(item, dict):
-                    raise SystemExit(f"error: SUMMARY is not an object at {stdout_path}:{line_no}")
-                summary = item
-                continue
-            if raw.startswith("#"):
-                continue
+    for line_no, line in enumerate(text.splitlines(), 1):
+        raw = line.strip()
+        if not raw or raw in IGNORABLE_FENCES:
+            continue
+        if raw.startswith("# SUMMARY "):
+            if summary is not None:
+                raise SystemExit(f"error: record after terminal SUMMARY at {source}:{line_no}")
             try:
-                item = loads_strict_json(raw)
+                item = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
             except ValueError as exc:
-                raise SystemExit(f"error: invalid JSONL at {stdout_path}:{line_no}: {exc}")
+                raise SystemExit(f"error: invalid SUMMARY JSON at {source}:{line_no}: {exc}")
             if not isinstance(item, dict):
-                raise SystemExit(f"error: JSONL item is not an object at {stdout_path}:{line_no}")
-            severity = str(item.get("severity") or "").upper()
-            if severity not in FINDING_SEVERITIES:
-                raise SystemExit(f"error: finding missing valid severity at {stdout_path}:{line_no}")
-            findings.append(item)
+                raise SystemExit(f"error: SUMMARY is not an object at {source}:{line_no}")
+            if item.get("verdict") not in VERDICT_RANK:
+                raise SystemExit(f"error: SUMMARY has unknown verdict at {source}:{line_no}")
+            summary = item
+            continue
+        if raw.startswith("#"):
+            continue
+        if summary is not None:
+            raise SystemExit(f"error: record after terminal SUMMARY at {source}:{line_no}")
+        try:
+            item = loads_strict_json(raw)
+        except ValueError as exc:
+            raise SystemExit(f"error: invalid JSONL at {source}:{line_no}: {exc}")
+        if not isinstance(item, dict):
+            raise SystemExit(f"error: JSONL item is not an object at {source}:{line_no}")
+        if set(item) == {"verdict"}:
+            if item["verdict"] not in VERDICT_RANK:
+                raise SystemExit(f"error: SUMMARY has unknown verdict at {source}:{line_no}")
+            summary = item
+            continue
+        severity = str(item.get("severity") or "").upper()
+        if severity not in FINDING_SEVERITIES:
+            raise SystemExit(f"error: finding missing valid severity at {source}:{line_no}")
+        findings.append(item)
+    if summary is not None and summary["verdict"] == "PASS" and any(
+        finding_rank(finding) == 2 for finding in findings
+    ):
+        raise SystemExit("error: verdict-binding finding cannot have a PASS SUMMARY")
     if not findings and (summary is None or summary.get("verdict") != "PASS"):
         raise SystemExit("error: non-PASS SUMMARY without JSONL findings")
     return findings, summary
 
 
+def recover_envelope_text(
+    text: str, source: pathlib.Path
+) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
+    encoded = text.encode("utf-8")
+    for cut, byte in enumerate(encoded[:-1], 1):
+        if byte not in NARRATIVE_PREAMBLE_BYTES:
+            break
+        try:
+            candidate = encoded[cut:].decode("utf-8")
+        except UnicodeDecodeError:
+            continue
+        try:
+            findings, summary = collect_text(candidate, source)
+        except SystemExit:
+            continue
+        if (
+            summary is not None
+            and summary["verdict"] == "NEEDS_WORK"
+            and any(finding_rank(finding) == 2 for finding in findings)
+        ):
+            return findings, summary
+    raise SystemExit("error: no admissible envelope recovery")
+
+
+def collect_stdout(stdout_path: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
+    stdout_text = stdout_path.read_text(encoding="utf-8")
+    try:
+        candidate = loads_strict_json(stdout_text)
+    except ValueError:
+        candidate = None
+    if (
+        not isinstance(candidate, dict)
+        or not ENVELOPE_KEYS.issubset(candidate)
+        or "severity" in candidate
+    ):
+        return collect_text(stdout_text, stdout_path)
+
+    stop_reason = candidate["stopReason"]
+    if stop_reason != "EndTurn":
+        raise SystemExit(
+            f"error: envelope stopReason must be 'EndTurn' at {stdout_path}; "
+            f"got {stop_reason!r}"
+        )
+    envelope_text = candidate["text"]
+    if not isinstance(envelope_text, str):
+        raise SystemExit(f"error: envelope text must be a string at {stdout_path}")
+
+    try:
+        return collect_text(envelope_text, stdout_path)
+    except SystemExit as exc:
+        try:
+            return recover_envelope_text(envelope_text, stdout_path)
+        except SystemExit:
+            raise SystemExit(f"error: envelope text rejected: {exc}") from None
+
+
 def self_test() -> int:
     with tempfile.TemporaryDirectory() as tmp:
         root = pathlib.Path(tmp)
-        stdout_path = root / "codex-judge.stdout"
+        stdout_path = root / "pair-judge.stdout"
         out_path = root / "verify.pair.findings.jsonl"
-        summary_path = root / "codex-judge.summary.json"
+        summary_path = root / "pair-judge.summary.json"
+
+        def assert_rejected(text: str, label: str, *message_parts: str) -> None:
+            stdout_path.write_text(text, encoding="utf-8")
+            try:
+                collect_stdout(stdout_path)
+            except SystemExit as exc:
+                message = str(exc)
+                assert all(part in message for part in message_parts)
+            else:
+                raise AssertionError(f"{label} must be rejected")
+
+        plain_finding = {"id": "a", "severity": "HIGH"}
+        plain_summary = {"verdict": "NEEDS_WORK"}
         stdout_path.write_text(
-            json.dumps({"id": "a", "severity": "HIGH"}) + "\n"
-            + '# SUMMARY {"verdict":"NEEDS_WORK"}\n',
+            json.dumps(plain_finding) + "\n"
+            + "# SUMMARY " + json.dumps(plain_summary) + "\n",
             encoding="utf-8",
         )
-        findings, summary = collect(stdout_path)
+        findings, summary = collect_stdout(stdout_path)
+        assert findings == [plain_finding]
+        assert summary == plain_summary
         write_outputs(findings, summary, out_path, summary_path)
         assert out_path.read_text(encoding="utf-8").count("\n") == 1
         assert loads_strict_json(summary_path.read_text(encoding="utf-8"))["verdict"] == "NEEDS_WORK"
-        stdout_path.write_text('{"id":"nan","severity":NaN}\n', encoding="utf-8")
-        try:
-            collect(stdout_path)
-        except SystemExit as exc:
-            assert "invalid JSON numeric constant: NaN" in str(exc)
-        else:
-            raise AssertionError("NaN Codex stdout finding must not normalize")
+        assert_rejected(
+            '{"id":"nan","severity":NaN}\n',
+            "NaN pair-JUDGE stdout finding",
+            "invalid JSON numeric constant: NaN",
+        )
         rejection_cases = (
-            ("", "no SUMMARY line"),
-            ('# SUMMARY {}\n', "empty SUMMARY object"),
-            ('# SUMMARY {"verdict":"pass"}\n', "lowercase pass"),
-            ('# SUMMARY {"verdict":"UNKNOWN"}\n', "unknown verdict"),
+            ("", "no SUMMARY line", "non-PASS SUMMARY without JSONL findings"),
+            ('# SUMMARY {}\n', "empty SUMMARY object", "SUMMARY has unknown verdict"),
+            ('# SUMMARY {"verdict":"pass"}\n', "lowercase pass", "SUMMARY has unknown verdict"),
+            ('# SUMMARY {"verdict":"UNKNOWN"}\n', "unknown verdict", "SUMMARY has unknown verdict"),
         )
-        for text, label in rejection_cases:
-            stdout_path.write_text(text, encoding="utf-8")
-            try:
-                collect(stdout_path)
-            except SystemExit as exc:
-                assert str(exc) == "error: non-PASS SUMMARY without JSONL findings"
-            else:
-                raise AssertionError(f"{label} must not normalize to PASS")
+        for text, label, message in rejection_cases:
+            assert_rejected(
+                text,
+                label,
+                message,
+            )
+
+        def envelope(text: Any, stop_reason: str = "EndTurn") -> str:
+            return json.dumps(
+                {
+                    "text": text,
+                    "stopReason": stop_reason,
+                    "sessionId": "session",
+                    "requestId": "request",
+                }
+            )
+
+        envelope_finding = {
+            "id": "envelope-finding",
+            "severity": "HIGH",
+            "detail": {"preserved": True},
+        }
+        envelope_summary = {"verdict": "NEEDS_WORK"}
+        stdout_path.write_text(
+            envelope(
+                json.dumps(envelope_finding)
+                + "\n# SUMMARY "
+                + json.dumps(envelope_summary)
+                + "\n"
+            ),
+            encoding="utf-8",
+        )
+        findings, summary = collect_stdout(stdout_path)
+        assert findings == [envelope_finding]
+        assert summary == envelope_summary
+
+        assert_rejected(
+            envelope("review completed without findings"),
+            "findings-less envelope text",
+            "error: envelope text rejected: error: invalid JSONL",
+        )
+        assert_rejected(
+            envelope('{"findings":[],"verdict":"PASS"}', "Cancelled"),
+            "Cancelled envelope",
+            "envelope stopReason must be 'EndTurn'",
+            "'Cancelled'",
+        )
+        assert_rejected(
+            envelope("", "UnknownStop"),
+            "unknown envelope stopReason",
+            "envelope stopReason must be 'EndTurn'",
+            "'UnknownStop'",
+        )
+        assert_rejected(
+            envelope(None),
+            "non-string envelope text",
+            "error: envelope text must be a string",
+        )
+        welded = (
+            "I will inspect the result."
+            + json.dumps(envelope_finding)
+            + "\n# SUMMARY "
+            + json.dumps(envelope_summary)
+            + "\n"
+        )
+        stdout_path.write_text(envelope(welded), encoding="utf-8")
+        findings, summary = collect_stdout(stdout_path)
+        assert findings == [envelope_finding]
+        assert summary == envelope_summary
+        dual_document = (
+            json.dumps(envelope_finding)
+            + json.dumps({"id": "second", "severity": "LOW"})
+            + "\n# SUMMARY "
+            + json.dumps(envelope_summary)
+            + "\n"
+        )
+        assert_rejected(
+            envelope(dual_document),
+            "concatenated envelope documents",
+            "error: envelope text rejected: error: invalid JSONL",
+        )
+        clean_summary = {"verdict": "PASS"}
+        stdout_path.write_text(
+            envelope("# SUMMARY " + json.dumps(clean_summary) + "\n"),
+            encoding="utf-8",
+        )
+        findings, summary = collect_stdout(stdout_path)
+        assert findings == []
+        assert summary == clean_summary
     return 0
 
 
@@ -122,9 +298,9 @@ def write_outputs(
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--devlyn-dir", default=".devlyn")
-    parser.add_argument("--stdout-file", default="codex-judge.stdout")
+    parser.add_argument("--stdout-file", default="pair-judge.stdout")
     parser.add_argument("--out", default="verify.pair.findings.jsonl")
-    parser.add_argument("--summary-out", default="codex-judge.summary.json")
+    parser.add_argument("--summary-out", default="pair-judge.summary.json")
     parser.add_argument("--self-test", action="store_true")
     args = parser.parse_args()
     if args.self_test:
@@ -135,7 +311,7 @@ def main() -> int:
     if not stdout_path.is_file():
         sys.stderr.write(f"error: {stdout_path} not found\n")
         return 1
-    findings, summary = collect(stdout_path)
+    findings, summary = collect_stdout(stdout_path)
     write_outputs(findings, summary, devlyn / args.out, devlyn / args.summary_out)
     print(json.dumps({"findings_count": len(findings), "summary": summary}, sort_keys=True))
     return 0

package/config/skills/_shared/expected.schema.json

@@ -24,6 +24,13 @@
             "description": "Required exit code. Default 0 if omitted.",
             "default": 0
           },
+          "timeout_sec": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 600,
+            "description": "Per-command timeout in seconds. Default 60 if omitted; maximum 600.",
+            "default": 60
+          },
           "stdout_contains": {
             "type": "array",
             "description": "Each substring must appear verbatim in (stdout + stderr) for pass.",

package/config/skills/_shared/grok-anchor-guard.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Deny pair-judge shell outside the probe anchor, with a model-readable reason.
+
+Registered as a native grok `PreToolUse` hook. It fires at step 1 of the
+authorization pipeline, before the permission rules and before the mode policy
+whose auto-deny silently terminates the review at exit 0 (iter-0081).
+
+The anchor is supplied via DEVLYN_PROBE_ANCHOR, derived by the orchestrator from
+the same source as the --allow rule.
+
+Measured (iter-0081 M-C6 v2, external isolation, admissible cell): Line A 6/6 —
+the denial arrives as a model-visible tool_result, the judge re-routes to
+read_file, the anchor still executes inside the grok process, EndTurn with zero
+PermissionCancelled. Line A-chain 1/1 — `<anchor> && <write>` is denied
+model-visibly, neither segment executes, and the judge then runs the bare anchor.
+"""
+import json
+import os
+import sys
+
+REASON = (
+    "DEVLYN-0081-HOOK-DENY: this shell command is outside the permitted probe "
+    "anchor. Use the read_file tool for file reads. The only permitted shell "
+    "command is the anchor named in your prompt, unchained."
+)
+
+SHELL_TOOLS = {"run_terminal_command", "run_terminal_cmd", "Bash"}
+
+# A conservative, quote-OBLIVIOUS lexical veto — deliberately not a shell parser.
+# It rejects these characters even inside quotes, over-denying rather than ever
+# under-denying a chain. Adding quote-, escape-, or operator-aware exceptions is
+# how this becomes a parser; that is a registered falsifier, not a polish.
+FORBIDDEN = set(";&|`$()<>") | {"\n", "\r"}
+
+
+def main():
+    anchor = os.environ.get("DEVLYN_PROBE_ANCHOR", "")
+    payload = json.load(sys.stdin)
+
+    if payload.get("toolName") not in SHELL_TOOLS or not anchor:
+        print(json.dumps({"decision": "allow"}))
+        return
+
+    # The anchor, or the anchor plus argv, and nothing that could start a second
+    # command. A bare prefix rule would allow `<anchor> && <other>` — the vendor's
+    # own documented footgun. Exact match would instead deny `<anchor> --n 5`,
+    # mechanically destroying the "bounded input variations" that verify.md
+    # requires of every engine. Both were rejected by both seats (iter-0081 v2).
+    command = (payload.get("toolInput") or {}).get("command", "").strip()
+    if command == anchor or (
+        command.startswith(anchor + " ")
+        and not any(ch in FORBIDDEN for ch in command)
+    ):
+        print(json.dumps({"decision": "allow"}))
+        return
+
+    print(json.dumps({"decision": "deny", "reason": REASON}))
+
+
+if __name__ == "__main__":
+    main()

package/config/skills/_shared/phase-prompt-render.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Render exact phase-prompt bytes and print their SHA-256 digest."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import pathlib
+import re
+import tempfile
+
+
+EXCLUDED_ADAPTER_SECTION = re.compile(
+    rb"^## (?:Role eligibility|Invocation)(?:\r?\n|\Z).*?(?=^## |\Z)",
+    re.MULTILINE | re.DOTALL,
+)
+EXCLUDED_ADAPTER_HEADING = re.compile(
+    rb"^## (?:Role eligibility|Invocation)\r?$", re.MULTILINE
+)
+
+
+def project_adapter(content: bytes) -> bytes:
+    projected = EXCLUDED_ADAPTER_SECTION.sub(b"", content)
+    assert EXCLUDED_ADAPTER_HEADING.search(projected) is None
+    return projected
+
+
+def validate_plan_context(task_context: pathlib.Path, content: bytes) -> None:
+    if task_context.name != "plan.task-context":
+        return
+    working_directory = pathlib.Path.cwd()
+    expected = (
+        b"Working directory: "
+        + os.fsencode(working_directory)
+        + b"\nPlan output: "
+        + os.fsencode(working_directory / ".devlyn" / "plan.md")
+        + b"\n"
+    )
+    if not content.startswith(expected):
+        raise SystemExit("error: invalid PLAN task-context header")
+
+
+def render_prompt(
+    adapter: pathlib.Path,
+    canonical_body: pathlib.Path,
+    task_context: pathlib.Path,
+    output: pathlib.Path,
+) -> str:
+    try:
+        adapter_bytes = adapter.read_bytes()
+        body_bytes = canonical_body.read_bytes()
+        context_bytes = task_context.read_bytes()
+    except OSError as exc:
+        raise SystemExit(f"error: prompt input unreadable: {exc}") from exc
+    projected_adapter = project_adapter(adapter_bytes)
+    validate_plan_context(task_context, context_bytes)
+    rendered = projected_adapter + body_bytes + context_bytes
+    rendered = rendered.rstrip(b"\n")
+    if not output.parent.is_dir():
+        raise SystemExit(f"error: prompt output parent is not a directory: {output.parent}")
+    fd, temporary = tempfile.mkstemp(dir=output.parent, prefix=output.name + ".tmp.")
+    temporary_path = pathlib.Path(temporary)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(rendered)
+            handle.flush()
+            os.fsync(handle.fileno())
+        temporary_path.replace(output)
+    except BaseException:
+        temporary_path.unlink(missing_ok=True)
+        raise
+    return hashlib.sha256(rendered).hexdigest()
+
+
+def self_test() -> int:
+    with tempfile.TemporaryDirectory(prefix="phase-prompt-render-") as raw:
+        root = pathlib.Path(raw)
+        adapter = root / "adapter.md"
+        body = root / "plan.md"
+        context = root / "plan.task-context"
+        output = root / ".devlyn" / "plan.prompt"
+        output.parent.mkdir()
+        original_directory = pathlib.Path.cwd()
+        os.chdir(root)
+        try:
+            working_directory = pathlib.Path.cwd()
+            adapter_bytes = (
+                b"# Claude adapter\r\n"
+                b"## Identity\r\nkept-identity-\xff\r\n"
+                b"## Role eligibility\nremove-role\r\n"
+                b"## Output discipline\r\nkept-output-\x80\n"
+                b"## Invocation\r\nremove-invocation\n"
+                b"## Anti-patterns\nkept-tail-without-newline"
+            )
+            projected_adapter = (
+                b"# Claude adapter\r\n"
+                b"## Identity\r\nkept-identity-\xff\r\n"
+                b"## Output discipline\r\nkept-output-\x80\n"
+                b"## Anti-patterns\nkept-tail-without-newline"
+            )
+            for metadata_free in (
+                b"# Codex adapter\r\n## Identity\nkept-\xfe",
+                b"# omp adapter\n## Output discipline\r\nkept\x81",
+            ):
+                assert project_adapter(metadata_free) == metadata_free
+
+            body_bytes = b"# PHASE 1 \xe2\x80\x94 PLAN\n"
+            context_bytes = (
+                b"Working directory: "
+                + os.fsencode(working_directory)
+                + b"\nPlan output: "
+                + os.fsencode(working_directory / ".devlyn" / "plan.md")
+                + b"\ncontext-without-newline"
+            )
+            for path, content in (
+                (adapter, adapter_bytes),
+                (body, body_bytes),
+                (context, context_bytes),
+            ):
+                path.write_bytes(content)
+            expected = projected_adapter + body_bytes + context_bytes
+            digest = render_prompt(adapter, body, context, output)
+            assert output.read_bytes() == expected
+            assert digest == hashlib.sha256(expected).hexdigest()
+            assert render_prompt(adapter, body, context, output) == digest
+            assert output.read_bytes() == expected
+
+            context_prefix = context_bytes.removesuffix(b"context-without-newline")
+            for terminal_lfs in (b"", b"\n", b"\n\n"):
+                case_context = context_prefix + b"context-terminal-lf-case" + terminal_lfs
+                context.write_bytes(case_context)
+                expected = (projected_adapter + body_bytes + case_context).rstrip(b"\n")
+                digest = render_prompt(adapter, body, context, output)
+                written = output.read_bytes()
+                assert not written.endswith(b"\n")
+                assert digest == hashlib.sha256(written).hexdigest()
+                assert written == expected
+
+            invalid_contexts = (
+                b"context-without-header",
+                b"Working directory: relative\nPlan output: relative/.devlyn/plan.md\n",
+                b"Working directory: /mismatch\nPlan output: /mismatch/.devlyn/plan.md\n",
+            )
+            for invalid_context in invalid_contexts:
+                context.write_bytes(invalid_context)
+                output.write_bytes(b"unchanged")
+                try:
+                    render_prompt(adapter, body, context, output)
+                except SystemExit:
+                    pass
+                else:
+                    raise AssertionError("invalid PLAN task-context header accepted")
+                assert output.read_bytes() == b"unchanged"
+        finally:
+            os.chdir(original_directory)
+    print("SELFTEST PASS: projected exact bytes + PLAN context validation")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--adapter", type=pathlib.Path)
+    parser.add_argument("--canonical-body", type=pathlib.Path)
+    parser.add_argument("--task-context", type=pathlib.Path)
+    parser.add_argument("--output", type=pathlib.Path)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+    if args.self_test:
+        if any((args.adapter, args.canonical_body, args.task_context, args.output)):
+            parser.error("render paths are not allowed with --self-test")
+        return self_test()
+    if not all((args.adapter, args.canonical_body, args.task_context, args.output)):
+        parser.error(
+            "--adapter, --canonical-body, --task-context, and --output are required"
+        )
+    print(render_prompt(args.adapter, args.canonical_body, args.task_context, args.output))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())