devlyn-cli 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/README.md +2 -2
- package/benchmark/auto-resolve/scripts/run-fixture.sh +11 -1
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +83 -1
- package/bin/devlyn.js +53 -8
- package/config/skills/_shared/adapters/grok.md +34 -12
- package/config/skills/_shared/collect-codex-findings.py +228 -52
- package/config/skills/_shared/expected.schema.json +7 -0
- package/config/skills/_shared/grok-anchor-guard.py +61 -0
- package/config/skills/_shared/phase-prompt-render.py +181 -0
- package/config/skills/_shared/spec-verify-check.py +308 -31
- package/config/skills/_shared/state-phase-write.py +720 -32
- package/config/skills/_shared/verify-merge-findings.py +170 -0
- package/config/skills/devlyn:engines/SKILL.md +1 -1
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:resolve/SKILL.md +7 -6
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +2 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/plan.md +1 -5
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +4 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +5 -1
- package/config/skills/devlyn:resolve/references/state-schema.md +2 -2
- package/package.json +1 -1
- package/scripts/lint-fixtures.sh +4 -1
- package/scripts/lint-skills.sh +138 -10
|
@@ -260,13 +260,19 @@ EXPECTED_TOP_LEVEL_KEYS = {
|
|
|
260
260
|
EXPECTED_VERIFICATION_COMMAND_KEYS = {
|
|
261
261
|
"cmd",
|
|
262
262
|
"exit_code",
|
|
263
|
+
"timeout_sec",
|
|
263
264
|
"stdout_contains",
|
|
264
265
|
"stdout_not_contains",
|
|
265
266
|
"contract_refs",
|
|
266
267
|
}
|
|
268
|
+
DEFAULT_TIMEOUT_SEC = 60
|
|
267
269
|
SPEC_COMPLEXITY_VALUES = {"trivial", "medium", "high", "large"}
|
|
268
270
|
|
|
269
271
|
|
|
272
|
+
def verification_timeout_sec(command: dict) -> int:
|
|
273
|
+
return command.get("timeout_sec", DEFAULT_TIMEOUT_SEC)
|
|
274
|
+
|
|
275
|
+
|
|
270
276
|
def extract_verification_block(text: str) -> tuple[bool, str | None]:
|
|
271
277
|
"""Locate the verification section via the `<!-- devlyn:verification -->`
|
|
272
278
|
sentinel (language-neutral — the human-readable heading after it may be
|
|
@@ -579,10 +585,12 @@ def validate_shape(data) -> str | None:
|
|
|
579
585
|
Schema (iter-0019.8): top-level object with a non-empty
|
|
580
586
|
`verification_commands` list of objects. Each object requires a
|
|
581
587
|
non-empty string `cmd`; `exit_code` defaults to 0 and must be a
|
|
582
|
-
non-bool int; `
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
588
|
+
non-bool int; `timeout_sec` defaults to DEFAULT_TIMEOUT_SEC and must be a
|
|
589
|
+
non-bool int from 1 through 600; `stdout_contains` and
|
|
590
|
+
`stdout_not_contains` default to empty list and must be lists of strings.
|
|
591
|
+
Bool is rejected explicitly because Python's `bool` subclasses `int` —
|
|
592
|
+
`isinstance(True, int) is True` would otherwise let numeric fields accept
|
|
593
|
+
true.
|
|
586
594
|
"""
|
|
587
595
|
if not isinstance(data, dict):
|
|
588
596
|
return "top-level must be a JSON object"
|
|
@@ -600,6 +608,13 @@ def validate_shape(data) -> str | None:
|
|
|
600
608
|
ec = c.get("exit_code", 0)
|
|
601
609
|
if isinstance(ec, bool) or not isinstance(ec, int):
|
|
602
610
|
return f"verification_commands[{i}].exit_code must be int (not bool)"
|
|
611
|
+
timeout_sec = c.get("timeout_sec", DEFAULT_TIMEOUT_SEC)
|
|
612
|
+
if (
|
|
613
|
+
isinstance(timeout_sec, bool)
|
|
614
|
+
or not isinstance(timeout_sec, int)
|
|
615
|
+
or not 1 <= timeout_sec <= 600
|
|
616
|
+
):
|
|
617
|
+
return f"verification_commands[{i}].timeout_sec must be int from 1 to 600 (not bool)"
|
|
603
618
|
for k in ("stdout_contains", "stdout_not_contains"):
|
|
604
619
|
v = c.get(k, [])
|
|
605
620
|
if not isinstance(v, list) or not all(isinstance(s, str) for s in v):
|
|
@@ -810,7 +825,7 @@ def validate_risk_probe(
|
|
|
810
825
|
if not isinstance(actual, list) or not all(isinstance(item, str) for item in actual):
|
|
811
826
|
return f"risk-probes[{index}].tag_evidence.shape_contract must be a list of strings"
|
|
812
827
|
required_shape = set(SHAPE_CONTRACT_REQUIRED_EVIDENCE)
|
|
813
|
-
if
|
|
828
|
+
if "visible_text_names_exact_json_error_object" in set(actual):
|
|
814
829
|
required_shape.add("asserts_exact_error_object")
|
|
815
830
|
missing_shape = sorted(required_shape - set(actual))
|
|
816
831
|
if missing_shape:
|
|
@@ -1738,6 +1753,13 @@ def run_check_expected_mode(expected_path: Path) -> int:
|
|
|
1738
1753
|
|
|
1739
1754
|
|
|
1740
1755
|
def run_self_test() -> int:
|
|
1756
|
+
# Hermeticity: a pipeline replay exports BENCH_WORKDIR at the live repo;
|
|
1757
|
+
# inherited into scenario children it wins over their tmp cwd (the
|
|
1758
|
+
# default-mode work resolution below) and re-executes the LIVE
|
|
1759
|
+
# .devlyn/spec-verify.json — with lint as the live command this mutually
|
|
1760
|
+
# recursed (lint runs this self-test) into an unbounded spawn storm
|
|
1761
|
+
# (observed 2026-08-03, iter-0089 canary).
|
|
1762
|
+
os.environ.pop("BENCH_WORKDIR", None)
|
|
1741
1763
|
script_path = str(Path(__file__).resolve())
|
|
1742
1764
|
with tempfile.TemporaryDirectory() as td:
|
|
1743
1765
|
work = Path(td)
|
|
@@ -1762,6 +1784,7 @@ def run_self_test() -> int:
|
|
|
1762
1784
|
"derived_from": "probe must pass visible marker.",
|
|
1763
1785
|
"cmd": "python3 .devlyn/probes/P1.py",
|
|
1764
1786
|
"exit_code": 0,
|
|
1787
|
+
"timeout_sec": 5,
|
|
1765
1788
|
"stdout_contains": ["probe-ok"],
|
|
1766
1789
|
"stdout_not_contains": [],
|
|
1767
1790
|
"tags": ["shape_contract"],
|
|
@@ -1774,6 +1797,211 @@ def run_self_test() -> int:
|
|
|
1774
1797
|
},
|
|
1775
1798
|
}
|
|
1776
1799
|
(devlyn / "risk-probes.jsonl").write_text(json.dumps(risk_probe_payload) + "\n")
|
|
1800
|
+
loaded_probes, loaded_probe_error = load_risk_probes(
|
|
1801
|
+
devlyn, spec_md, require_present=True
|
|
1802
|
+
)
|
|
1803
|
+
if loaded_probe_error or loaded_probes[0].get("timeout_sec") != 5:
|
|
1804
|
+
print("risk-probe timeout_sec was not preserved", file=sys.stderr)
|
|
1805
|
+
print(loaded_probe_error, file=sys.stderr)
|
|
1806
|
+
return 1
|
|
1807
|
+
|
|
1808
|
+
if verification_timeout_sec({"cmd": "printf default"}) != DEFAULT_TIMEOUT_SEC:
|
|
1809
|
+
print("absent timeout_sec did not resolve to DEFAULT_TIMEOUT_SEC", file=sys.stderr)
|
|
1810
|
+
return 1
|
|
1811
|
+
|
|
1812
|
+
inline_timeout_spec = work / "inline-timeout-spec.md"
|
|
1813
|
+
inline_timeout_devlyn = work / "inline-timeout-devlyn"
|
|
1814
|
+
for invalid_timeout in (True, 0, -1, 601, "60"):
|
|
1815
|
+
inline_timeout_spec.write_text(
|
|
1816
|
+
"# Timeout validation\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
1817
|
+
+ json.dumps({
|
|
1818
|
+
"verification_commands": [
|
|
1819
|
+
{"cmd": "printf ok", "timeout_sec": invalid_timeout}
|
|
1820
|
+
]
|
|
1821
|
+
})
|
|
1822
|
+
+ "\n```\n",
|
|
1823
|
+
encoding="utf-8",
|
|
1824
|
+
)
|
|
1825
|
+
staged, inline_error = stage_from_source(
|
|
1826
|
+
inline_timeout_spec, inline_timeout_devlyn
|
|
1827
|
+
)
|
|
1828
|
+
if staged or not inline_error or "timeout_sec" not in inline_error:
|
|
1829
|
+
print(
|
|
1830
|
+
f"inline carrier accepted invalid timeout_sec={invalid_timeout!r}",
|
|
1831
|
+
file=sys.stderr,
|
|
1832
|
+
)
|
|
1833
|
+
return 1
|
|
1834
|
+
|
|
1835
|
+
inline_timeout_spec.write_text(
|
|
1836
|
+
"# Timeout preservation\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
1837
|
+
+ json.dumps({
|
|
1838
|
+
"verification_commands": [
|
|
1839
|
+
{"cmd": "printf ok", "timeout_sec": 5}
|
|
1840
|
+
]
|
|
1841
|
+
})
|
|
1842
|
+
+ "\n```\n",
|
|
1843
|
+
encoding="utf-8",
|
|
1844
|
+
)
|
|
1845
|
+
staged, inline_error = stage_from_source(
|
|
1846
|
+
inline_timeout_spec, inline_timeout_devlyn
|
|
1847
|
+
)
|
|
1848
|
+
inline_staged = loads_strict_json(
|
|
1849
|
+
(inline_timeout_devlyn / "spec-verify.json").read_text()
|
|
1850
|
+
)
|
|
1851
|
+
if (
|
|
1852
|
+
not staged
|
|
1853
|
+
or inline_error
|
|
1854
|
+
or inline_staged["verification_commands"][0].get("timeout_sec") != 5
|
|
1855
|
+
):
|
|
1856
|
+
print("inline carrier did not preserve timeout_sec", file=sys.stderr)
|
|
1857
|
+
return 1
|
|
1858
|
+
|
|
1859
|
+
sibling_timeout_root = work / "sibling-timeout"
|
|
1860
|
+
sibling_timeout_root.mkdir()
|
|
1861
|
+
sibling_timeout_devlyn = sibling_timeout_root / ".devlyn"
|
|
1862
|
+
sibling_timeout_spec = sibling_timeout_root / "spec.md"
|
|
1863
|
+
sibling_timeout_spec.write_text("# Sibling timeout\n", encoding="utf-8")
|
|
1864
|
+
sibling_timeout_expected = sibling_timeout_root / "spec.expected.json"
|
|
1865
|
+
for invalid_timeout in (True, 601):
|
|
1866
|
+
sibling_timeout_expected.write_text(json.dumps({
|
|
1867
|
+
"verification_commands": [
|
|
1868
|
+
{"cmd": "printf ok", "timeout_sec": invalid_timeout}
|
|
1869
|
+
]
|
|
1870
|
+
}) + "\n")
|
|
1871
|
+
found, staged, sibling_error, _path, _data = stage_from_expected(
|
|
1872
|
+
sibling_timeout_spec, sibling_timeout_devlyn
|
|
1873
|
+
)
|
|
1874
|
+
if not found or staged or not sibling_error or "timeout_sec" not in sibling_error:
|
|
1875
|
+
print(
|
|
1876
|
+
f"sibling carrier accepted invalid timeout_sec={invalid_timeout!r}",
|
|
1877
|
+
file=sys.stderr,
|
|
1878
|
+
)
|
|
1879
|
+
return 1
|
|
1880
|
+
|
|
1881
|
+
sibling_timeout_expected.write_text(json.dumps({
|
|
1882
|
+
"verification_commands": [
|
|
1883
|
+
{"cmd": "printf ok", "timeout_sec": 5}
|
|
1884
|
+
]
|
|
1885
|
+
}) + "\n")
|
|
1886
|
+
found, staged, sibling_error, _path, _data = stage_from_expected(
|
|
1887
|
+
sibling_timeout_spec, sibling_timeout_devlyn
|
|
1888
|
+
)
|
|
1889
|
+
sibling_staged = loads_strict_json(
|
|
1890
|
+
(sibling_timeout_devlyn / "spec-verify.json").read_text()
|
|
1891
|
+
)
|
|
1892
|
+
if (
|
|
1893
|
+
not found
|
|
1894
|
+
or not staged
|
|
1895
|
+
or sibling_error
|
|
1896
|
+
or sibling_staged["verification_commands"][0].get("timeout_sec") != 5
|
|
1897
|
+
):
|
|
1898
|
+
print("sibling carrier did not preserve timeout_sec", file=sys.stderr)
|
|
1899
|
+
return 1
|
|
1900
|
+
|
|
1901
|
+
timeout_run_root = work / "timeout-run"
|
|
1902
|
+
timeout_run_root.mkdir()
|
|
1903
|
+
timeout_run_devlyn = timeout_run_root / ".devlyn"
|
|
1904
|
+
timeout_run_devlyn.mkdir()
|
|
1905
|
+
timeout_run_spec = timeout_run_root / "spec.md"
|
|
1906
|
+
timeout_run_spec.write_text(
|
|
1907
|
+
"# Timeout execution\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
1908
|
+
+ json.dumps({
|
|
1909
|
+
"verification_commands": [
|
|
1910
|
+
{
|
|
1911
|
+
"cmd": "python3 -c \"import time; time.sleep(2)\"",
|
|
1912
|
+
"timeout_sec": 1,
|
|
1913
|
+
},
|
|
1914
|
+
{
|
|
1915
|
+
"cmd": "python3 -c \"import time; time.sleep(1)\"",
|
|
1916
|
+
"timeout_sec": 5,
|
|
1917
|
+
},
|
|
1918
|
+
]
|
|
1919
|
+
})
|
|
1920
|
+
+ "\n```\n",
|
|
1921
|
+
encoding="utf-8",
|
|
1922
|
+
)
|
|
1923
|
+
(timeout_run_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
1924
|
+
"source": {"type": "spec", "spec_path": str(timeout_run_spec)}
|
|
1925
|
+
}))
|
|
1926
|
+
timeout_run = subprocess.run(
|
|
1927
|
+
[sys.executable, script_path],
|
|
1928
|
+
cwd=timeout_run_root,
|
|
1929
|
+
capture_output=True,
|
|
1930
|
+
text=True,
|
|
1931
|
+
)
|
|
1932
|
+
if timeout_run.returncode == 0:
|
|
1933
|
+
print("declared one-second timeout did not fail", file=sys.stderr)
|
|
1934
|
+
return 1
|
|
1935
|
+
timeout_results = loads_strict_json(
|
|
1936
|
+
(timeout_run_devlyn / "spec-verify.results.json").read_text()
|
|
1937
|
+
)["commands"]
|
|
1938
|
+
timeout_findings = [
|
|
1939
|
+
loads_strict_json(line)
|
|
1940
|
+
for line in (timeout_run_devlyn / output_findings_name()).read_text().splitlines()
|
|
1941
|
+
if line.strip()
|
|
1942
|
+
]
|
|
1943
|
+
if (
|
|
1944
|
+
timeout_results[0].get("reason") != "timeout"
|
|
1945
|
+
or timeout_results[0].get("timeout_sec") != 1
|
|
1946
|
+
or not timeout_results[1].get("pass")
|
|
1947
|
+
):
|
|
1948
|
+
print("declared timeout budgets were not honored", file=sys.stderr)
|
|
1949
|
+
print(timeout_results, file=sys.stderr)
|
|
1950
|
+
return 1
|
|
1951
|
+
timeout_finding = timeout_findings[0] if timeout_findings else {}
|
|
1952
|
+
if (
|
|
1953
|
+
timeout_finding.get("rule_id") != "correctness.verification-timeout"
|
|
1954
|
+
or "after 1s" not in timeout_finding.get("message", "")
|
|
1955
|
+
or "timeout_sec" not in timeout_finding.get("message", "")
|
|
1956
|
+
or "600" not in timeout_finding.get("fix_hint", "")
|
|
1957
|
+
):
|
|
1958
|
+
print("timeout finding did not carry the distinct budget contract", file=sys.stderr)
|
|
1959
|
+
print(timeout_finding, file=sys.stderr)
|
|
1960
|
+
return 1
|
|
1961
|
+
|
|
1962
|
+
runner_env_root = work / "runner-env-isolation"
|
|
1963
|
+
runner_env_root.mkdir()
|
|
1964
|
+
runner_env_devlyn = runner_env_root / ".devlyn"
|
|
1965
|
+
runner_env_devlyn.mkdir()
|
|
1966
|
+
runner_env_spec = runner_env_root / "spec.md"
|
|
1967
|
+
runner_env_spec.write_text(
|
|
1968
|
+
"# Runner environment isolation\n\n<!-- devlyn:verification -->\n"
|
|
1969
|
+
"## Verification\n\n```json\n"
|
|
1970
|
+
+ json.dumps({
|
|
1971
|
+
"verification_commands": [{
|
|
1972
|
+
"cmd": (
|
|
1973
|
+
"python3 -c \"import os, sys; sys.exit(any(name in os.environ "
|
|
1974
|
+
"for name in ('SPEC_VERIFY_PHASE', 'SPEC_VERIFY_FINDINGS_FILE', "
|
|
1975
|
+
"'SPEC_VERIFY_FINDING_PREFIX')))\""
|
|
1976
|
+
)
|
|
1977
|
+
}]
|
|
1978
|
+
})
|
|
1979
|
+
+ "\n```\n",
|
|
1980
|
+
encoding="utf-8",
|
|
1981
|
+
)
|
|
1982
|
+
(runner_env_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
1983
|
+
"source": {"type": "spec", "spec_path": str(runner_env_spec)}
|
|
1984
|
+
}))
|
|
1985
|
+
runner_env = os.environ.copy()
|
|
1986
|
+
runner_env.update({
|
|
1987
|
+
"SPEC_VERIFY_PHASE": "verify_mechanical",
|
|
1988
|
+
"SPEC_VERIFY_FINDINGS_FILE": "runner-env.findings.jsonl",
|
|
1989
|
+
"SPEC_VERIFY_FINDING_PREFIX": "RUNNER-ENV",
|
|
1990
|
+
})
|
|
1991
|
+
runner_env_run = subprocess.run(
|
|
1992
|
+
[sys.executable, script_path],
|
|
1993
|
+
cwd=runner_env_root,
|
|
1994
|
+
env=runner_env,
|
|
1995
|
+
capture_output=True,
|
|
1996
|
+
text=True,
|
|
1997
|
+
)
|
|
1998
|
+
runner_env_results = loads_strict_json(
|
|
1999
|
+
(runner_env_devlyn / "spec-verify.results.json").read_text()
|
|
2000
|
+
)["commands"]
|
|
2001
|
+
if runner_env_run.returncode != 0 or not runner_env_results[0].get("pass"):
|
|
2002
|
+
print("runner-directed environment leaked into a verification command", file=sys.stderr)
|
|
2003
|
+
print(runner_env_run.stderr, file=sys.stderr)
|
|
2004
|
+
return 1
|
|
1777
2005
|
env = os.environ.copy()
|
|
1778
2006
|
env["BENCH_WORKDIR"] = str(work)
|
|
1779
2007
|
validate_without_digest = subprocess.run(
|
|
@@ -1830,7 +2058,7 @@ def run_self_test() -> int:
|
|
|
1830
2058
|
if mutated_script.returncode == 0:
|
|
1831
2059
|
print("--include-risk-probes accepted mutated probe script bytes", file=sys.stderr)
|
|
1832
2060
|
return 1
|
|
1833
|
-
integrity_findings = (devlyn /
|
|
2061
|
+
integrity_findings = (devlyn / output_findings_name()).read_text(encoding="utf-8")
|
|
1834
2062
|
if "correctness.risk-probe-integrity" not in integrity_findings:
|
|
1835
2063
|
print("mutated probe script did not emit correctness.risk-probe-integrity", file=sys.stderr)
|
|
1836
2064
|
print(integrity_findings, file=sys.stderr)
|
|
@@ -1850,7 +2078,7 @@ def run_self_test() -> int:
|
|
|
1850
2078
|
if mutated_jsonl.returncode == 0:
|
|
1851
2079
|
print("--include-risk-probes accepted mutated risk-probes.jsonl bytes", file=sys.stderr)
|
|
1852
2080
|
return 1
|
|
1853
|
-
integrity_findings = (devlyn /
|
|
2081
|
+
integrity_findings = (devlyn / output_findings_name()).read_text(encoding="utf-8")
|
|
1854
2082
|
if "correctness.risk-probe-integrity" not in integrity_findings:
|
|
1855
2083
|
print("mutated risk-probes.jsonl did not emit correctness.risk-probe-integrity", file=sys.stderr)
|
|
1856
2084
|
print(integrity_findings, file=sys.stderr)
|
|
@@ -1871,7 +2099,7 @@ def run_self_test() -> int:
|
|
|
1871
2099
|
if missing_digest.returncode == 0:
|
|
1872
2100
|
print("--include-risk-probes accepted enabled risk probes with missing digest", file=sys.stderr)
|
|
1873
2101
|
return 1
|
|
1874
|
-
integrity_findings = (devlyn /
|
|
2102
|
+
integrity_findings = (devlyn / output_findings_name()).read_text(encoding="utf-8")
|
|
1875
2103
|
if "correctness.risk-probe-integrity" not in integrity_findings:
|
|
1876
2104
|
print("missing risk_probes_digest did not emit correctness.risk-probe-integrity", file=sys.stderr)
|
|
1877
2105
|
print(integrity_findings, file=sys.stderr)
|
|
@@ -3132,7 +3360,7 @@ def run_self_test() -> int:
|
|
|
3132
3360
|
if contract_run.returncode == 0:
|
|
3133
3361
|
print("expected contract violations were accepted", file=sys.stderr)
|
|
3134
3362
|
return 1
|
|
3135
|
-
findings_text = (contract_devlyn /
|
|
3363
|
+
findings_text = (contract_devlyn / output_findings_name()).read_text()
|
|
3136
3364
|
for rule_id in (
|
|
3137
3365
|
"correctness.forbidden-pattern",
|
|
3138
3366
|
"correctness.required-file-missing",
|
|
@@ -3169,7 +3397,8 @@ def run_self_test() -> int:
|
|
|
3169
3397
|
error_spec = error_root / "spec.md"
|
|
3170
3398
|
error_spec.write_text(
|
|
3171
3399
|
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3172
|
-
"- Invalid input must print
|
|
3400
|
+
"- Invalid input must print JSON error object `{ \"error\": \"bad_input\" }` to stderr and exit 2.\n"
|
|
3401
|
+
"- Malformed input must exit 2 and print stderr JSON with keys `code` and `detail`; values are implementation-defined.\n"
|
|
3173
3402
|
)
|
|
3174
3403
|
(error_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
3175
3404
|
"source": {"type": "spec", "spec_path": str(error_spec)}
|
|
@@ -3221,7 +3450,7 @@ def run_self_test() -> int:
|
|
|
3221
3450
|
|
|
3222
3451
|
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3223
3452
|
"id": "P7c",
|
|
3224
|
-
"derived_from": "Invalid input must print
|
|
3453
|
+
"derived_from": "Invalid input must print JSON error object `{ \"error\": \"bad_input\" }` to stderr and exit 2.",
|
|
3225
3454
|
"cmd": "printf json-error-shape-contract-missing-exact",
|
|
3226
3455
|
"exit_code": 2,
|
|
3227
3456
|
"tags": ["stdout_stderr_contract", "error_contract", "shape_contract"],
|
|
@@ -3235,6 +3464,7 @@ def run_self_test() -> int:
|
|
|
3235
3464
|
"uses_visible_input_key_names",
|
|
3236
3465
|
"asserts_visible_output_key_names",
|
|
3237
3466
|
"asserts_no_unexpected_output_keys",
|
|
3467
|
+
"visible_text_names_exact_json_error_object",
|
|
3238
3468
|
],
|
|
3239
3469
|
},
|
|
3240
3470
|
}) + "\n")
|
|
@@ -3246,7 +3476,7 @@ def run_self_test() -> int:
|
|
|
3246
3476
|
)
|
|
3247
3477
|
if missing_exact_error_object.returncode == 0:
|
|
3248
3478
|
print(
|
|
3249
|
-
"shape_contract
|
|
3479
|
+
"shape_contract claiming visible exact error object without "
|
|
3250
3480
|
"asserts_exact_error_object was accepted",
|
|
3251
3481
|
file=sys.stderr,
|
|
3252
3482
|
)
|
|
@@ -3254,7 +3484,7 @@ def run_self_test() -> int:
|
|
|
3254
3484
|
|
|
3255
3485
|
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3256
3486
|
"id": "P7d",
|
|
3257
|
-
"derived_from": "Invalid input must print
|
|
3487
|
+
"derived_from": "Invalid input must print JSON error object `{ \"error\": \"bad_input\" }` to stderr and exit 2.",
|
|
3258
3488
|
"cmd": "printf json-error-shape-contract",
|
|
3259
3489
|
"exit_code": 2,
|
|
3260
3490
|
"tags": ["stdout_stderr_contract", "error_contract", "shape_contract"],
|
|
@@ -3268,6 +3498,7 @@ def run_self_test() -> int:
|
|
|
3268
3498
|
"uses_visible_input_key_names",
|
|
3269
3499
|
"asserts_visible_output_key_names",
|
|
3270
3500
|
"asserts_no_unexpected_output_keys",
|
|
3501
|
+
"visible_text_names_exact_json_error_object",
|
|
3271
3502
|
"asserts_exact_error_object",
|
|
3272
3503
|
],
|
|
3273
3504
|
},
|
|
@@ -3283,6 +3514,38 @@ def run_self_test() -> int:
|
|
|
3283
3514
|
print(strong_json_error_shape_contract.stderr, file=sys.stderr)
|
|
3284
3515
|
return 1
|
|
3285
3516
|
|
|
3517
|
+
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3518
|
+
"id": "P7e",
|
|
3519
|
+
"derived_from": (
|
|
3520
|
+
"Malformed input must exit 2 and print stderr JSON with keys `code` and `detail`; "
|
|
3521
|
+
"values are implementation-defined."
|
|
3522
|
+
),
|
|
3523
|
+
"cmd": "printf error-exit-shape-contract",
|
|
3524
|
+
"exit_code": 2,
|
|
3525
|
+
"tags": ["error_contract", "shape_contract"],
|
|
3526
|
+
"tag_evidence": {
|
|
3527
|
+
"error_contract": [
|
|
3528
|
+
"asserts_error_payload_or_stderr",
|
|
3529
|
+
"asserts_nonzero_or_exit_2",
|
|
3530
|
+
],
|
|
3531
|
+
"shape_contract": [
|
|
3532
|
+
"uses_visible_input_key_names",
|
|
3533
|
+
"asserts_visible_output_key_names",
|
|
3534
|
+
"asserts_no_unexpected_output_keys",
|
|
3535
|
+
],
|
|
3536
|
+
},
|
|
3537
|
+
}) + "\n")
|
|
3538
|
+
error_exit_shape_contract = subprocess.run(
|
|
3539
|
+
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3540
|
+
cwd=error_root,
|
|
3541
|
+
capture_output=True,
|
|
3542
|
+
text=True,
|
|
3543
|
+
)
|
|
3544
|
+
if error_exit_shape_contract.returncode != 0:
|
|
3545
|
+
print("error-exit shape_contract without visible exact error object was rejected", file=sys.stderr)
|
|
3546
|
+
print(error_exit_shape_contract.stderr, file=sys.stderr)
|
|
3547
|
+
return 1
|
|
3548
|
+
|
|
3286
3549
|
http_error_root = work / "http-error-contract-risk-probe"
|
|
3287
3550
|
http_error_root.mkdir()
|
|
3288
3551
|
http_error_devlyn = http_error_root / ".devlyn"
|
|
@@ -3336,6 +3599,7 @@ def run_self_test() -> int:
|
|
|
3336
3599
|
"uses_visible_input_key_names",
|
|
3337
3600
|
"asserts_visible_output_key_names",
|
|
3338
3601
|
"asserts_no_unexpected_output_keys",
|
|
3602
|
+
"visible_text_names_exact_json_error_object",
|
|
3339
3603
|
],
|
|
3340
3604
|
},
|
|
3341
3605
|
}) + "\n")
|
|
@@ -3367,6 +3631,7 @@ def run_self_test() -> int:
|
|
|
3367
3631
|
"uses_visible_input_key_names",
|
|
3368
3632
|
"asserts_visible_output_key_names",
|
|
3369
3633
|
"asserts_no_unexpected_output_keys",
|
|
3634
|
+
"visible_text_names_exact_json_error_object",
|
|
3370
3635
|
"asserts_exact_error_object",
|
|
3371
3636
|
],
|
|
3372
3637
|
},
|
|
@@ -3670,12 +3935,15 @@ def run_self_test() -> int:
|
|
|
3670
3935
|
print("--write-untracked-baseline wrote wrong content", file=sys.stderr)
|
|
3671
3936
|
print(repr(baseline_lines), file=sys.stderr)
|
|
3672
3937
|
return 1
|
|
3673
|
-
scope_findings_path = scope_devlyn /
|
|
3938
|
+
scope_findings_path = scope_devlyn / output_findings_name()
|
|
3939
|
+
scope_build_gate_env = os.environ.copy()
|
|
3940
|
+
scope_build_gate_env["SPEC_VERIFY_PHASE"] = "build_gate"
|
|
3674
3941
|
|
|
3675
3942
|
# Test 1: no .devlyn/plan.md at all -> fail-closed CRITICAL, not a no-op.
|
|
3676
3943
|
(scope_root / "bin" / "cli.js").write_text("module.exports = { ok: true };\n")
|
|
3677
3944
|
missing_plan_run = subprocess.run(
|
|
3678
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
3945
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
3946
|
+
capture_output=True, text=True,
|
|
3679
3947
|
)
|
|
3680
3948
|
if missing_plan_run.returncode == 0:
|
|
3681
3949
|
print("BUILD_GATE accepted a run with no plan.md", file=sys.stderr)
|
|
@@ -3689,7 +3957,8 @@ def run_self_test() -> int:
|
|
|
3689
3957
|
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n- `bin/cli.js` (edit): ship the fix.\n"
|
|
3690
3958
|
)
|
|
3691
3959
|
malformed_block_run = subprocess.run(
|
|
3692
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
3960
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
3961
|
+
capture_output=True, text=True,
|
|
3693
3962
|
)
|
|
3694
3963
|
if malformed_block_run.returncode == 0:
|
|
3695
3964
|
print("BUILD_GATE accepted plan.md with no authorized_surface block", file=sys.stderr)
|
|
@@ -3734,7 +4003,8 @@ def run_self_test() -> int:
|
|
|
3734
4003
|
return 1
|
|
3735
4004
|
(scope_devlyn / "untracked.baseline").unlink()
|
|
3736
4005
|
missing_baseline_run = subprocess.run(
|
|
3737
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
4006
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
4007
|
+
capture_output=True, text=True,
|
|
3738
4008
|
)
|
|
3739
4009
|
if missing_baseline_run.returncode == 0:
|
|
3740
4010
|
print("BUILD_GATE accepted missing .devlyn/untracked.baseline", file=sys.stderr)
|
|
@@ -3750,7 +4020,8 @@ def run_self_test() -> int:
|
|
|
3750
4020
|
print("--write-untracked-baseline re-run failed", file=sys.stderr)
|
|
3751
4021
|
return 1
|
|
3752
4022
|
in_scope_run = subprocess.run(
|
|
3753
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
4023
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
4024
|
+
capture_output=True, text=True,
|
|
3754
4025
|
)
|
|
3755
4026
|
if in_scope_run.returncode != 0:
|
|
3756
4027
|
print("in-scope-only diff was rejected", file=sys.stderr)
|
|
@@ -3767,7 +4038,8 @@ def run_self_test() -> int:
|
|
|
3767
4038
|
(scope_root / "data" / "usage-stats.json").write_text('{"leaked": true}\n')
|
|
3768
4039
|
(scope_root / "data" / "scratch.json").write_text('{"untracked": true}\n')
|
|
3769
4040
|
out_of_scope_run = subprocess.run(
|
|
3770
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
4041
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
4042
|
+
capture_output=True, text=True,
|
|
3771
4043
|
)
|
|
3772
4044
|
if out_of_scope_run.returncode == 0:
|
|
3773
4045
|
print("out-of-scope file was accepted", file=sys.stderr)
|
|
@@ -3803,7 +4075,8 @@ def run_self_test() -> int:
|
|
|
3803
4075
|
# (directory-prefix boundary, not a bare string-prefix match).
|
|
3804
4076
|
(scope_root / "lib2" / "keep.js").write_text("module.exports = { touched: true };\n")
|
|
3805
4077
|
subprocess.run(
|
|
3806
|
-
[sys.executable, script_path], cwd=scope_root,
|
|
4078
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
4079
|
+
capture_output=True, text=True,
|
|
3807
4080
|
)
|
|
3808
4081
|
boundary_flagged = {
|
|
3809
4082
|
loads_strict_json(line)["file"]
|
|
@@ -4100,6 +4373,12 @@ def main() -> int:
|
|
|
4100
4373
|
findings_path = devlyn_dir / output_findings_name()
|
|
4101
4374
|
|
|
4102
4375
|
verify_env = os.environ.copy()
|
|
4376
|
+
for runner_key in (
|
|
4377
|
+
"SPEC_VERIFY_PHASE",
|
|
4378
|
+
"SPEC_VERIFY_FINDINGS_FILE",
|
|
4379
|
+
"SPEC_VERIFY_FINDING_PREFIX",
|
|
4380
|
+
):
|
|
4381
|
+
verify_env.pop(runner_key, None)
|
|
4103
4382
|
verify_env["BENCH_WORKDIR"] = str(work)
|
|
4104
4383
|
|
|
4105
4384
|
results: list[dict] = []
|
|
@@ -4117,6 +4396,7 @@ def main() -> int:
|
|
|
4117
4396
|
expected_exit = vc.get("exit_code", 0)
|
|
4118
4397
|
stdout_contains = vc.get("stdout_contains", []) or []
|
|
4119
4398
|
stdout_not_contains = vc.get("stdout_not_contains", []) or []
|
|
4399
|
+
timeout_sec = verification_timeout_sec(vc)
|
|
4120
4400
|
|
|
4121
4401
|
try:
|
|
4122
4402
|
proc = subprocess.run(
|
|
@@ -4126,7 +4406,7 @@ def main() -> int:
|
|
|
4126
4406
|
env=verify_env,
|
|
4127
4407
|
capture_output=True,
|
|
4128
4408
|
text=True,
|
|
4129
|
-
timeout=
|
|
4409
|
+
timeout=timeout_sec,
|
|
4130
4410
|
)
|
|
4131
4411
|
# Mirror run-fixture.sh post-run verifier: combined stdout+stderr.
|
|
4132
4412
|
out = (proc.stdout or "") + (proc.stderr or "")
|
|
@@ -4226,20 +4506,16 @@ def main() -> int:
|
|
|
4226
4506
|
|
|
4227
4507
|
except subprocess.TimeoutExpired:
|
|
4228
4508
|
results.append({"index": idx, "cmd": cmd, "pass": False,
|
|
4229
|
-
"reason": "timeout"})
|
|
4230
|
-
rule_id = (
|
|
4231
|
-
"correctness.risk-probe-failed"
|
|
4232
|
-
if vc.get("_risk_probe")
|
|
4233
|
-
else "correctness.spec-literal-mismatch"
|
|
4234
|
-
)
|
|
4509
|
+
"reason": "timeout", "timeout_sec": timeout_sec})
|
|
4235
4510
|
findings.append({
|
|
4236
4511
|
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
4237
|
-
"rule_id":
|
|
4512
|
+
"rule_id": "correctness.verification-timeout",
|
|
4238
4513
|
"level": "error",
|
|
4239
4514
|
"severity": "CRITICAL",
|
|
4240
4515
|
"confidence": 1.0,
|
|
4241
4516
|
"message": (
|
|
4242
|
-
f"Verification command #{idx + 1} timed out after
|
|
4517
|
+
f"Verification command #{idx + 1} timed out after {timeout_sec}s "
|
|
4518
|
+
f"(timeout_sec={timeout_sec}, maximum 600)."
|
|
4243
4519
|
),
|
|
4244
4520
|
"file": ".devlyn/risk-probes.jsonl" if vc.get("_risk_probe") else ".devlyn/spec-verify.json",
|
|
4245
4521
|
"line": 1,
|
|
@@ -4250,8 +4526,9 @@ def main() -> int:
|
|
|
4250
4526
|
else f"spec-verify://verification_commands/{idx}"
|
|
4251
4527
|
),
|
|
4252
4528
|
"fix_hint": (
|
|
4253
|
-
f"Command `{cmd}` exceeded
|
|
4254
|
-
|
|
4529
|
+
f"Command `{cmd}` exceeded its {timeout_sec}s timeout_sec budget. "
|
|
4530
|
+
"Increase timeout_sec up to 600 when the verification legitimately "
|
|
4531
|
+
"needs more time, or fix a hang in the implementation."
|
|
4255
4532
|
),
|
|
4256
4533
|
"blocking": True,
|
|
4257
4534
|
"status": "open",
|