devlyn-cli 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +9 -1
- package/CLAUDE.md +28 -11
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +8 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +28 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +15 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +57 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +10 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +50 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +59 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +9 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +88 -0
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +45 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +30 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +16 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +64 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +10 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +21 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +58 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +9 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +54 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +82 -0
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +77 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +99 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +18 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +64 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +10 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +4 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +61 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +9 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +135 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +100 -0
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +75 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +93 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +16 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +65 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +10 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +18 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +66 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +9 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +84 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +184 -0
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +74 -0
- package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +12 -0
- package/benchmark/auto-resolve/scripts/run-fixture.sh +39 -1
- package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +1 -1
- package/bin/devlyn.js +42 -1
- package/config/skills/_shared/adapters/README.md +7 -0
- package/config/skills/_shared/adapters/claude.md +47 -0
- package/config/skills/_shared/adapters/omp.md +17 -0
- package/config/skills/_shared/archive_run.py +8 -2
- package/config/skills/_shared/codex-config.md +22 -6
- package/config/skills/_shared/engine-doctor.sh +107 -0
- package/config/skills/_shared/engine-preflight.md +18 -11
- package/config/skills/_shared/expected.schema.json +26 -0
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/spec-verify-check.py +773 -486
- package/config/skills/_shared/state-phase-write.py +374 -0
- package/config/skills/_shared/verify-merge-findings.py +177 -61
- package/config/skills/devlyn:engines/SKILL.md +47 -0
- package/config/skills/devlyn:ideate/SKILL.md +22 -3
- package/config/skills/devlyn:ideate/references/elicitation.md +2 -2
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +2 -2
- package/config/skills/devlyn:ideate/references/project-mode.md +1 -1
- package/config/skills/devlyn:ideate/references/spec-template.md +15 -4
- package/config/skills/devlyn:queue/SKILL.md +26 -0
- package/config/skills/devlyn:resolve/SKILL.md +54 -24
- package/config/skills/devlyn:resolve/references/free-form-mode.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +2 -2
- package/config/skills/devlyn:resolve/references/phases/cleanup.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/plan.md +3 -2
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +8 -0
- package/config/skills/devlyn:resolve/references/phases/verify.md +18 -9
- package/config/skills/devlyn:resolve/references/state-schema.md +10 -6
- package/optional-skills/devlyn:pencil-pull/SKILL.md +5 -0
- package/optional-skills/devlyn:pencil-push/SKILL.md +5 -0
- package/package.json +1 -1
- package/scripts/lint-skills.sh +48 -41
|
@@ -116,7 +116,10 @@ def output_finding_prefix() -> str:
|
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
VERIFICATION_SECTION_RE = re.compile(
|
|
119
|
-
r'(?ms)
|
|
119
|
+
r'(?ms)^<!--[ \t]*devlyn:verification[ \t]*-->[ \t]*\n(#{1,6}[ \t]+[^\n]*\n.*?)(?=^#{1,6}[ \t]+|\Z)'
|
|
120
|
+
)
|
|
121
|
+
FILES_TO_TOUCH_SECTION_RE = re.compile(
|
|
122
|
+
r'(?ms)^<!--[ \t]*devlyn:authorized-surface[ \t]*-->[ \t]*\n(#{1,6}[ \t]+[^\n]*\n.*?)(?=^#{1,6}[ \t]+|\Z)'
|
|
120
123
|
)
|
|
121
124
|
JSON_FENCE_RE = re.compile(r'(?ms)^```json[ \t]*\n(.*?)\n```[ \t]*$')
|
|
122
125
|
FORBIDDEN_RISK_PROBE_CMD_RE = re.compile(
|
|
@@ -243,6 +246,8 @@ EXPECTED_TOP_LEVEL_KEYS = {
|
|
|
243
246
|
"tier_a_waivers",
|
|
244
247
|
"spec_output_files",
|
|
245
248
|
"max_deps_added",
|
|
249
|
+
"pure_design",
|
|
250
|
+
"required_risk_probe_requirements",
|
|
246
251
|
}
|
|
247
252
|
EXPECTED_VERIFICATION_COMMAND_KEYS = {
|
|
248
253
|
"cmd",
|
|
@@ -251,22 +256,28 @@ EXPECTED_VERIFICATION_COMMAND_KEYS = {
|
|
|
251
256
|
"stdout_not_contains",
|
|
252
257
|
"contract_refs",
|
|
253
258
|
}
|
|
254
|
-
PURE_DESIGN_ESCAPE = "all Requirements are pure-design"
|
|
255
259
|
SPEC_COMPLEXITY_VALUES = {"trivial", "medium", "high", "large"}
|
|
256
260
|
|
|
257
261
|
|
|
258
|
-
def extract_verification_block(text: str) -> str | None:
|
|
259
|
-
"""
|
|
260
|
-
|
|
262
|
+
def extract_verification_block(text: str) -> tuple[bool, str | None]:
|
|
263
|
+
"""Locate the verification section via the `<!-- devlyn:verification -->`
|
|
264
|
+
sentinel (language-neutral — the human-readable heading after it may be
|
|
265
|
+
any text, any language, any ATX heading level 1-6) and return
|
|
266
|
+
(section_found, json_block).
|
|
267
|
+
|
|
268
|
+
section_found=False: the sentinel is absent entirely — a legitimate
|
|
269
|
+
handwritten spec with no mechanical verification contract. Callers treat
|
|
270
|
+
this as a silent no-op, matching pre-existing backward compat.
|
|
261
271
|
|
|
262
|
-
|
|
263
|
-
|
|
272
|
+
section_found=True, json_block=None: the sentinel is present (the author
|
|
273
|
+
clearly intended a verification section) but no fenced ```json``` block
|
|
274
|
+
was found inside it — this is a malformed carrier, not a no-op.
|
|
264
275
|
"""
|
|
265
276
|
section = VERIFICATION_SECTION_RE.search(text)
|
|
266
277
|
if not section:
|
|
267
|
-
return None
|
|
278
|
+
return (False, None)
|
|
268
279
|
fence = JSON_FENCE_RE.search(section.group(1))
|
|
269
|
-
return fence.group(1) if fence else None
|
|
280
|
+
return (True, fence.group(1) if fence else None)
|
|
270
281
|
|
|
271
282
|
|
|
272
283
|
def extract_verification_text(text: str) -> str:
|
|
@@ -274,6 +285,18 @@ def extract_verification_text(text: str) -> str:
|
|
|
274
285
|
return section.group(1) if section else ""
|
|
275
286
|
|
|
276
287
|
|
|
288
|
+
def extract_authorized_surface_block(text: str) -> tuple[bool, str | None]:
|
|
289
|
+
"""Locate PLAN's `Files to touch` section via the
|
|
290
|
+
`<!-- devlyn:authorized-surface -->` sentinel and return
|
|
291
|
+
(section_found, json_block) — same semantics as
|
|
292
|
+
`extract_verification_block()`."""
|
|
293
|
+
section = FILES_TO_TOUCH_SECTION_RE.search(text)
|
|
294
|
+
if not section:
|
|
295
|
+
return (False, None)
|
|
296
|
+
fence = JSON_FENCE_RE.search(section.group(1))
|
|
297
|
+
return (True, fence.group(1) if fence else None)
|
|
298
|
+
|
|
299
|
+
|
|
277
300
|
def extract_frontmatter_field(text: str, field: str) -> str | None:
|
|
278
301
|
if not text.startswith("---"):
|
|
279
302
|
return None
|
|
@@ -514,6 +537,26 @@ def validate_expected_shape(data) -> str | None:
|
|
|
514
537
|
max_deps = data.get("max_deps_added", 0)
|
|
515
538
|
if isinstance(max_deps, bool) or not isinstance(max_deps, int) or max_deps < 0:
|
|
516
539
|
return "max_deps_added must be a non-negative integer"
|
|
540
|
+
if "pure_design" in data and not isinstance(data["pure_design"], bool):
|
|
541
|
+
return "pure_design must be a boolean"
|
|
542
|
+
requirements = data.get("required_risk_probe_requirements", [])
|
|
543
|
+
if not isinstance(requirements, list):
|
|
544
|
+
return "required_risk_probe_requirements must be a list"
|
|
545
|
+
for i, requirement in enumerate(requirements):
|
|
546
|
+
if not isinstance(requirement, dict):
|
|
547
|
+
return f"required_risk_probe_requirements[{i}] must be an object"
|
|
548
|
+
unknown_requirement_keys = sorted(set(requirement) - {"tag", "derived_from"})
|
|
549
|
+
if unknown_requirement_keys:
|
|
550
|
+
return (
|
|
551
|
+
f"required_risk_probe_requirements[{i}] unknown key(s): "
|
|
552
|
+
f"{', '.join(unknown_requirement_keys)}"
|
|
553
|
+
)
|
|
554
|
+
tag = requirement.get("tag")
|
|
555
|
+
if not isinstance(tag, str) or tag not in RISK_PROBE_TAGS:
|
|
556
|
+
return f"required_risk_probe_requirements[{i}].tag must be one of: {', '.join(sorted(RISK_PROBE_TAGS))}"
|
|
557
|
+
derived_from = requirement.get("derived_from")
|
|
558
|
+
if not isinstance(derived_from, str) or not derived_from:
|
|
559
|
+
return f"required_risk_probe_requirements[{i}].derived_from must be a non-empty string"
|
|
517
560
|
patterns = data.get("forbidden_patterns", [])
|
|
518
561
|
if not isinstance(patterns, list):
|
|
519
562
|
return "forbidden_patterns must be a list"
|
|
@@ -562,13 +605,16 @@ def validate_expected_against_sibling_spec(expected_path: Path, data: object) ->
|
|
|
562
605
|
)
|
|
563
606
|
if solo_headroom_command_err:
|
|
564
607
|
return solo_headroom_command_err
|
|
608
|
+
pure_design = data.get("pure_design") is True
|
|
565
609
|
if commands:
|
|
610
|
+
if pure_design:
|
|
611
|
+
return 'pure_design: true is contradictory with a non-empty verification_commands'
|
|
566
612
|
return None
|
|
567
|
-
if
|
|
613
|
+
if pure_design:
|
|
568
614
|
return None
|
|
569
615
|
return (
|
|
570
|
-
|
|
571
|
-
|
|
616
|
+
'verification_commands must contain at least one entry unless '
|
|
617
|
+
'spec.expected.json declares "pure_design": true'
|
|
572
618
|
)
|
|
573
619
|
|
|
574
620
|
|
|
@@ -620,24 +666,6 @@ def validate_risk_probe(probe: object, index: int, verification_text: str) -> st
|
|
|
620
666
|
unknown_tags = sorted(set(tags) - RISK_PROBE_TAGS)
|
|
621
667
|
if unknown_tags:
|
|
622
668
|
return f"risk-probes[{index}].tags contains unknown tag(s): {', '.join(unknown_tags)}"
|
|
623
|
-
if "error_contract" in tags and not re.search(
|
|
624
|
-
r'stderr|exit[ `]*2|(?:prints?|writes?).{0,40}json[ -]?error|'
|
|
625
|
-
r'json[ -]?error.{0,40}(?:stderr|exit)',
|
|
626
|
-
derived_from,
|
|
627
|
-
re.IGNORECASE,
|
|
628
|
-
):
|
|
629
|
-
return (
|
|
630
|
-
f"risk-probes[{index}].derived_from for error_contract must name "
|
|
631
|
-
"a stderr, JSON-error stream, or exit-2 verification bullet"
|
|
632
|
-
)
|
|
633
|
-
if "http_error_contract" in tags and not (
|
|
634
|
-
re.search(r'\b(?:400|401|403|404|409|422|5[0-9][0-9])\b|http|status', derived_from, re.IGNORECASE)
|
|
635
|
-
and re.search(r'error|invalid', derived_from, re.IGNORECASE)
|
|
636
|
-
):
|
|
637
|
-
return (
|
|
638
|
-
f"risk-probes[{index}].derived_from for http_error_contract must name "
|
|
639
|
-
"an HTTP/status error response verification bullet"
|
|
640
|
-
)
|
|
641
669
|
evidence = probe.get("tag_evidence")
|
|
642
670
|
if not isinstance(evidence, dict):
|
|
643
671
|
return f"risk-probes[{index}].tag_evidence must be an object"
|
|
@@ -654,15 +682,12 @@ def validate_risk_probe(probe: object, index: int, verification_text: str) -> st
|
|
|
654
682
|
f"risk-probes[{index}].tag_evidence.{tag} missing required "
|
|
655
683
|
f"item(s): {', '.join(missing_evidence)}"
|
|
656
684
|
)
|
|
657
|
-
if "shape_contract" in tags
|
|
685
|
+
if "shape_contract" in tags:
|
|
658
686
|
actual = evidence.get("shape_contract")
|
|
659
687
|
if not isinstance(actual, list) or not all(isinstance(item, str) for item in actual):
|
|
660
688
|
return f"risk-probes[{index}].tag_evidence.shape_contract must be a list of strings"
|
|
661
689
|
required_shape = set(SHAPE_CONTRACT_REQUIRED_EVIDENCE)
|
|
662
|
-
if
|
|
663
|
-
INLINE_JSON_OBJECT_RE.search(derived_from)
|
|
664
|
-
and re.search(r'error|invalid', derived_from, re.IGNORECASE)
|
|
665
|
-
):
|
|
690
|
+
if {"error_contract", "http_error_contract"} & set(tags):
|
|
666
691
|
required_shape.add("asserts_exact_error_object")
|
|
667
692
|
missing_shape = sorted(required_shape - set(actual))
|
|
668
693
|
if missing_shape:
|
|
@@ -673,75 +698,66 @@ def validate_risk_probe(probe: object, index: int, verification_text: str) -> st
|
|
|
673
698
|
return None
|
|
674
699
|
|
|
675
700
|
|
|
676
|
-
def
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
)
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
):
|
|
737
|
-
required.add("concurrent_state_consistency")
|
|
738
|
-
if re.search(
|
|
739
|
-
r'one valid \+ one invalid|valid \+ one invalid|all-or-nothing|'
|
|
740
|
-
r'same list as before|0 inserts|no partial updates',
|
|
741
|
-
text,
|
|
742
|
-
):
|
|
743
|
-
required.add("atomic_batch_state")
|
|
744
|
-
return required
|
|
701
|
+
def validate_required_risk_probe_requirement(
|
|
702
|
+
requirement: object, index: int, verification_text: str,
|
|
703
|
+
) -> str | None:
|
|
704
|
+
if not isinstance(requirement, dict):
|
|
705
|
+
return f"required_risk_probe_requirements[{index}] must be a JSON object"
|
|
706
|
+
tag = requirement.get("tag")
|
|
707
|
+
if not isinstance(tag, str) or tag not in RISK_PROBE_TAGS:
|
|
708
|
+
return (
|
|
709
|
+
f"required_risk_probe_requirements[{index}].tag must be one of: "
|
|
710
|
+
f"{', '.join(sorted(RISK_PROBE_TAGS))}"
|
|
711
|
+
)
|
|
712
|
+
derived_from = requirement.get("derived_from")
|
|
713
|
+
if not isinstance(derived_from, str) or not derived_from.strip():
|
|
714
|
+
return f"required_risk_probe_requirements[{index}].derived_from must be a non-empty string"
|
|
715
|
+
if derived_from not in verification_text:
|
|
716
|
+
return (
|
|
717
|
+
f"required_risk_probe_requirements[{index}].derived_from must be an "
|
|
718
|
+
"exact substring of the source verification section"
|
|
719
|
+
)
|
|
720
|
+
return None
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def resolve_required_risk_probe_requirements(
|
|
724
|
+
source_md: Path | None,
|
|
725
|
+
) -> tuple[list[dict], str | None]:
|
|
726
|
+
"""Return the spec author's declared `required_risk_probe_requirements`
|
|
727
|
+
(a language-neutral replacement for a keyword-prose classifier): a list
|
|
728
|
+
of `{"tag": ..., "derived_from": ...}` obligations that risk-probes.jsonl
|
|
729
|
+
must cover when risk probes are required. Resolved from whichever
|
|
730
|
+
carrier `verification_commands` itself would come from — sibling
|
|
731
|
+
`spec.expected.json` wins when present, else the inline fenced block
|
|
732
|
+
under source_md's `<!-- devlyn:verification -->` sentinel. Absence in
|
|
733
|
+
either carrier is not an error: it means the spec author declared no
|
|
734
|
+
required risk-probe obligations, not that none apply.
|
|
735
|
+
"""
|
|
736
|
+
if source_md is None or not source_md.is_file():
|
|
737
|
+
return ([], None)
|
|
738
|
+
expected_path = source_md.with_name("spec.expected.json")
|
|
739
|
+
if expected_path.is_file():
|
|
740
|
+
data, err = load_expected_contract(expected_path)
|
|
741
|
+
if err:
|
|
742
|
+
return ([], err)
|
|
743
|
+
reqs = (data or {}).get("required_risk_probe_requirements", [])
|
|
744
|
+
else:
|
|
745
|
+
_section_found, block = extract_verification_block(source_md.read_text())
|
|
746
|
+
if block is None:
|
|
747
|
+
return ([], None)
|
|
748
|
+
try:
|
|
749
|
+
parsed = loads_strict_json(block)
|
|
750
|
+
except ValueError as e:
|
|
751
|
+
return ([], f"<!-- devlyn:verification --> ```json``` block in {source_md} has invalid JSON: {e}")
|
|
752
|
+
reqs = parsed.get("required_risk_probe_requirements", []) if isinstance(parsed, dict) else []
|
|
753
|
+
if not isinstance(reqs, list):
|
|
754
|
+
return ([], "required_risk_probe_requirements must be a list")
|
|
755
|
+
verification_text = extract_verification_text(source_md.read_text())
|
|
756
|
+
for i, req in enumerate(reqs):
|
|
757
|
+
err = validate_required_risk_probe_requirement(req, i, verification_text)
|
|
758
|
+
if err:
|
|
759
|
+
return ([], err)
|
|
760
|
+
return (reqs, None)
|
|
745
761
|
|
|
746
762
|
|
|
747
763
|
def load_risk_probes(
|
|
@@ -760,7 +776,7 @@ def load_risk_probes(
|
|
|
760
776
|
|
|
761
777
|
verification_text = extract_verification_text(source_md.read_text())
|
|
762
778
|
if not verification_text:
|
|
763
|
-
return ([], "risk-probes.jsonl exists but source has no
|
|
779
|
+
return ([], "risk-probes.jsonl exists but source has no <!-- devlyn:verification --> section")
|
|
764
780
|
|
|
765
781
|
probes: list[dict] = []
|
|
766
782
|
for index, line in enumerate(probes_path.read_text().splitlines()):
|
|
@@ -782,10 +798,19 @@ def load_risk_probes(
|
|
|
782
798
|
if require_present and not probes:
|
|
783
799
|
return ([], "risk-probes.jsonl must contain at least one probe")
|
|
784
800
|
if require_present:
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
801
|
+
required_reqs, req_err = resolve_required_risk_probe_requirements(source_md)
|
|
802
|
+
if req_err:
|
|
803
|
+
return ([], req_err)
|
|
804
|
+
missing = [
|
|
805
|
+
req for req in required_reqs
|
|
806
|
+
if not any(
|
|
807
|
+
req["tag"] in probe.get("tags", []) and probe.get("derived_from") == req["derived_from"]
|
|
808
|
+
for probe in probes
|
|
809
|
+
)
|
|
810
|
+
]
|
|
811
|
+
if missing:
|
|
812
|
+
formatted = "; ".join(f"{r['tag']} (derived_from={r['derived_from']!r})" for r in missing)
|
|
813
|
+
return ([], f"risk-probes.jsonl missing required probe(s): {formatted}")
|
|
789
814
|
solo_headroom_probe_err = validate_risk_probes_cover_solo_headroom_hypothesis(
|
|
790
815
|
probes,
|
|
791
816
|
verification_text,
|
|
@@ -896,20 +921,25 @@ def stage_from_source(md: Path, devlyn_dir: Path) -> tuple[bool, str | None]:
|
|
|
896
921
|
"""Materialize .devlyn/spec-verify.json from the json block in `md`.
|
|
897
922
|
|
|
898
923
|
Returns (staged, error). staged=True → wrote spec-verify.json. error
|
|
899
|
-
non-None → carrier was found but malformed (caller emits CRITICAL)
|
|
900
|
-
|
|
901
|
-
|
|
924
|
+
non-None → carrier was found but malformed (caller emits CRITICAL) —
|
|
925
|
+
this now also covers a `<!-- devlyn:verification -->` sentinel present
|
|
926
|
+
with no fenced ```json``` block inside it (the author clearly intended a
|
|
927
|
+
contract; a missing fence is a mistake, not a no-op). staged=False,
|
|
928
|
+
error=None → the sentinel is absent entirely (handwritten spec or
|
|
929
|
+
generated source missing the contract).
|
|
902
930
|
"""
|
|
903
|
-
block = extract_verification_block(md.read_text())
|
|
904
|
-
if
|
|
931
|
+
section_found, block = extract_verification_block(md.read_text())
|
|
932
|
+
if not section_found:
|
|
905
933
|
return (False, None)
|
|
934
|
+
if block is None:
|
|
935
|
+
return (False, f"`<!-- devlyn:verification -->` section in {md} has no fenced ```json``` block")
|
|
906
936
|
try:
|
|
907
937
|
data = loads_strict_json(block)
|
|
908
938
|
except ValueError as e:
|
|
909
|
-
return (False, f"
|
|
939
|
+
return (False, f"`<!-- devlyn:verification -->` ```json``` block in {md} has invalid JSON: {e}")
|
|
910
940
|
err = validate_shape(data)
|
|
911
941
|
if err:
|
|
912
|
-
return (False, f"
|
|
942
|
+
return (False, f"`<!-- devlyn:verification -->` ```json``` block in {md}: {err}")
|
|
913
943
|
normalized = {"verification_commands": data["verification_commands"]}
|
|
914
944
|
devlyn_dir.mkdir(parents=True, exist_ok=True)
|
|
915
945
|
(devlyn_dir / "spec-verify.json").write_text(json.dumps(normalized, indent=2) + "\n")
|
|
@@ -1170,12 +1200,178 @@ def expected_contract_findings(
|
|
|
1170
1200
|
return (findings, seq)
|
|
1171
1201
|
|
|
1172
1202
|
|
|
1203
|
+
def validate_authorized_surface_shape(data: object) -> str | None:
|
|
1204
|
+
if not isinstance(data, dict):
|
|
1205
|
+
return "top-level must be a JSON object"
|
|
1206
|
+
err = validate_string_list(data, "authorized_surface")
|
|
1207
|
+
if err:
|
|
1208
|
+
return err
|
|
1209
|
+
surface = data.get("authorized_surface", [])
|
|
1210
|
+
if not surface:
|
|
1211
|
+
return "authorized_surface must contain at least one entry"
|
|
1212
|
+
seen: set[str] = set()
|
|
1213
|
+
for entry in surface:
|
|
1214
|
+
if entry in seen:
|
|
1215
|
+
return f"authorized_surface has a duplicate entry: {entry!r}"
|
|
1216
|
+
seen.add(entry)
|
|
1217
|
+
if entry == "." or entry.startswith("/") or entry.startswith("./"):
|
|
1218
|
+
return (
|
|
1219
|
+
"authorized_surface entry must be a repo-relative path "
|
|
1220
|
+
f"without a leading '/' or './': {entry!r}"
|
|
1221
|
+
)
|
|
1222
|
+
stem = entry[:-3] if entry.endswith("/**") else entry
|
|
1223
|
+
if entry.endswith("/**") and not stem:
|
|
1224
|
+
return f"authorized_surface directory grant needs a non-empty prefix before '/**': {entry!r}"
|
|
1225
|
+
if ".." in stem.split("/"):
|
|
1226
|
+
return f"authorized_surface entry must not contain '..': {entry!r}"
|
|
1227
|
+
return None
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def path_matches_surface(path: str, surface: list[str]) -> bool:
|
|
1231
|
+
for entry in surface:
|
|
1232
|
+
if entry.endswith("/**"):
|
|
1233
|
+
prefix = entry[:-3]
|
|
1234
|
+
if path == prefix or path.startswith(prefix + "/"):
|
|
1235
|
+
return True
|
|
1236
|
+
elif path == entry:
|
|
1237
|
+
return True
|
|
1238
|
+
return False
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def authorized_surface_findings(
|
|
1242
|
+
work: Path, devlyn_dir: Path, state: dict, finding_start: int,
|
|
1243
|
+
) -> tuple[list[dict], int]:
|
|
1244
|
+
"""BUILD_GATE-only (caller gates on `output_phase() == "build_gate"`):
|
|
1245
|
+
enforce PLAN's declared `authorized_surface` against this run's diff.
|
|
1246
|
+
|
|
1247
|
+
iter-0046: closes the scope-leak drift class measured in
|
|
1248
|
+
autoresearch/iterations/0042 + 0045 — bare-model diffs leak an
|
|
1249
|
+
out-of-scope tracked file even with the full CLAUDE.md contract loaded,
|
|
1250
|
+
on every measured model tier. `fix_hint` deliberately never offers
|
|
1251
|
+
"widen the surface" — the fix-loop respawns the same IMPLEMENT worker
|
|
1252
|
+
that produced the leak, and letting it edit `plan.md` to authorize its
|
|
1253
|
+
own diff would let it self-authorize the exact drift this gate exists
|
|
1254
|
+
to catch. A persistent finding exhausts the existing BUILD_GATE
|
|
1255
|
+
fix-loop budget and halts for user/orchestrator review instead.
|
|
1256
|
+
|
|
1257
|
+
Not re-run at VERIFY MECHANICAL time (post-CLEANUP): CLEANUP's own
|
|
1258
|
+
allowlist (tooling artifacts, dead code, doc-reference fixes) licenses
|
|
1259
|
+
paths PLAN never declared, so re-checking there would false-positive on
|
|
1260
|
+
CLEANUP's own sanctioned changes.
|
|
1261
|
+
"""
|
|
1262
|
+
plan_path = devlyn_dir / "plan.md"
|
|
1263
|
+
if not plan_path.is_file():
|
|
1264
|
+
return ([{
|
|
1265
|
+
"id": f"{output_finding_prefix()}-{finding_start:04d}",
|
|
1266
|
+
"rule_id": "scope.authorized-surface-malformed",
|
|
1267
|
+
"level": "error",
|
|
1268
|
+
"severity": "CRITICAL",
|
|
1269
|
+
"confidence": 1.0,
|
|
1270
|
+
"message": "BUILD_GATE requires .devlyn/plan.md with a declared authorized_surface; the file is missing.",
|
|
1271
|
+
"file": ".devlyn/plan.md",
|
|
1272
|
+
"line": 1,
|
|
1273
|
+
"phase": output_phase(),
|
|
1274
|
+
"criterion_ref": "plan.md/authorized_surface",
|
|
1275
|
+
"fix_hint": "PLAN must write .devlyn/plan.md with a `Files to touch` section and an authorized_surface json block before BUILD_GATE can run.",
|
|
1276
|
+
"blocking": True,
|
|
1277
|
+
"status": "open",
|
|
1278
|
+
}], finding_start + 1)
|
|
1279
|
+
try:
|
|
1280
|
+
plan_text = plan_path.read_text(encoding="utf-8")
|
|
1281
|
+
except OSError as e:
|
|
1282
|
+
return ([{
|
|
1283
|
+
"id": f"{output_finding_prefix()}-{finding_start:04d}",
|
|
1284
|
+
"rule_id": "scope.authorized-surface-malformed",
|
|
1285
|
+
"level": "error",
|
|
1286
|
+
"severity": "CRITICAL",
|
|
1287
|
+
"confidence": 1.0,
|
|
1288
|
+
"message": f"Cannot read {plan_path}: {e}",
|
|
1289
|
+
"file": ".devlyn/plan.md",
|
|
1290
|
+
"line": 1,
|
|
1291
|
+
"phase": output_phase(),
|
|
1292
|
+
"criterion_ref": "plan.md/authorized_surface",
|
|
1293
|
+
"fix_hint": "Ensure .devlyn/plan.md is a readable UTF-8 file.",
|
|
1294
|
+
"blocking": True,
|
|
1295
|
+
"status": "open",
|
|
1296
|
+
}], finding_start + 1)
|
|
1297
|
+
|
|
1298
|
+
_section_found, block = extract_authorized_surface_block(plan_text)
|
|
1299
|
+
parse_error: str | None = None
|
|
1300
|
+
data: dict | None = None
|
|
1301
|
+
if block is None:
|
|
1302
|
+
parse_error = (
|
|
1303
|
+
"plan.md must include a `<!-- devlyn:authorized-surface -->` "
|
|
1304
|
+
"section with a fenced ```json``` block: "
|
|
1305
|
+
"{\"authorized_surface\": [...]}."
|
|
1306
|
+
)
|
|
1307
|
+
else:
|
|
1308
|
+
try:
|
|
1309
|
+
data = loads_strict_json(block)
|
|
1310
|
+
except ValueError as e:
|
|
1311
|
+
parse_error = f"authorized_surface json block is invalid JSON: {e}"
|
|
1312
|
+
if parse_error is None:
|
|
1313
|
+
parse_error = validate_authorized_surface_shape(data)
|
|
1314
|
+
if parse_error is not None:
|
|
1315
|
+
return ([{
|
|
1316
|
+
"id": f"{output_finding_prefix()}-{finding_start:04d}",
|
|
1317
|
+
"rule_id": "scope.authorized-surface-malformed",
|
|
1318
|
+
"level": "error",
|
|
1319
|
+
"severity": "CRITICAL",
|
|
1320
|
+
"confidence": 1.0,
|
|
1321
|
+
"message": f"plan.md authorized_surface is malformed: {parse_error}",
|
|
1322
|
+
"file": ".devlyn/plan.md",
|
|
1323
|
+
"line": 1,
|
|
1324
|
+
"phase": output_phase(),
|
|
1325
|
+
"criterion_ref": "plan.md/authorized_surface",
|
|
1326
|
+
"fix_hint": (
|
|
1327
|
+
"Restate section 1's already-listed paths as a fenced "
|
|
1328
|
+
"```json``` block immediately after the `Files to touch` "
|
|
1329
|
+
'list: `{"authorized_surface": ["path/one", "path/two"]}`. '
|
|
1330
|
+
"Do not invent new paths."
|
|
1331
|
+
),
|
|
1332
|
+
"blocking": True,
|
|
1333
|
+
"status": "open",
|
|
1334
|
+
}], finding_start + 1)
|
|
1335
|
+
|
|
1336
|
+
surface = data["authorized_surface"]
|
|
1337
|
+
findings: list[dict] = []
|
|
1338
|
+
seq = finding_start
|
|
1339
|
+
for path in changed_files(work, state, devlyn_dir):
|
|
1340
|
+
if path_matches_surface(path, surface):
|
|
1341
|
+
continue
|
|
1342
|
+
findings.append({
|
|
1343
|
+
"id": f"{output_finding_prefix()}-{seq:04d}",
|
|
1344
|
+
"rule_id": "scope.out-of-scope-file",
|
|
1345
|
+
"level": "error",
|
|
1346
|
+
"severity": "CRITICAL",
|
|
1347
|
+
"confidence": 1.0,
|
|
1348
|
+
"message": f"{path} is outside PLAN's declared authorized_surface.",
|
|
1349
|
+
"file": path,
|
|
1350
|
+
"line": 1,
|
|
1351
|
+
"phase": output_phase(),
|
|
1352
|
+
"criterion_ref": "plan.md/authorized_surface",
|
|
1353
|
+
"fix_hint": (
|
|
1354
|
+
f"Remove {path} from the diff. Do not widen plan.md's "
|
|
1355
|
+
"authorized_surface to include it — that would let this "
|
|
1356
|
+
"fix loop self-authorize its own scope leak. If the file "
|
|
1357
|
+
"is genuinely required, halt per implement.md's contract "
|
|
1358
|
+
"so this finding reaches the user/orchestrator for a new run."
|
|
1359
|
+
),
|
|
1360
|
+
"blocking": True,
|
|
1361
|
+
"status": "open",
|
|
1362
|
+
})
|
|
1363
|
+
seq += 1
|
|
1364
|
+
return (findings, seq)
|
|
1365
|
+
|
|
1366
|
+
|
|
1173
1367
|
def run_check_mode(md_path: Path) -> int:
|
|
1174
1368
|
"""`--check <markdown>` — validate the verification carrier without
|
|
1175
1369
|
running any commands. Used by /devlyn:ideate after item-spec write.
|
|
1176
1370
|
|
|
1177
|
-
Exit 0:
|
|
1178
|
-
|
|
1371
|
+
Exit 0: `<!-- devlyn:verification -->` sentinel absent OR section present
|
|
1372
|
+
and well-formed.
|
|
1373
|
+
Exit 2: sentinel present but the section is malformed — no fenced json
|
|
1374
|
+
block, invalid JSON, or bad shape (so ideate can re-prompt).
|
|
1179
1375
|
"""
|
|
1180
1376
|
if not md_path.is_file():
|
|
1181
1377
|
print(f"[spec-verify --check] error: {md_path} not found", file=sys.stderr)
|
|
@@ -1193,18 +1389,25 @@ def run_check_mode(md_path: Path) -> int:
|
|
|
1193
1389
|
if solo_ceiling_err:
|
|
1194
1390
|
print(f"[spec-verify --check] {md_path}: {solo_ceiling_err}", file=sys.stderr)
|
|
1195
1391
|
return 2
|
|
1196
|
-
block = extract_verification_block(text)
|
|
1197
|
-
if
|
|
1198
|
-
#
|
|
1199
|
-
#
|
|
1200
|
-
#
|
|
1392
|
+
section_found, block = extract_verification_block(text)
|
|
1393
|
+
if not section_found:
|
|
1394
|
+
# Sentinel absent entirely — opt-in nature preserved for ideate (a
|
|
1395
|
+
# spec without machine verification is still valid; it just won't
|
|
1396
|
+
# activate the BUILD_GATE gate).
|
|
1201
1397
|
return 0
|
|
1398
|
+
if block is None:
|
|
1399
|
+
print(
|
|
1400
|
+
f"[spec-verify --check] {md_path}: `<!-- devlyn:verification -->` "
|
|
1401
|
+
"section found but no fenced ```json``` block inside it",
|
|
1402
|
+
file=sys.stderr,
|
|
1403
|
+
)
|
|
1404
|
+
return 2
|
|
1202
1405
|
try:
|
|
1203
1406
|
data = loads_strict_json(block)
|
|
1204
1407
|
except ValueError as e:
|
|
1205
1408
|
print(
|
|
1206
|
-
f"[spec-verify --check] {md_path}: invalid JSON in
|
|
1207
|
-
f"```json``` block: {e}",
|
|
1409
|
+
f"[spec-verify --check] {md_path}: invalid JSON in "
|
|
1410
|
+
f"`<!-- devlyn:verification -->` ```json``` block: {e}",
|
|
1208
1411
|
file=sys.stderr,
|
|
1209
1412
|
)
|
|
1210
1413
|
return 2
|
|
@@ -1249,7 +1452,7 @@ def run_self_test() -> int:
|
|
|
1249
1452
|
devlyn = work / ".devlyn"
|
|
1250
1453
|
devlyn.mkdir()
|
|
1251
1454
|
spec_md = work / "spec.md"
|
|
1252
|
-
spec_md.write_text("# Spec\n\n## Verification\n\n- probe must pass visible marker.\n")
|
|
1455
|
+
spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n")
|
|
1253
1456
|
(devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
1254
1457
|
"source": {"type": "spec", "spec_path": str(spec_md)}
|
|
1255
1458
|
}))
|
|
@@ -1266,7 +1469,13 @@ def run_self_test() -> int:
|
|
|
1266
1469
|
"stdout_contains": ["probe-ok"],
|
|
1267
1470
|
"stdout_not_contains": [],
|
|
1268
1471
|
"tags": ["shape_contract"],
|
|
1269
|
-
"tag_evidence": {
|
|
1472
|
+
"tag_evidence": {
|
|
1473
|
+
"shape_contract": [
|
|
1474
|
+
"uses_visible_input_key_names",
|
|
1475
|
+
"asserts_visible_output_key_names",
|
|
1476
|
+
"asserts_no_unexpected_output_keys",
|
|
1477
|
+
],
|
|
1478
|
+
},
|
|
1270
1479
|
}) + "\n")
|
|
1271
1480
|
env = os.environ.copy()
|
|
1272
1481
|
env["BENCH_WORKDIR"] = str(work)
|
|
@@ -1366,7 +1575,13 @@ def run_self_test() -> int:
|
|
|
1366
1575
|
"stdout_contains": ["probe-ok"],
|
|
1367
1576
|
"stdout_not_contains": [],
|
|
1368
1577
|
"tags": ["shape_contract"],
|
|
1369
|
-
"tag_evidence": {
|
|
1578
|
+
"tag_evidence": {
|
|
1579
|
+
"shape_contract": [
|
|
1580
|
+
"uses_visible_input_key_names",
|
|
1581
|
+
"asserts_visible_output_key_names",
|
|
1582
|
+
"asserts_no_unexpected_output_keys",
|
|
1583
|
+
],
|
|
1584
|
+
},
|
|
1370
1585
|
}) + "\n")
|
|
1371
1586
|
|
|
1372
1587
|
good_complexity = work / "good-complexity.md"
|
|
@@ -1405,7 +1620,7 @@ def run_self_test() -> int:
|
|
|
1405
1620
|
|
|
1406
1621
|
weak_solo_headroom = work / "weak-solo-headroom.md"
|
|
1407
1622
|
weak_solo_headroom.write_text(
|
|
1408
|
-
"# Weak\n\n## Verification\n\n"
|
|
1623
|
+
"# Weak\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1409
1624
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling.\n"
|
|
1410
1625
|
"- Observable command: `node check.js` exposes behavior.\n",
|
|
1411
1626
|
encoding="utf-8",
|
|
@@ -1426,7 +1641,7 @@ def run_self_test() -> int:
|
|
|
1426
1641
|
|
|
1427
1642
|
weak_descriptive_backtick = work / "weak-descriptive-backtick.md"
|
|
1428
1643
|
weak_descriptive_backtick.write_text(
|
|
1429
|
-
"# Weak descriptive backtick\n\n## Verification\n\n"
|
|
1644
|
+
"# Weak descriptive backtick\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1430
1645
|
"- solo-headroom hypothesis: solo_claude should miss behavior where observable `priority rollback` exposes the miss.\n",
|
|
1431
1646
|
encoding="utf-8",
|
|
1432
1647
|
)
|
|
@@ -1477,7 +1692,7 @@ def run_self_test() -> int:
|
|
|
1477
1692
|
|
|
1478
1693
|
weak_solo_ceiling = work / "weak-solo-ceiling.md"
|
|
1479
1694
|
weak_solo_ceiling.write_text(
|
|
1480
|
-
"# Weak ceiling\n\n## Verification\n\n"
|
|
1695
|
+
"# Weak ceiling\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1481
1696
|
"- solo ceiling avoidance: this is not like the previous ones.\n",
|
|
1482
1697
|
encoding="utf-8",
|
|
1483
1698
|
)
|
|
@@ -1515,7 +1730,7 @@ def run_self_test() -> int:
|
|
|
1515
1730
|
|
|
1516
1731
|
inline_mismatched_solo = work / "inline-mismatched-solo.md"
|
|
1517
1732
|
inline_mismatched_solo.write_text(
|
|
1518
|
-
"# Inline mismatch\n\n## Verification\n\n"
|
|
1733
|
+
"# Inline mismatch\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1519
1734
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
1520
1735
|
"`node check.js` exposes the miss.\n\n"
|
|
1521
1736
|
"```json\n"
|
|
@@ -1539,7 +1754,7 @@ def run_self_test() -> int:
|
|
|
1539
1754
|
|
|
1540
1755
|
inline_matched_solo = work / "inline-matched-solo.md"
|
|
1541
1756
|
inline_matched_solo.write_text(
|
|
1542
|
-
"# Inline match\n\n## Verification\n\n"
|
|
1757
|
+
"# Inline match\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1543
1758
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
1544
1759
|
"`printf ok` exposes the miss.\n\n"
|
|
1545
1760
|
"```json\n"
|
|
@@ -1611,7 +1826,7 @@ def run_self_test() -> int:
|
|
|
1611
1826
|
return 1
|
|
1612
1827
|
|
|
1613
1828
|
spec_md.write_text(
|
|
1614
|
-
"# Spec\n\n## Verification\n\n"
|
|
1829
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1615
1830
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
1616
1831
|
"`printf ok` exposes the miss.\n"
|
|
1617
1832
|
"- probe must pass visible marker.\n",
|
|
@@ -1625,7 +1840,13 @@ def run_self_test() -> int:
|
|
|
1625
1840
|
"stdout_contains": ["unrelated"],
|
|
1626
1841
|
"stdout_not_contains": [],
|
|
1627
1842
|
"tags": ["shape_contract"],
|
|
1628
|
-
"tag_evidence": {
|
|
1843
|
+
"tag_evidence": {
|
|
1844
|
+
"shape_contract": [
|
|
1845
|
+
"uses_visible_input_key_names",
|
|
1846
|
+
"asserts_visible_output_key_names",
|
|
1847
|
+
"asserts_no_unexpected_output_keys",
|
|
1848
|
+
],
|
|
1849
|
+
},
|
|
1629
1850
|
}) + "\n")
|
|
1630
1851
|
bad_solo_headroom_probe = subprocess.run(
|
|
1631
1852
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
@@ -1650,7 +1871,13 @@ def run_self_test() -> int:
|
|
|
1650
1871
|
"stdout_contains": ["ok"],
|
|
1651
1872
|
"stdout_not_contains": [],
|
|
1652
1873
|
"tags": ["shape_contract"],
|
|
1653
|
-
"tag_evidence": {
|
|
1874
|
+
"tag_evidence": {
|
|
1875
|
+
"shape_contract": [
|
|
1876
|
+
"uses_visible_input_key_names",
|
|
1877
|
+
"asserts_visible_output_key_names",
|
|
1878
|
+
"asserts_no_unexpected_output_keys",
|
|
1879
|
+
],
|
|
1880
|
+
},
|
|
1654
1881
|
}) + "\n")
|
|
1655
1882
|
bad_solo_headroom_derived_from = subprocess.run(
|
|
1656
1883
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
@@ -1676,7 +1903,13 @@ def run_self_test() -> int:
|
|
|
1676
1903
|
"stdout_contains": ["first-unrelated"],
|
|
1677
1904
|
"stdout_not_contains": [],
|
|
1678
1905
|
"tags": ["shape_contract"],
|
|
1679
|
-
"tag_evidence": {
|
|
1906
|
+
"tag_evidence": {
|
|
1907
|
+
"shape_contract": [
|
|
1908
|
+
"uses_visible_input_key_names",
|
|
1909
|
+
"asserts_visible_output_key_names",
|
|
1910
|
+
"asserts_no_unexpected_output_keys",
|
|
1911
|
+
],
|
|
1912
|
+
},
|
|
1680
1913
|
}) + "\n" + json.dumps({
|
|
1681
1914
|
"id": "P5b",
|
|
1682
1915
|
"derived_from": "probe must pass visible marker.",
|
|
@@ -1685,7 +1918,13 @@ def run_self_test() -> int:
|
|
|
1685
1918
|
"stdout_contains": ["ok"],
|
|
1686
1919
|
"stdout_not_contains": [],
|
|
1687
1920
|
"tags": ["shape_contract"],
|
|
1688
|
-
"tag_evidence": {
|
|
1921
|
+
"tag_evidence": {
|
|
1922
|
+
"shape_contract": [
|
|
1923
|
+
"uses_visible_input_key_names",
|
|
1924
|
+
"asserts_visible_output_key_names",
|
|
1925
|
+
"asserts_no_unexpected_output_keys",
|
|
1926
|
+
],
|
|
1927
|
+
},
|
|
1689
1928
|
}) + "\n"
|
|
1690
1929
|
)
|
|
1691
1930
|
late_solo_headroom_probe = subprocess.run(
|
|
@@ -1707,7 +1946,13 @@ def run_self_test() -> int:
|
|
|
1707
1946
|
"stdout_contains": ["ok2"],
|
|
1708
1947
|
"stdout_not_contains": [],
|
|
1709
1948
|
"tags": ["shape_contract"],
|
|
1710
|
-
"tag_evidence": {
|
|
1949
|
+
"tag_evidence": {
|
|
1950
|
+
"shape_contract": [
|
|
1951
|
+
"uses_visible_input_key_names",
|
|
1952
|
+
"asserts_visible_output_key_names",
|
|
1953
|
+
"asserts_no_unexpected_output_keys",
|
|
1954
|
+
],
|
|
1955
|
+
},
|
|
1711
1956
|
}) + "\n")
|
|
1712
1957
|
prefix_solo_headroom_probe = subprocess.run(
|
|
1713
1958
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
@@ -1728,7 +1973,13 @@ def run_self_test() -> int:
|
|
|
1728
1973
|
"stdout_contains": ["ok"],
|
|
1729
1974
|
"stdout_not_contains": [],
|
|
1730
1975
|
"tags": ["shape_contract"],
|
|
1731
|
-
"tag_evidence": {
|
|
1976
|
+
"tag_evidence": {
|
|
1977
|
+
"shape_contract": [
|
|
1978
|
+
"uses_visible_input_key_names",
|
|
1979
|
+
"asserts_visible_output_key_names",
|
|
1980
|
+
"asserts_no_unexpected_output_keys",
|
|
1981
|
+
],
|
|
1982
|
+
},
|
|
1732
1983
|
}) + "\n")
|
|
1733
1984
|
good_solo_headroom_probe = subprocess.run(
|
|
1734
1985
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
@@ -1759,7 +2010,7 @@ def run_self_test() -> int:
|
|
|
1759
2010
|
"max_deps_added": 0,
|
|
1760
2011
|
}) + "\n")
|
|
1761
2012
|
spec_md.write_text(
|
|
1762
|
-
"# Spec\n\n## Verification\n\n"
|
|
2013
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1763
2014
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling.\n"
|
|
1764
2015
|
"- Observable command: `node check.js` exposes behavior.\n"
|
|
1765
2016
|
)
|
|
@@ -1778,7 +2029,7 @@ def run_self_test() -> int:
|
|
|
1778
2029
|
return 1
|
|
1779
2030
|
|
|
1780
2031
|
spec_md.write_text(
|
|
1781
|
-
"# Spec\n\n## Verification\n\n"
|
|
2032
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1782
2033
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
1783
2034
|
"`node check.js` exposes the miss.\n",
|
|
1784
2035
|
encoding="utf-8",
|
|
@@ -1798,7 +2049,7 @@ def run_self_test() -> int:
|
|
|
1798
2049
|
return 1
|
|
1799
2050
|
|
|
1800
2051
|
spec_md.write_text(
|
|
1801
|
-
"# Spec\n\n## Verification\n\n"
|
|
2052
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1802
2053
|
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
1803
2054
|
"`printf ok` exposes the miss.\n",
|
|
1804
2055
|
encoding="utf-8",
|
|
@@ -1815,7 +2066,7 @@ def run_self_test() -> int:
|
|
|
1815
2066
|
return 1
|
|
1816
2067
|
|
|
1817
2068
|
spec_md.write_text(
|
|
1818
|
-
"# Spec\n\n## Verification\n\n"
|
|
2069
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1819
2070
|
"- Solo-headroom hypothesis: the spec must literally contain `solo_claude`, `miss`, and an observable command; "
|
|
1820
2071
|
"`printf ok` exposes the miss.\n",
|
|
1821
2072
|
encoding="utf-8",
|
|
@@ -1832,7 +2083,7 @@ def run_self_test() -> int:
|
|
|
1832
2083
|
return 1
|
|
1833
2084
|
|
|
1834
2085
|
spec_md.write_text(
|
|
1835
|
-
"# Spec\n\n## Verification\n\n"
|
|
2086
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1836
2087
|
"- solo ceiling avoidance: this differs from controls but omits the required baseline.\n",
|
|
1837
2088
|
encoding="utf-8",
|
|
1838
2089
|
)
|
|
@@ -1851,7 +2102,7 @@ def run_self_test() -> int:
|
|
|
1851
2102
|
return 1
|
|
1852
2103
|
|
|
1853
2104
|
spec_md.write_text(
|
|
1854
|
-
"# Spec\n\n## Verification\n\n"
|
|
2105
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
1855
2106
|
"- solo ceiling avoidance: unlike solo-saturated `S2`-`S6`, this includes a "
|
|
1856
2107
|
"multi-run temporal dependency because solo_claude headroom should remain.\n",
|
|
1857
2108
|
encoding="utf-8",
|
|
@@ -1867,7 +2118,7 @@ def run_self_test() -> int:
|
|
|
1867
2118
|
print(strong_sibling_solo_ceiling.stderr, file=sys.stderr)
|
|
1868
2119
|
return 1
|
|
1869
2120
|
|
|
1870
|
-
spec_md.write_text("# Spec\n\n## Verification\n\n- probe must pass visible marker.\n")
|
|
2121
|
+
spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n")
|
|
1871
2122
|
expected_good = subprocess.run(
|
|
1872
2123
|
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
1873
2124
|
cwd=work,
|
|
@@ -1879,7 +2130,7 @@ def run_self_test() -> int:
|
|
|
1879
2130
|
return 1
|
|
1880
2131
|
|
|
1881
2132
|
spec_md.write_text(
|
|
1882
|
-
"---\nid: bad-sibling\ncomplexity: hihg\n---\n\n# Bad sibling\n\n## Verification\n\n- ok\n",
|
|
2133
|
+
"---\nid: bad-sibling\ncomplexity: hihg\n---\n\n# Bad sibling\n\n<!-- devlyn:verification -->\n## Verification\n\n- ok\n",
|
|
1883
2134
|
encoding="utf-8",
|
|
1884
2135
|
)
|
|
1885
2136
|
expected_bad_sibling_complexity = subprocess.run(
|
|
@@ -1895,7 +2146,7 @@ def run_self_test() -> int:
|
|
|
1895
2146
|
print("--check-expected did not report unsupported sibling spec complexity", file=sys.stderr)
|
|
1896
2147
|
print(expected_bad_sibling_complexity.stderr, file=sys.stderr)
|
|
1897
2148
|
return 1
|
|
1898
|
-
spec_md.write_text("# Spec\n\n## Verification\n\n- probe must pass visible marker.\n")
|
|
2149
|
+
spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n")
|
|
1899
2150
|
|
|
1900
2151
|
expected_json.write_text(json.dumps({"verification_commands": []}) + "\n")
|
|
1901
2152
|
expected_empty_runtime = subprocess.run(
|
|
@@ -1912,11 +2163,14 @@ def run_self_test() -> int:
|
|
|
1912
2163
|
pure_root.mkdir()
|
|
1913
2164
|
pure_spec = pure_root / "spec.md"
|
|
1914
2165
|
pure_spec.write_text(
|
|
1915
|
-
"# Pure design\n\n## Verification\n\n-
|
|
2166
|
+
"# Pure design\n\n<!-- devlyn:verification -->\n## Verification\n\n- no runtime verification commands.\n",
|
|
1916
2167
|
encoding="utf-8",
|
|
1917
2168
|
)
|
|
1918
2169
|
pure_expected = pure_root / "spec.expected.json"
|
|
1919
|
-
pure_expected.write_text(
|
|
2170
|
+
pure_expected.write_text(
|
|
2171
|
+
json.dumps({"verification_commands": [], "pure_design": True}) + "\n",
|
|
2172
|
+
encoding="utf-8",
|
|
2173
|
+
)
|
|
1920
2174
|
expected_empty_design = subprocess.run(
|
|
1921
2175
|
[sys.executable, script_path, "--check-expected", str(pure_expected)],
|
|
1922
2176
|
cwd=work,
|
|
@@ -1928,6 +2182,45 @@ def run_self_test() -> int:
|
|
|
1928
2182
|
print(expected_empty_design.stderr, file=sys.stderr)
|
|
1929
2183
|
return 1
|
|
1930
2184
|
|
|
2185
|
+
pure_expected.write_text(
|
|
2186
|
+
json.dumps({
|
|
2187
|
+
"verification_commands": [{"cmd": "printf ok", "stdout_contains": ["ok"]}],
|
|
2188
|
+
"pure_design": True,
|
|
2189
|
+
}) + "\n",
|
|
2190
|
+
encoding="utf-8",
|
|
2191
|
+
)
|
|
2192
|
+
pure_design_contradiction = subprocess.run(
|
|
2193
|
+
[sys.executable, script_path, "--check-expected", str(pure_expected)],
|
|
2194
|
+
cwd=work,
|
|
2195
|
+
capture_output=True,
|
|
2196
|
+
text=True,
|
|
2197
|
+
)
|
|
2198
|
+
if pure_design_contradiction.returncode == 0:
|
|
2199
|
+
print("pure_design: true with non-empty verification_commands was accepted", file=sys.stderr)
|
|
2200
|
+
return 1
|
|
2201
|
+
if "contradictory" not in pure_design_contradiction.stderr:
|
|
2202
|
+
print("pure_design contradiction did not report the right error", file=sys.stderr)
|
|
2203
|
+
print(pure_design_contradiction.stderr, file=sys.stderr)
|
|
2204
|
+
return 1
|
|
2205
|
+
|
|
2206
|
+
pure_expected.write_text(
|
|
2207
|
+
json.dumps({"verification_commands": [], "pure_design": "yes"}) + "\n",
|
|
2208
|
+
encoding="utf-8",
|
|
2209
|
+
)
|
|
2210
|
+
pure_design_not_boolean = subprocess.run(
|
|
2211
|
+
[sys.executable, script_path, "--check-expected", str(pure_expected)],
|
|
2212
|
+
cwd=work,
|
|
2213
|
+
capture_output=True,
|
|
2214
|
+
text=True,
|
|
2215
|
+
)
|
|
2216
|
+
if pure_design_not_boolean.returncode == 0:
|
|
2217
|
+
print("non-boolean pure_design was accepted", file=sys.stderr)
|
|
2218
|
+
return 1
|
|
2219
|
+
if "pure_design must be a boolean" not in pure_design_not_boolean.stderr:
|
|
2220
|
+
print("non-boolean pure_design did not report the right error", file=sys.stderr)
|
|
2221
|
+
print(pure_design_not_boolean.stderr, file=sys.stderr)
|
|
2222
|
+
return 1
|
|
2223
|
+
|
|
1931
2224
|
expected_json.write_text(json.dumps({"unknown": True}) + "\n")
|
|
1932
2225
|
expected_bad = subprocess.run(
|
|
1933
2226
|
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
@@ -2009,7 +2302,7 @@ def run_self_test() -> int:
|
|
|
2009
2302
|
spec_integrity_devlyn.mkdir()
|
|
2010
2303
|
integrity_spec = spec_integrity / "spec.md"
|
|
2011
2304
|
integrity_spec.write_text(
|
|
2012
|
-
"# Spec\n\n## Verification\n\n```json\n"
|
|
2305
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
2013
2306
|
"{\"verification_commands\":[{\"cmd\":\"printf spec-hash-ok\",\"stdout_contains\":[\"spec-hash-ok\"]}]}\n"
|
|
2014
2307
|
"```\n",
|
|
2015
2308
|
encoding="utf-8",
|
|
@@ -2063,7 +2356,7 @@ def run_self_test() -> int:
|
|
|
2063
2356
|
generated_devlyn.mkdir()
|
|
2064
2357
|
generated_criteria = generated_user / ".devlyn" / "criteria.generated.md"
|
|
2065
2358
|
generated_criteria.write_text(
|
|
2066
|
-
"# Criteria\n\n## Verification\n\n```json\n"
|
|
2359
|
+
"# Criteria\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
2067
2360
|
"{\"verification_commands\":[{\"cmd\":\"printf generated-ok\",\"stdout_contains\":[\"generated-ok\"]}]}\n"
|
|
2068
2361
|
"```\n",
|
|
2069
2362
|
encoding="utf-8",
|
|
@@ -2160,7 +2453,7 @@ def run_self_test() -> int:
|
|
|
2160
2453
|
real_devlyn.mkdir()
|
|
2161
2454
|
real_spec = real_user / "spec.md"
|
|
2162
2455
|
real_spec.write_text(
|
|
2163
|
-
"# Spec\n\n## Verification\n\n- sibling command must print sibling-ok.\n"
|
|
2456
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- sibling command must print sibling-ok.\n"
|
|
2164
2457
|
)
|
|
2165
2458
|
(real_user / "spec.expected.json").write_text(json.dumps({
|
|
2166
2459
|
"verification_commands": [
|
|
@@ -2190,7 +2483,7 @@ def run_self_test() -> int:
|
|
|
2190
2483
|
malformed_devlyn.mkdir()
|
|
2191
2484
|
malformed_spec = malformed / "spec.md"
|
|
2192
2485
|
malformed_spec.write_text(
|
|
2193
|
-
"# Spec\n\n## Verification\n\n```json\n"
|
|
2486
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n```json\n"
|
|
2194
2487
|
"{\"verification_commands\":[{\"cmd\":\"printf inline-ok\"}]}\n"
|
|
2195
2488
|
"```\n"
|
|
2196
2489
|
)
|
|
@@ -2209,7 +2502,7 @@ def run_self_test() -> int:
|
|
|
2209
2502
|
return 1
|
|
2210
2503
|
|
|
2211
2504
|
bench_spec = work / "bench-spec.md"
|
|
2212
|
-
bench_spec.write_text("# Spec\n\n## Verification\n\n- benchmark pre-staged wins.\n")
|
|
2505
|
+
bench_spec.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- benchmark pre-staged wins.\n")
|
|
2213
2506
|
(work / "spec.expected.json").write_text(json.dumps({
|
|
2214
2507
|
"verification_commands": [
|
|
2215
2508
|
{"cmd": "printf wrong", "stdout_contains": ["wrong"]}
|
|
@@ -2243,7 +2536,7 @@ def run_self_test() -> int:
|
|
|
2243
2536
|
verify_devlyn = verify_output / ".devlyn"
|
|
2244
2537
|
verify_devlyn.mkdir()
|
|
2245
2538
|
verify_spec = verify_output / "spec.md"
|
|
2246
|
-
verify_spec.write_text("# Spec\n\n## Verification\n\n- verify mechanical output.\n")
|
|
2539
|
+
verify_spec.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- verify mechanical output.\n")
|
|
2247
2540
|
(verify_output / "spec.expected.json").write_text(json.dumps({
|
|
2248
2541
|
"verification_commands": [
|
|
2249
2542
|
{"cmd": "printf wrong", "stdout_contains": ["expected"]}
|
|
@@ -2293,7 +2586,7 @@ def run_self_test() -> int:
|
|
|
2293
2586
|
text=True,
|
|
2294
2587
|
).strip()
|
|
2295
2588
|
contract_spec = contract_root / "spec.md"
|
|
2296
|
-
contract_spec.write_text("# Spec\n\n## Verification\n\n- expected contract checks.\n")
|
|
2589
|
+
contract_spec.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- expected contract checks.\n")
|
|
2297
2590
|
(contract_root / "app.js").write_text("try { work(); } catch { return null; }\n")
|
|
2298
2591
|
(contract_root / "forbidden.txt").write_text("forbidden\n")
|
|
2299
2592
|
(contract_root / "package.json").write_text(
|
|
@@ -2355,52 +2648,13 @@ def run_self_test() -> int:
|
|
|
2355
2648
|
print("incomplete boundary_overlap evidence was accepted", file=sys.stderr)
|
|
2356
2649
|
return 1
|
|
2357
2650
|
|
|
2358
|
-
rollback_root = work / "rollback-risk-probe"
|
|
2359
|
-
rollback_root.mkdir()
|
|
2360
|
-
rollback_devlyn = rollback_root / ".devlyn"
|
|
2361
|
-
rollback_devlyn.mkdir()
|
|
2362
|
-
rollback_spec = rollback_root / "spec.md"
|
|
2363
|
-
rollback_spec.write_text(
|
|
2364
|
-
"# Spec\n\n## Verification\n\n"
|
|
2365
|
-
"- A failed all-or-nothing operation must roll back tentative state "
|
|
2366
|
-
"so later orders can use the released stock.\n"
|
|
2367
|
-
)
|
|
2368
|
-
(rollback_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2369
|
-
"source": {"type": "spec", "spec_path": str(rollback_spec)}
|
|
2370
|
-
}))
|
|
2371
|
-
(rollback_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2372
|
-
"id": "P5",
|
|
2373
|
-
"derived_from": (
|
|
2374
|
-
"A failed all-or-nothing operation must roll back tentative "
|
|
2375
|
-
"state so later orders can use the released stock."
|
|
2376
|
-
),
|
|
2377
|
-
"cmd": "printf weak-rollback",
|
|
2378
|
-
"exit_code": 0,
|
|
2379
|
-
"tags": ["prior_consumption"],
|
|
2380
|
-
"tag_evidence": {
|
|
2381
|
-
"prior_consumption": [
|
|
2382
|
-
"same_resource_consumed_first",
|
|
2383
|
-
"later_entity_fails_or_reroutes",
|
|
2384
|
-
],
|
|
2385
|
-
},
|
|
2386
|
-
}) + "\n")
|
|
2387
|
-
weak_rollback = subprocess.run(
|
|
2388
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2389
|
-
cwd=rollback_root,
|
|
2390
|
-
capture_output=True,
|
|
2391
|
-
text=True,
|
|
2392
|
-
)
|
|
2393
|
-
if weak_rollback.returncode == 0:
|
|
2394
|
-
print("rollback verification text did not require rollback_state probe tag", file=sys.stderr)
|
|
2395
|
-
return 1
|
|
2396
|
-
|
|
2397
2651
|
error_root = work / "error-contract-risk-probe"
|
|
2398
2652
|
error_root.mkdir()
|
|
2399
2653
|
error_devlyn = error_root / ".devlyn"
|
|
2400
2654
|
error_devlyn.mkdir()
|
|
2401
2655
|
error_spec = error_root / "spec.md"
|
|
2402
2656
|
error_spec.write_text(
|
|
2403
|
-
"# Spec\n\n## Verification\n\n"
|
|
2657
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2404
2658
|
"- Invalid input must print a JSON error object to stderr and exit 2.\n"
|
|
2405
2659
|
)
|
|
2406
2660
|
(error_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
@@ -2451,34 +2705,10 @@ def run_self_test() -> int:
|
|
|
2451
2705
|
print("stdout_stderr_contract without stream evidence was accepted", file=sys.stderr)
|
|
2452
2706
|
return 1
|
|
2453
2707
|
|
|
2454
|
-
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2455
|
-
"id": "P7b",
|
|
2456
|
-
"derived_from": "Invalid input must print a JSON error object to stderr and exit 2.",
|
|
2457
|
-
"cmd": "printf weak-json-error-shape-contract",
|
|
2458
|
-
"exit_code": 2,
|
|
2459
|
-
"tags": ["stdout_stderr_contract", "error_contract"],
|
|
2460
|
-
"tag_evidence": {
|
|
2461
|
-
"stdout_stderr_contract": ["asserts_named_stream_output"],
|
|
2462
|
-
"error_contract": [
|
|
2463
|
-
"asserts_error_payload_or_stderr",
|
|
2464
|
-
"asserts_nonzero_or_exit_2",
|
|
2465
|
-
],
|
|
2466
|
-
},
|
|
2467
|
-
}) + "\n")
|
|
2468
|
-
weak_json_error_shape_contract = subprocess.run(
|
|
2469
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2470
|
-
cwd=error_root,
|
|
2471
|
-
capture_output=True,
|
|
2472
|
-
text=True,
|
|
2473
|
-
)
|
|
2474
|
-
if weak_json_error_shape_contract.returncode == 0:
|
|
2475
|
-
print("JSON error object text did not require shape_contract tag", file=sys.stderr)
|
|
2476
|
-
return 1
|
|
2477
|
-
|
|
2478
2708
|
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2479
2709
|
"id": "P7c",
|
|
2480
2710
|
"derived_from": "Invalid input must print a JSON error object to stderr and exit 2.",
|
|
2481
|
-
"cmd": "printf json-error-shape-contract",
|
|
2711
|
+
"cmd": "printf json-error-shape-contract-missing-exact",
|
|
2482
2712
|
"exit_code": 2,
|
|
2483
2713
|
"tags": ["stdout_stderr_contract", "error_contract", "shape_contract"],
|
|
2484
2714
|
"tag_evidence": {
|
|
@@ -2491,53 +2721,27 @@ def run_self_test() -> int:
|
|
|
2491
2721
|
"uses_visible_input_key_names",
|
|
2492
2722
|
"asserts_visible_output_key_names",
|
|
2493
2723
|
"asserts_no_unexpected_output_keys",
|
|
2494
|
-
"asserts_exact_error_object",
|
|
2495
2724
|
],
|
|
2496
2725
|
},
|
|
2497
2726
|
}) + "\n")
|
|
2498
|
-
|
|
2727
|
+
missing_exact_error_object = subprocess.run(
|
|
2499
2728
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2500
2729
|
cwd=error_root,
|
|
2501
2730
|
capture_output=True,
|
|
2502
2731
|
text=True,
|
|
2503
2732
|
)
|
|
2504
|
-
if
|
|
2505
|
-
print(
|
|
2506
|
-
|
|
2733
|
+
if missing_exact_error_object.returncode == 0:
|
|
2734
|
+
print(
|
|
2735
|
+
"shape_contract co-occurring with error_contract without "
|
|
2736
|
+
"asserts_exact_error_object was accepted",
|
|
2737
|
+
file=sys.stderr,
|
|
2738
|
+
)
|
|
2507
2739
|
return 1
|
|
2508
2740
|
|
|
2509
|
-
inline_json_error = (
|
|
2510
|
-
'Invalid input prints JSON error `{ "error": "invalid" }` to stderr and exits 2.'
|
|
2511
|
-
)
|
|
2512
|
-
error_spec.write_text("# Spec\n\n## Verification\n\n- " + inline_json_error + "\n")
|
|
2513
2741
|
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2514
2742
|
"id": "P7d",
|
|
2515
|
-
"derived_from":
|
|
2516
|
-
"cmd": "printf
|
|
2517
|
-
"exit_code": 2,
|
|
2518
|
-
"tags": ["stdout_stderr_contract", "error_contract"],
|
|
2519
|
-
"tag_evidence": {
|
|
2520
|
-
"stdout_stderr_contract": ["asserts_named_stream_output"],
|
|
2521
|
-
"error_contract": [
|
|
2522
|
-
"asserts_error_payload_or_stderr",
|
|
2523
|
-
"asserts_nonzero_or_exit_2",
|
|
2524
|
-
],
|
|
2525
|
-
},
|
|
2526
|
-
}) + "\n")
|
|
2527
|
-
weak_inline_json_error_shape_contract = subprocess.run(
|
|
2528
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2529
|
-
cwd=error_root,
|
|
2530
|
-
capture_output=True,
|
|
2531
|
-
text=True,
|
|
2532
|
-
)
|
|
2533
|
-
if weak_inline_json_error_shape_contract.returncode == 0:
|
|
2534
|
-
print("inline JSON error text did not require shape_contract tag", file=sys.stderr)
|
|
2535
|
-
return 1
|
|
2536
|
-
|
|
2537
|
-
(error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2538
|
-
"id": "P7e",
|
|
2539
|
-
"derived_from": inline_json_error,
|
|
2540
|
-
"cmd": "printf inline-json-error-shape-contract",
|
|
2743
|
+
"derived_from": "Invalid input must print a JSON error object to stderr and exit 2.",
|
|
2744
|
+
"cmd": "printf json-error-shape-contract",
|
|
2541
2745
|
"exit_code": 2,
|
|
2542
2746
|
"tags": ["stdout_stderr_contract", "error_contract", "shape_contract"],
|
|
2543
2747
|
"tag_evidence": {
|
|
@@ -2554,15 +2758,15 @@ def run_self_test() -> int:
|
|
|
2554
2758
|
],
|
|
2555
2759
|
},
|
|
2556
2760
|
}) + "\n")
|
|
2557
|
-
|
|
2761
|
+
strong_json_error_shape_contract = subprocess.run(
|
|
2558
2762
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2559
2763
|
cwd=error_root,
|
|
2560
2764
|
capture_output=True,
|
|
2561
2765
|
text=True,
|
|
2562
2766
|
)
|
|
2563
|
-
if
|
|
2564
|
-
print("
|
|
2565
|
-
print(
|
|
2767
|
+
if strong_json_error_shape_contract.returncode != 0:
|
|
2768
|
+
print("JSON error object shape_contract with exact object evidence was rejected", file=sys.stderr)
|
|
2769
|
+
print(strong_json_error_shape_contract.stderr, file=sys.stderr)
|
|
2566
2770
|
return 1
|
|
2567
2771
|
|
|
2568
2772
|
http_error_root = work / "http-error-contract-risk-probe"
|
|
@@ -2571,33 +2775,12 @@ def run_self_test() -> int:
|
|
|
2571
2775
|
http_error_devlyn.mkdir()
|
|
2572
2776
|
http_error_spec = http_error_root / "spec.md"
|
|
2573
2777
|
http_error_spec.write_text(
|
|
2574
|
-
"# Spec\n\n## Verification\n\n"
|
|
2778
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2575
2779
|
"- An invalid query returns HTTP 400 with JSON error body `{ \"error\": \"invalid_query\", \"field\": \"per_page\" }`.\n"
|
|
2576
2780
|
)
|
|
2577
2781
|
(http_error_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2578
2782
|
"source": {"type": "spec", "spec_path": str(http_error_spec)}
|
|
2579
2783
|
}))
|
|
2580
|
-
(http_error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2581
|
-
"id": "P8",
|
|
2582
|
-
"derived_from": (
|
|
2583
|
-
"An invalid query returns HTTP 400 with JSON error body "
|
|
2584
|
-
"`{ \"error\": \"invalid_query\", \"field\": \"per_page\" }`."
|
|
2585
|
-
),
|
|
2586
|
-
"cmd": "printf weak-http-error-contract",
|
|
2587
|
-
"exit_code": 0,
|
|
2588
|
-
"tags": ["shape_contract"],
|
|
2589
|
-
"tag_evidence": {},
|
|
2590
|
-
}) + "\n")
|
|
2591
|
-
weak_http_error_contract = subprocess.run(
|
|
2592
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2593
|
-
cwd=http_error_root,
|
|
2594
|
-
capture_output=True,
|
|
2595
|
-
text=True,
|
|
2596
|
-
)
|
|
2597
|
-
if weak_http_error_contract.returncode == 0:
|
|
2598
|
-
print("http error text did not require http_error_contract tag", file=sys.stderr)
|
|
2599
|
-
return 1
|
|
2600
|
-
|
|
2601
2784
|
(http_error_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2602
2785
|
"id": "P8b",
|
|
2603
2786
|
"derived_from": (
|
|
@@ -2691,7 +2874,7 @@ def run_self_test() -> int:
|
|
|
2691
2874
|
shape_devlyn.mkdir()
|
|
2692
2875
|
shape_spec = shape_root / "spec.md"
|
|
2693
2876
|
shape_spec.write_text(
|
|
2694
|
-
"# Spec\n\n## Verification\n\n"
|
|
2877
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2695
2878
|
"- On success, output is one JSON object with keys `applied`, `rejected`, and `accounts`; "
|
|
2696
2879
|
"`rejected` rows have keys `id` and `reason`.\n"
|
|
2697
2880
|
)
|
|
@@ -2699,7 +2882,7 @@ def run_self_test() -> int:
|
|
|
2699
2882
|
"source": {"type": "spec", "spec_path": str(shape_spec)}
|
|
2700
2883
|
}))
|
|
2701
2884
|
(shape_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2702
|
-
"id": "
|
|
2885
|
+
"id": "P8e",
|
|
2703
2886
|
"derived_from": (
|
|
2704
2887
|
"On success, output is one JSON object with keys `applied`, `rejected`, and `accounts`; "
|
|
2705
2888
|
"`rejected` rows have keys `id` and `reason`."
|
|
@@ -2716,11 +2899,11 @@ def run_self_test() -> int:
|
|
|
2716
2899
|
text=True,
|
|
2717
2900
|
)
|
|
2718
2901
|
if weak_shape_contract.returncode == 0:
|
|
2719
|
-
print("shape_contract without
|
|
2902
|
+
print("shape_contract without any evidence was accepted", file=sys.stderr)
|
|
2720
2903
|
return 1
|
|
2721
2904
|
|
|
2722
2905
|
(shape_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2723
|
-
"id": "
|
|
2906
|
+
"id": "P8f",
|
|
2724
2907
|
"derived_from": (
|
|
2725
2908
|
"On success, output is one JSON object with keys `applied`, `rejected`, and `accounts`; "
|
|
2726
2909
|
"`rejected` rows have keys `id` and `reason`."
|
|
@@ -2747,219 +2930,138 @@ def run_self_test() -> int:
|
|
|
2747
2930
|
print(strong_shape_contract.stderr, file=sys.stderr)
|
|
2748
2931
|
return 1
|
|
2749
2932
|
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
cwd=shape_root,
|
|
2765
|
-
capture_output=True,
|
|
2766
|
-
text=True,
|
|
2933
|
+
# iter-0049 F3: required_risk_probe_requirements replaces the deleted
|
|
2934
|
+
# required_risk_probe_tags() English-keyword classifier. The spec
|
|
2935
|
+
# author declares required {tag, derived_from} obligations directly
|
|
2936
|
+
# in the verification carrier instead of the harness guessing from
|
|
2937
|
+
# prose -- this works identically for any human language.
|
|
2938
|
+
required_root = work / "required-risk-probe-requirements"
|
|
2939
|
+
required_root.mkdir()
|
|
2940
|
+
required_devlyn = required_root / ".devlyn"
|
|
2941
|
+
required_devlyn.mkdir()
|
|
2942
|
+
required_spec = required_root / "spec.md"
|
|
2943
|
+
required_spec.write_text(
|
|
2944
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2945
|
+
"- A failed all-or-nothing operation must roll back tentative state "
|
|
2946
|
+
"so later orders can use the released stock.\n"
|
|
2767
2947
|
)
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2948
|
+
(required_root / "spec.expected.json").write_text(json.dumps({
|
|
2949
|
+
"verification_commands": [{"cmd": "printf ok", "stdout_contains": ["ok"]}],
|
|
2950
|
+
"required_risk_probe_requirements": [
|
|
2951
|
+
{
|
|
2952
|
+
"tag": "rollback_state",
|
|
2953
|
+
"derived_from": (
|
|
2954
|
+
"A failed all-or-nothing operation must roll back "
|
|
2955
|
+
"tentative state so later orders can use the released stock."
|
|
2956
|
+
),
|
|
2957
|
+
},
|
|
2958
|
+
],
|
|
2959
|
+
}) + "\n")
|
|
2960
|
+
(required_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2961
|
+
"source": {"type": "spec", "spec_path": str(required_spec)}
|
|
2962
|
+
}))
|
|
2963
|
+
(required_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2964
|
+
"id": "P14",
|
|
2965
|
+
"derived_from": (
|
|
2966
|
+
"A failed all-or-nothing operation must roll back tentative "
|
|
2967
|
+
"state so later orders can use the released stock."
|
|
2968
|
+
),
|
|
2969
|
+
"cmd": "printf weak-rollback",
|
|
2776
2970
|
"exit_code": 0,
|
|
2777
|
-
"tags": ["
|
|
2971
|
+
"tags": ["prior_consumption"],
|
|
2778
2972
|
"tag_evidence": {
|
|
2779
|
-
"
|
|
2780
|
-
|
|
2781
|
-
"
|
|
2782
|
-
"asserts_visible_output_key_names",
|
|
2783
|
-
"asserts_no_unexpected_output_keys",
|
|
2973
|
+
"prior_consumption": [
|
|
2974
|
+
"same_resource_consumed_first",
|
|
2975
|
+
"later_entity_fails_or_reroutes",
|
|
2784
2976
|
],
|
|
2785
2977
|
},
|
|
2786
2978
|
}) + "\n")
|
|
2787
|
-
|
|
2979
|
+
missing_declared_requirement = subprocess.run(
|
|
2788
2980
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2789
|
-
cwd=
|
|
2981
|
+
cwd=required_root,
|
|
2790
2982
|
capture_output=True,
|
|
2791
2983
|
text=True,
|
|
2792
2984
|
)
|
|
2793
|
-
if
|
|
2794
|
-
print(
|
|
2795
|
-
|
|
2985
|
+
if missing_declared_requirement.returncode == 0:
|
|
2986
|
+
print(
|
|
2987
|
+
"risk-probes.jsonl missing a declared required_risk_probe_requirements "
|
|
2988
|
+
"entry was accepted",
|
|
2989
|
+
file=sys.stderr,
|
|
2990
|
+
)
|
|
2796
2991
|
return 1
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
forbidden_text_spec.write_text(
|
|
2804
|
-
"# Spec\n\n## Verification\n\n"
|
|
2805
|
-
"- The diff must not add forbidden fallback patterns.\n"
|
|
2806
|
-
)
|
|
2807
|
-
(forbidden_text_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2808
|
-
"source": {"type": "spec", "spec_path": str(forbidden_text_spec)}
|
|
2809
|
-
}))
|
|
2810
|
-
(forbidden_text_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2811
|
-
"id": "P8",
|
|
2812
|
-
"derived_from": "The diff must not add forbidden fallback patterns.",
|
|
2813
|
-
"cmd": "printf forbidden-pattern-static-check",
|
|
2814
|
-
"exit_code": 0,
|
|
2815
|
-
"tags": ["shape_contract"],
|
|
2816
|
-
"tag_evidence": {},
|
|
2817
|
-
}) + "\n")
|
|
2818
|
-
forbidden_pattern_probe = subprocess.run(
|
|
2819
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2820
|
-
cwd=forbidden_text_root,
|
|
2821
|
-
capture_output=True,
|
|
2822
|
-
text=True,
|
|
2823
|
-
)
|
|
2824
|
-
if forbidden_pattern_probe.returncode != 0:
|
|
2825
|
-
print("generic forbidden-pattern verification text incorrectly required boundary_overlap", file=sys.stderr)
|
|
2826
|
-
print(forbidden_pattern_probe.stderr, file=sys.stderr)
|
|
2992
|
+
if (
|
|
2993
|
+
"missing required probe(s)" not in missing_declared_requirement.stderr
|
|
2994
|
+
or "rollback_state" not in missing_declared_requirement.stderr
|
|
2995
|
+
):
|
|
2996
|
+
print("missing required_risk_probe_requirements coverage had the wrong error", file=sys.stderr)
|
|
2997
|
+
print(missing_declared_requirement.stderr, file=sys.stderr)
|
|
2827
2998
|
return 1
|
|
2828
2999
|
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
stock_validation_devlyn = stock_validation_root / ".devlyn"
|
|
2832
|
-
stock_validation_devlyn.mkdir()
|
|
2833
|
-
stock_validation_spec = stock_validation_root / "spec.md"
|
|
2834
|
-
stock_validation_spec.write_text(
|
|
2835
|
-
"# Spec\n\n## Verification\n\n"
|
|
2836
|
-
"- A quote over combined stock exits `2`, prints one JSON error to stderr, and prints no stdout.\n"
|
|
2837
|
-
)
|
|
2838
|
-
(stock_validation_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2839
|
-
"source": {"type": "spec", "spec_path": str(stock_validation_spec)}
|
|
2840
|
-
}))
|
|
2841
|
-
(stock_validation_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2842
|
-
"id": "P9",
|
|
3000
|
+
(required_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3001
|
+
"id": "P15",
|
|
2843
3002
|
"derived_from": (
|
|
2844
|
-
"A
|
|
2845
|
-
"
|
|
3003
|
+
"A failed all-or-nothing operation must roll back tentative "
|
|
3004
|
+
"state so later orders can use the released stock."
|
|
2846
3005
|
),
|
|
2847
|
-
"cmd": "printf
|
|
2848
|
-
"exit_code":
|
|
2849
|
-
"tags": ["
|
|
3006
|
+
"cmd": "printf good-rollback",
|
|
3007
|
+
"exit_code": 0,
|
|
3008
|
+
"tags": ["rollback_state"],
|
|
2850
3009
|
"tag_evidence": {
|
|
2851
|
-
"
|
|
2852
|
-
|
|
2853
|
-
"
|
|
2854
|
-
"asserts_nonzero_or_exit_2",
|
|
3010
|
+
"rollback_state": [
|
|
3011
|
+
"failed_entity_tentative_state_absent",
|
|
3012
|
+
"later_entity_uses_released_state",
|
|
2855
3013
|
],
|
|
2856
3014
|
},
|
|
2857
3015
|
}) + "\n")
|
|
2858
|
-
|
|
3016
|
+
covered_declared_requirement = subprocess.run(
|
|
2859
3017
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2860
|
-
cwd=
|
|
3018
|
+
cwd=required_root,
|
|
2861
3019
|
capture_output=True,
|
|
2862
3020
|
text=True,
|
|
2863
3021
|
)
|
|
2864
|
-
if
|
|
2865
|
-
print(
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
webhook_devlyn = webhook_root / ".devlyn"
|
|
2872
|
-
webhook_devlyn.mkdir()
|
|
2873
|
-
webhook_spec = webhook_root / "spec.md"
|
|
2874
|
-
webhook_spec.write_text(
|
|
2875
|
-
"# Spec\n\n## Verification\n\n"
|
|
2876
|
-
"- A POST whose body has been modified after signing returns 401.\n"
|
|
2877
|
-
"- A second POST with the same accepted `id` returns 409 even if the duplicate body would otherwise fail shape validation.\n"
|
|
2878
|
-
)
|
|
2879
|
-
(webhook_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2880
|
-
"source": {"type": "spec", "spec_path": str(webhook_spec)}
|
|
2881
|
-
}))
|
|
2882
|
-
(webhook_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2883
|
-
"id": "P10",
|
|
2884
|
-
"derived_from": "A POST whose body has been modified after signing returns 401.",
|
|
2885
|
-
"cmd": "printf weak-webhook",
|
|
2886
|
-
"exit_code": 0,
|
|
2887
|
-
"tags": ["shape_contract"],
|
|
2888
|
-
"tag_evidence": {},
|
|
2889
|
-
}) + "\n")
|
|
2890
|
-
weak_webhook_probe = subprocess.run(
|
|
2891
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2892
|
-
cwd=webhook_root,
|
|
2893
|
-
capture_output=True,
|
|
2894
|
-
text=True,
|
|
2895
|
-
)
|
|
2896
|
-
if weak_webhook_probe.returncode == 0:
|
|
2897
|
-
print("webhook signature/replay text did not require auth/idempotency probe tags", file=sys.stderr)
|
|
3022
|
+
if covered_declared_requirement.returncode != 0:
|
|
3023
|
+
print(
|
|
3024
|
+
"risk-probes.jsonl covering a declared required_risk_probe_requirements "
|
|
3025
|
+
"entry was rejected",
|
|
3026
|
+
file=sys.stderr,
|
|
3027
|
+
)
|
|
3028
|
+
print(covered_declared_requirement.stderr, file=sys.stderr)
|
|
2898
3029
|
return 1
|
|
2899
3030
|
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
duplicate_sku_spec.write_text(
|
|
2906
|
-
"# Spec\n\n## Verification\n\n"
|
|
2907
|
-
"- A cart with duplicate SKUs combines quantities before stock validation.\n"
|
|
2908
|
-
)
|
|
2909
|
-
(duplicate_sku_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2910
|
-
"source": {"type": "spec", "spec_path": str(duplicate_sku_spec)}
|
|
2911
|
-
}))
|
|
2912
|
-
(duplicate_sku_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2913
|
-
"id": "P11",
|
|
2914
|
-
"derived_from": "A cart with duplicate SKUs combines quantities before stock validation.",
|
|
2915
|
-
"cmd": "printf duplicate-sku-shape",
|
|
2916
|
-
"exit_code": 0,
|
|
2917
|
-
"tags": ["shape_contract"],
|
|
2918
|
-
"tag_evidence": {},
|
|
3031
|
+
(required_root / "spec.expected.json").write_text(json.dumps({
|
|
3032
|
+
"verification_commands": [{"cmd": "printf ok", "stdout_contains": ["ok"]}],
|
|
3033
|
+
"required_risk_probe_requirements": [
|
|
3034
|
+
{"tag": "not-a-real-tag", "derived_from": "irrelevant"},
|
|
3035
|
+
],
|
|
2919
3036
|
}) + "\n")
|
|
2920
|
-
|
|
3037
|
+
malformed_requirement_tag = subprocess.run(
|
|
2921
3038
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2922
|
-
cwd=
|
|
3039
|
+
cwd=required_root,
|
|
2923
3040
|
capture_output=True,
|
|
2924
3041
|
text=True,
|
|
2925
3042
|
)
|
|
2926
|
-
if
|
|
2927
|
-
print("
|
|
2928
|
-
print(duplicate_sku_probe.stderr, file=sys.stderr)
|
|
3043
|
+
if malformed_requirement_tag.returncode == 0:
|
|
3044
|
+
print("required_risk_probe_requirements with an unknown tag was accepted", file=sys.stderr)
|
|
2929
3045
|
return 1
|
|
2930
3046
|
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
concurrent_spec.write_text(
|
|
2937
|
-
"# Spec\n\n## Verification\n\n"
|
|
2938
|
-
"- Several POST requests close together must all appear exactly once "
|
|
2939
|
-
"in GET output with distinct ids.\n"
|
|
2940
|
-
)
|
|
2941
|
-
(concurrent_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2942
|
-
"source": {"type": "spec", "spec_path": str(concurrent_spec)}
|
|
2943
|
-
}))
|
|
2944
|
-
(concurrent_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2945
|
-
"id": "P12",
|
|
2946
|
-
"derived_from": (
|
|
2947
|
-
"Several POST requests close together must all appear exactly "
|
|
2948
|
-
"once in GET output with distinct ids."
|
|
2949
|
-
),
|
|
2950
|
-
"cmd": "printf weak-concurrent-state",
|
|
2951
|
-
"exit_code": 0,
|
|
2952
|
-
"tags": ["shape_contract"],
|
|
2953
|
-
"tag_evidence": {},
|
|
3047
|
+
(required_root / "spec.expected.json").write_text(json.dumps({
|
|
3048
|
+
"verification_commands": [{"cmd": "printf ok", "stdout_contains": ["ok"]}],
|
|
3049
|
+
"required_risk_probe_requirements": [
|
|
3050
|
+
{"tag": "rollback_state", "derived_from": "text not present in the spec"},
|
|
3051
|
+
],
|
|
2954
3052
|
}) + "\n")
|
|
2955
|
-
|
|
3053
|
+
malformed_requirement_derived_from = subprocess.run(
|
|
2956
3054
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2957
|
-
cwd=
|
|
3055
|
+
cwd=required_root,
|
|
2958
3056
|
capture_output=True,
|
|
2959
3057
|
text=True,
|
|
2960
3058
|
)
|
|
2961
|
-
if
|
|
2962
|
-
print(
|
|
3059
|
+
if malformed_requirement_derived_from.returncode == 0:
|
|
3060
|
+
print(
|
|
3061
|
+
"required_risk_probe_requirements.derived_from not present in the "
|
|
3062
|
+
"spec was accepted",
|
|
3063
|
+
file=sys.stderr,
|
|
3064
|
+
)
|
|
2963
3065
|
return 1
|
|
2964
3066
|
|
|
2965
3067
|
atomic_batch_root = work / "atomic-batch-risk-probe"
|
|
@@ -2968,34 +3070,13 @@ def run_self_test() -> int:
|
|
|
2968
3070
|
atomic_batch_devlyn.mkdir()
|
|
2969
3071
|
atomic_batch_spec = atomic_batch_root / "spec.md"
|
|
2970
3072
|
atomic_batch_spec.write_text(
|
|
2971
|
-
"# Spec\n\n## Verification\n\n"
|
|
3073
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2972
3074
|
"- A POST with one valid + one invalid item returns `400`, AND a subsequent GET returns the same list as before the import.\n"
|
|
2973
3075
|
"- A POST with all-valid items returns `201`, and the items appear in GET output in order with distinct ids.\n"
|
|
2974
3076
|
)
|
|
2975
3077
|
(atomic_batch_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2976
3078
|
"source": {"type": "spec", "spec_path": str(atomic_batch_spec)}
|
|
2977
3079
|
}))
|
|
2978
|
-
(atomic_batch_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2979
|
-
"id": "P13",
|
|
2980
|
-
"derived_from": (
|
|
2981
|
-
"A POST with one valid + one invalid item returns `400`, AND "
|
|
2982
|
-
"a subsequent GET returns the same list as before the import."
|
|
2983
|
-
),
|
|
2984
|
-
"cmd": "printf weak-atomic-batch",
|
|
2985
|
-
"exit_code": 0,
|
|
2986
|
-
"tags": ["shape_contract"],
|
|
2987
|
-
"tag_evidence": {},
|
|
2988
|
-
}) + "\n")
|
|
2989
|
-
weak_atomic_batch_probe = subprocess.run(
|
|
2990
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2991
|
-
cwd=atomic_batch_root,
|
|
2992
|
-
capture_output=True,
|
|
2993
|
-
text=True,
|
|
2994
|
-
)
|
|
2995
|
-
if weak_atomic_batch_probe.returncode == 0:
|
|
2996
|
-
print("atomic batch text did not require atomic_batch_state tag", file=sys.stderr)
|
|
2997
|
-
return 1
|
|
2998
|
-
|
|
2999
3080
|
(atomic_batch_devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3000
3081
|
"id": "P13b",
|
|
3001
3082
|
"derived_from": (
|
|
@@ -3021,6 +3102,205 @@ def run_self_test() -> int:
|
|
|
3021
3102
|
if incomplete_atomic_batch_probe.returncode == 0:
|
|
3022
3103
|
print("atomic_batch_state without success-order evidence was accepted", file=sys.stderr)
|
|
3023
3104
|
return 1
|
|
3105
|
+
|
|
3106
|
+
# iter-0046: PLAN-declared authorized_surface enforced at BUILD_GATE.
|
|
3107
|
+
scope_root = work / "scope-gate"
|
|
3108
|
+
scope_root.mkdir()
|
|
3109
|
+
scope_devlyn = scope_root / ".devlyn"
|
|
3110
|
+
scope_devlyn.mkdir()
|
|
3111
|
+
(scope_root / "bin").mkdir()
|
|
3112
|
+
(scope_root / "lib").mkdir()
|
|
3113
|
+
(scope_root / "lib2").mkdir()
|
|
3114
|
+
(scope_root / "data").mkdir()
|
|
3115
|
+
(scope_root / "bin" / "cli.js").write_text("module.exports = {};\n")
|
|
3116
|
+
(scope_root / "lib" / "keep.js").write_text("module.exports = {};\n")
|
|
3117
|
+
(scope_root / "lib2" / "keep.js").write_text("module.exports = {};\n")
|
|
3118
|
+
(scope_root / "data" / "usage-stats.json").write_text("{}\n")
|
|
3119
|
+
scope_spec = scope_root / "spec.md"
|
|
3120
|
+
scope_spec.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- scope gate checks.\n")
|
|
3121
|
+
(scope_root / "spec.expected.json").write_text(json.dumps({
|
|
3122
|
+
"verification_commands": [{"cmd": "printf ok", "stdout_contains": ["ok"]}],
|
|
3123
|
+
}) + "\n")
|
|
3124
|
+
subprocess.run(["git", "init", "-q"], cwd=scope_root, check=True)
|
|
3125
|
+
subprocess.run(["git", "add", "-A"], cwd=scope_root, check=True)
|
|
3126
|
+
subprocess.run(
|
|
3127
|
+
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "base"],
|
|
3128
|
+
cwd=scope_root,
|
|
3129
|
+
check=True,
|
|
3130
|
+
)
|
|
3131
|
+
scope_base_sha = subprocess.check_output(
|
|
3132
|
+
["git", "rev-parse", "HEAD"], cwd=scope_root, text=True,
|
|
3133
|
+
).strip()
|
|
3134
|
+
(scope_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
3135
|
+
"source": {"type": "spec", "spec_path": str(scope_spec)},
|
|
3136
|
+
"base_ref": {"sha": scope_base_sha},
|
|
3137
|
+
}))
|
|
3138
|
+
scope_findings_path = scope_devlyn / "spec-verify-findings.jsonl"
|
|
3139
|
+
|
|
3140
|
+
# Test 1: no .devlyn/plan.md at all -> fail-closed CRITICAL, not a no-op.
|
|
3141
|
+
(scope_root / "bin" / "cli.js").write_text("module.exports = { ok: true };\n")
|
|
3142
|
+
missing_plan_run = subprocess.run(
|
|
3143
|
+
[sys.executable, script_path], cwd=scope_root, capture_output=True, text=True,
|
|
3144
|
+
)
|
|
3145
|
+
if missing_plan_run.returncode == 0:
|
|
3146
|
+
print("BUILD_GATE accepted a run with no plan.md", file=sys.stderr)
|
|
3147
|
+
return 1
|
|
3148
|
+
if "scope.authorized-surface-malformed" not in scope_findings_path.read_text():
|
|
3149
|
+
print("missing plan.md did not emit scope.authorized-surface-malformed", file=sys.stderr)
|
|
3150
|
+
return 1
|
|
3151
|
+
|
|
3152
|
+
# Test 2: plan.md present but no authorized_surface json block -> malformed.
|
|
3153
|
+
(scope_devlyn / "plan.md").write_text(
|
|
3154
|
+
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n- `bin/cli.js` (edit): ship the fix.\n"
|
|
3155
|
+
)
|
|
3156
|
+
malformed_block_run = subprocess.run(
|
|
3157
|
+
[sys.executable, script_path], cwd=scope_root, capture_output=True, text=True,
|
|
3158
|
+
)
|
|
3159
|
+
if malformed_block_run.returncode == 0:
|
|
3160
|
+
print("BUILD_GATE accepted plan.md with no authorized_surface block", file=sys.stderr)
|
|
3161
|
+
return 1
|
|
3162
|
+
if "scope.authorized-surface-malformed" not in scope_findings_path.read_text():
|
|
3163
|
+
print("missing authorized_surface block did not emit scope.authorized-surface-malformed", file=sys.stderr)
|
|
3164
|
+
return 1
|
|
3165
|
+
|
|
3166
|
+
# Test 3: valid surface, in-scope-only diff -> no scope findings, exit 0.
|
|
3167
|
+
(scope_devlyn / "plan.md").write_text(
|
|
3168
|
+
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n"
|
|
3169
|
+
"- `bin/cli.js` (edit): ship the fix.\n\n"
|
|
3170
|
+
"```json\n"
|
|
3171
|
+
'{"authorized_surface": ["bin/cli.js", "lib/**"]}\n'
|
|
3172
|
+
"```\n"
|
|
3173
|
+
)
|
|
3174
|
+
in_scope_run = subprocess.run(
|
|
3175
|
+
[sys.executable, script_path], cwd=scope_root, capture_output=True, text=True,
|
|
3176
|
+
)
|
|
3177
|
+
if in_scope_run.returncode != 0:
|
|
3178
|
+
print("in-scope-only diff was rejected", file=sys.stderr)
|
|
3179
|
+
print(in_scope_run.stderr, file=sys.stderr)
|
|
3180
|
+
return 1
|
|
3181
|
+
if "scope." in scope_findings_path.read_text():
|
|
3182
|
+
print("in-scope-only diff produced a spurious scope finding", file=sys.stderr)
|
|
3183
|
+
print(scope_findings_path.read_text(), file=sys.stderr)
|
|
3184
|
+
return 1
|
|
3185
|
+
|
|
3186
|
+
# Test 4: directory grant covers lib/**; an out-of-scope file must be
|
|
3187
|
+
# flagged, and the fix_hint must never suggest self-authorization.
|
|
3188
|
+
(scope_root / "lib" / "keep.js").write_text("module.exports = { touched: true };\n")
|
|
3189
|
+
(scope_root / "data" / "usage-stats.json").write_text('{"leaked": true}\n')
|
|
3190
|
+
out_of_scope_run = subprocess.run(
|
|
3191
|
+
[sys.executable, script_path], cwd=scope_root, capture_output=True, text=True,
|
|
3192
|
+
)
|
|
3193
|
+
if out_of_scope_run.returncode == 0:
|
|
3194
|
+
print("out-of-scope file was accepted", file=sys.stderr)
|
|
3195
|
+
return 1
|
|
3196
|
+
out_of_scope_lines = scope_findings_path.read_text().splitlines()
|
|
3197
|
+
out_of_scope_findings = [loads_strict_json(line) for line in out_of_scope_lines if line.strip()]
|
|
3198
|
+
flagged_files = {f["file"] for f in out_of_scope_findings if f.get("rule_id") == "scope.out-of-scope-file"}
|
|
3199
|
+
if "data/usage-stats.json" not in flagged_files:
|
|
3200
|
+
print("scope.out-of-scope-file did not name data/usage-stats.json", file=sys.stderr)
|
|
3201
|
+
return 1
|
|
3202
|
+
if "lib/keep.js" in flagged_files:
|
|
3203
|
+
print("lib/** directory grant did not cover lib/keep.js", file=sys.stderr)
|
|
3204
|
+
return 1
|
|
3205
|
+
for f in out_of_scope_findings:
|
|
3206
|
+
if f.get("rule_id") != "scope.out-of-scope-file":
|
|
3207
|
+
continue
|
|
3208
|
+
hint = f.get("fix_hint", "")
|
|
3209
|
+
if "Remove" not in hint or "amend" in hint.lower():
|
|
3210
|
+
print("scope.out-of-scope-file fix_hint must say remove, never amend/self-authorize", file=sys.stderr)
|
|
3211
|
+
print(hint, file=sys.stderr)
|
|
3212
|
+
return 1
|
|
3213
|
+
|
|
3214
|
+
# Test 5: lib2/keep.js must NOT be covered by the lib/** grant
|
|
3215
|
+
# (directory-prefix boundary, not a bare string-prefix match).
|
|
3216
|
+
(scope_root / "lib2" / "keep.js").write_text("module.exports = { touched: true };\n")
|
|
3217
|
+
subprocess.run(
|
|
3218
|
+
[sys.executable, script_path], cwd=scope_root, capture_output=True, text=True,
|
|
3219
|
+
)
|
|
3220
|
+
boundary_flagged = {
|
|
3221
|
+
loads_strict_json(line)["file"]
|
|
3222
|
+
for line in scope_findings_path.read_text().splitlines() if line.strip()
|
|
3223
|
+
}
|
|
3224
|
+
if "lib2/keep.js" not in boundary_flagged:
|
|
3225
|
+
print("lib/** incorrectly matched lib2/keep.js (directory-prefix boundary bug)", file=sys.stderr)
|
|
3226
|
+
return 1
|
|
3227
|
+
|
|
3228
|
+
# Test 6: VERIFY MECHANICAL (post-CLEANUP re-check) must never run this
|
|
3229
|
+
# gate, even with plan.md entirely absent -- CLEANUP's own allowlist
|
|
3230
|
+
# licenses paths PLAN never declared, so re-checking here would
|
|
3231
|
+
# false-positive on CLEANUP's own sanctioned changes.
|
|
3232
|
+
(scope_devlyn / "plan.md").unlink()
|
|
3233
|
+
verify_mech_env = os.environ.copy()
|
|
3234
|
+
verify_mech_env.update({
|
|
3235
|
+
"SPEC_VERIFY_PHASE": "verify_mechanical",
|
|
3236
|
+
"SPEC_VERIFY_FINDINGS_FILE": "verify-mechanical.findings.jsonl",
|
|
3237
|
+
"SPEC_VERIFY_FINDING_PREFIX": "VERIFY-MECH",
|
|
3238
|
+
})
|
|
3239
|
+
subprocess.run(
|
|
3240
|
+
[sys.executable, script_path], cwd=scope_root, env=verify_mech_env,
|
|
3241
|
+
capture_output=True, text=True,
|
|
3242
|
+
)
|
|
3243
|
+
verify_mech_findings = (scope_devlyn / "verify-mechanical.findings.jsonl").read_text()
|
|
3244
|
+
if "scope." in verify_mech_findings:
|
|
3245
|
+
print("VERIFY MECHANICAL ran the BUILD_GATE-only authorized_surface gate", file=sys.stderr)
|
|
3246
|
+
print(verify_mech_findings, file=sys.stderr)
|
|
3247
|
+
return 1
|
|
3248
|
+
|
|
3249
|
+
# Test 7: shape validation rejects absolute paths, `..`, and duplicates.
|
|
3250
|
+
for bad_surface in (
|
|
3251
|
+
{"authorized_surface": ["/etc/passwd"]},
|
|
3252
|
+
{"authorized_surface": ["bin/../etc/passwd"]},
|
|
3253
|
+
{"authorized_surface": ["bin/cli.js", "bin/cli.js"]},
|
|
3254
|
+
{"authorized_surface": []},
|
|
3255
|
+
):
|
|
3256
|
+
err = validate_authorized_surface_shape(bad_surface)
|
|
3257
|
+
if err is None:
|
|
3258
|
+
print(f"validate_authorized_surface_shape accepted invalid input: {bad_surface}", file=sys.stderr)
|
|
3259
|
+
return 1
|
|
3260
|
+
|
|
3261
|
+
# Test 8 (iter-0054): heading LEVEL after the sentinel is decoration,
|
|
3262
|
+
# same as heading text/language (iter-0049) -- any ATX level 1-6 must
|
|
3263
|
+
# be accepted, not just H2. Reproduces the real iter-0047 claude-small
|
|
3264
|
+
# compliance-cell defect (`# Files to touch` H1 was rejected as
|
|
3265
|
+
# malformed, burning a BUILD_GATE fix-loop round).
|
|
3266
|
+
for heading_prefix in ("#", "###"):
|
|
3267
|
+
h_level_text = (
|
|
3268
|
+
f"<!-- devlyn:authorized-surface -->\n{heading_prefix} Files to touch\n\n"
|
|
3269
|
+
"- `bin/cli.js` (edit): ship the fix.\n\n"
|
|
3270
|
+
'```json\n{"authorized_surface": ["bin/cli.js"]}\n```\n'
|
|
3271
|
+
)
|
|
3272
|
+
found, block = extract_authorized_surface_block(h_level_text)
|
|
3273
|
+
if not found or block is None:
|
|
3274
|
+
print(f"extract_authorized_surface_block rejected a {heading_prefix!r} heading after the sentinel", file=sys.stderr)
|
|
3275
|
+
return 1
|
|
3276
|
+
if loads_strict_json(block) != {"authorized_surface": ["bin/cli.js"]}:
|
|
3277
|
+
print(f"extract_authorized_surface_block parsed the wrong json for a {heading_prefix!r} heading", file=sys.stderr)
|
|
3278
|
+
return 1
|
|
3279
|
+
|
|
3280
|
+
for heading_prefix in ("#", "###"):
|
|
3281
|
+
h_level_verif_text = (
|
|
3282
|
+
f"<!-- devlyn:verification -->\n{heading_prefix} Verification\n\n"
|
|
3283
|
+
'```json\n{"verification_commands": []}\n```\n'
|
|
3284
|
+
)
|
|
3285
|
+
found, block = extract_verification_block(h_level_verif_text)
|
|
3286
|
+
if not found or block is None:
|
|
3287
|
+
print(f"extract_verification_block rejected a {heading_prefix!r} heading after the sentinel", file=sys.stderr)
|
|
3288
|
+
return 1
|
|
3289
|
+
|
|
3290
|
+
# Mixed levels across sections (the exact iter-0047 shape once fixed:
|
|
3291
|
+
# PLAN's own `## Files to touch` H2 followed by a `# Risks` H1) must
|
|
3292
|
+
# still bound the authorized_surface section correctly -- not swallow
|
|
3293
|
+
# the rest of the document.
|
|
3294
|
+
mixed_levels_text = (
|
|
3295
|
+
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## Files to touch\n\n"
|
|
3296
|
+
"- `bin/cli.js` (edit): ship the fix.\n\n"
|
|
3297
|
+
'```json\n{"authorized_surface": ["bin/cli.js"]}\n```\n\n'
|
|
3298
|
+
"# Risks\n\n- none.\n"
|
|
3299
|
+
)
|
|
3300
|
+
found, block = extract_authorized_surface_block(mixed_levels_text)
|
|
3301
|
+
if not found or block is None or loads_strict_json(block) != {"authorized_surface": ["bin/cli.js"]}:
|
|
3302
|
+
print("extract_authorized_surface_block mis-parsed the mixed H2/H1 section-boundary shape", file=sys.stderr)
|
|
3303
|
+
return 1
|
|
3024
3304
|
return 0
|
|
3025
3305
|
|
|
3026
3306
|
|
|
@@ -3399,6 +3679,13 @@ def main() -> int:
|
|
|
3399
3679
|
)
|
|
3400
3680
|
findings.extend(expected_findings)
|
|
3401
3681
|
|
|
3682
|
+
base_sha = ((state.get("base_ref") or {}).get("sha") or "").strip()
|
|
3683
|
+
if output_phase() == "build_gate" and base_sha:
|
|
3684
|
+
surface_findings, finding_seq = authorized_surface_findings(
|
|
3685
|
+
work, devlyn_dir, state, finding_seq,
|
|
3686
|
+
)
|
|
3687
|
+
findings.extend(surface_findings)
|
|
3688
|
+
|
|
3402
3689
|
results_path.write_text(json.dumps({"commands": results}, indent=2) + "\n")
|
|
3403
3690
|
|
|
3404
3691
|
# Append findings (jsonl). BUILD_GATE merge step concatenates this onto
|