devcouncil 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +12 -1
  2. package/package.json +1 -1
  3. package/pyproject.toml +1 -1
  4. package/src/devcouncil/app/config.py +181 -7
  5. package/src/devcouncil/app/orchestrator.py +10 -6
  6. package/src/devcouncil/app/state_machine.py +4 -0
  7. package/src/devcouncil/artifacts/graph.py +9 -2
  8. package/src/devcouncil/cli/commands/check.py +12 -1
  9. package/src/devcouncil/cli/commands/design.py +186 -0
  10. package/src/devcouncil/cli/commands/doctor.py +160 -3
  11. package/src/devcouncil/cli/commands/go.py +96 -16
  12. package/src/devcouncil/cli/commands/hook.py +172 -0
  13. package/src/devcouncil/cli/commands/init.py +7 -2
  14. package/src/devcouncil/cli/commands/integrate.py +492 -34
  15. package/src/devcouncil/cli/commands/logs.py +106 -0
  16. package/src/devcouncil/cli/commands/okf.py +245 -0
  17. package/src/devcouncil/cli/commands/plan.py +54 -14
  18. package/src/devcouncil/cli/commands/repair.py +12 -3
  19. package/src/devcouncil/cli/commands/run.py +128 -7
  20. package/src/devcouncil/cli/commands/skills.py +180 -1
  21. package/src/devcouncil/cli/commands/status.py +7 -16
  22. package/src/devcouncil/cli/commands/verify.py +16 -10
  23. package/src/devcouncil/cli/commands/watch.py +24 -4
  24. package/src/devcouncil/cli/main.py +36 -1
  25. package/src/devcouncil/domain/evidence.py +7 -0
  26. package/src/devcouncil/execution/checkpoints.py +12 -2
  27. package/src/devcouncil/execution/fs_watcher.py +27 -2
  28. package/src/devcouncil/execution/handoff.py +1 -1
  29. package/src/devcouncil/execution/patch.py +6 -0
  30. package/src/devcouncil/execution/permissions.py +7 -0
  31. package/src/devcouncil/execution/policy_engine.py +12 -5
  32. package/src/devcouncil/execution/prompt_builder.py +126 -10
  33. package/src/devcouncil/execution/shell_session.py +6 -0
  34. package/src/devcouncil/execution/task_runner.py +18 -7
  35. package/src/devcouncil/executors/agent_registry.py +22 -1
  36. package/src/devcouncil/executors/coding_cli.py +133 -5
  37. package/src/devcouncil/executors/mini_swe.py +6 -0
  38. package/src/devcouncil/executors/native/agent.py +15 -0
  39. package/src/devcouncil/executors/openhands.py +6 -0
  40. package/src/devcouncil/gating/checks/secret_scan_check.py +7 -0
  41. package/src/devcouncil/gating/policy.py +38 -7
  42. package/src/devcouncil/indexing/ast_matcher.py +16 -6
  43. package/src/devcouncil/indexing/repo_mapper.py +30 -8
  44. package/src/devcouncil/indexing/semantic_index.py +42 -26
  45. package/src/devcouncil/integrations/actions.py +24 -4
  46. package/src/devcouncil/integrations/check.py +7 -4
  47. package/src/devcouncil/integrations/claude_assets.py +444 -0
  48. package/src/devcouncil/integrations/code_review_graph.py +13 -2
  49. package/src/devcouncil/integrations/github_intent.py +8 -1
  50. package/src/devcouncil/integrations/gitnexus.py +10 -2
  51. package/src/devcouncil/integrations/mcp/server.py +404 -15
  52. package/src/devcouncil/integrations/pr_comments.py +9 -0
  53. package/src/devcouncil/knowledge/__init__.py +23 -0
  54. package/src/devcouncil/knowledge/design.py +374 -0
  55. package/src/devcouncil/knowledge/design_conformance.py +317 -0
  56. package/src/devcouncil/knowledge/fetch.py +223 -0
  57. package/src/devcouncil/knowledge/frontmatter.py +51 -0
  58. package/src/devcouncil/knowledge/okf.py +202 -0
  59. package/src/devcouncil/knowledge/skill_bridge.py +96 -0
  60. package/src/devcouncil/knowledge/sources.py +239 -0
  61. package/src/devcouncil/live/cards.py +20 -6
  62. package/src/devcouncil/live/repair_prompt.py +29 -6
  63. package/src/devcouncil/live/reviewer.py +72 -13
  64. package/src/devcouncil/live/summary.py +18 -8
  65. package/src/devcouncil/live/transcripts.py +38 -5
  66. package/src/devcouncil/llm/cache.py +14 -6
  67. package/src/devcouncil/llm/provider.py +179 -92
  68. package/src/devcouncil/llm/router.py +122 -23
  69. package/src/devcouncil/optimization/skillopt.py +673 -0
  70. package/src/devcouncil/planning/arbiter_service.py +10 -2
  71. package/src/devcouncil/planning/correction_manifest.py +47 -4
  72. package/src/devcouncil/planning/critique_service.py +9 -2
  73. package/src/devcouncil/planning/plan_service.py +69 -3
  74. package/src/devcouncil/planning/prompt_enhancer_service.py +124 -0
  75. package/src/devcouncil/planning/repair_service.py +8 -2
  76. package/src/devcouncil/planning/spec_service.py +10 -2
  77. package/src/devcouncil/repo/ci_scaffold.py +13 -5
  78. package/src/devcouncil/repo/sca.py +11 -1
  79. package/src/devcouncil/reporting/json_report.py +11 -0
  80. package/src/devcouncil/reporting/markdown_report.py +14 -1
  81. package/src/devcouncil/reporting/okf_bundle_writer.py +364 -0
  82. package/src/devcouncil/reporting/okf_html.py +323 -0
  83. package/src/devcouncil/reporting/report_builder.py +18 -1
  84. package/src/devcouncil/skills/registry.py +111 -33
  85. package/src/devcouncil/storage/db.py +58 -2
  86. package/src/devcouncil/storage/models.py +4 -0
  87. package/src/devcouncil/storage/native.py +20 -18
  88. package/src/devcouncil/storage/repositories.py +35 -18
  89. package/src/devcouncil/telemetry/logging_setup.py +244 -0
  90. package/src/devcouncil/telemetry/stages.py +141 -0
  91. package/src/devcouncil/telemetry/tracker.py +12 -1
  92. package/src/devcouncil/ui/dashboard.py +69 -5
  93. package/src/devcouncil/verification/acceptance_compiler.py +147 -19
  94. package/src/devcouncil/verification/ad_hoc_check.py +6 -0
  95. package/src/devcouncil/verification/implementation_reviewer.py +11 -2
  96. package/src/devcouncil/verification/sandbox.py +7 -4
  97. package/src/devcouncil/verification/verifier.py +905 -517
  98. package/uv.lock +1 -1
@@ -1,3 +1,4 @@
1
+ import asyncio
1
2
  import hashlib
2
3
  import os
3
4
  import shutil
@@ -11,7 +12,7 @@ import re
11
12
  import shlex
12
13
  from dataclasses import dataclass, asdict
13
14
  from pathlib import Path
14
- from typing import List, Dict, Any, Optional, Tuple
15
+ from typing import List, Dict, Any, Literal, Optional, Tuple
15
16
 
16
17
  from devcouncil.app.config import load_config
17
18
 
@@ -86,6 +87,14 @@ class Verifier:
86
87
  # When set, overrides the (measure, enforce, min_ratio) diff-coverage settings
87
88
  # that would otherwise come from config. Used by ad-hoc checks and tests.
88
89
  self._diff_coverage_override: Optional[Tuple[bool, bool, float]] = None
90
+ # Per-verify_task memos (primed at verify_task entry, cleared before it returns)
91
+ # so the hot path does not re-run `git ls-files` or re-load config repeatedly.
92
+ # None outside a verify_task call, so all other callers behave exactly as before.
93
+ self._untracked_cache: Optional[List[str]] = None
94
+ self._command_timeout_cache: Optional[int] = None
95
+ # Project dependency names (lower-cased), loaded once per verify_task and cleared
96
+ # in its finally so a reused Verifier re-reads them for a later task.
97
+ self._project_deps_cache: Optional[set] = None
89
98
 
90
99
  def _next_gap_id(self, task_id: str, suffix: str) -> str:
91
100
  """Generate unique gap IDs to prevent SQLite overwrites."""
@@ -125,15 +134,15 @@ class Verifier:
125
134
  changed.difference_update(self._load_task_snapshot_files(task_id))
126
135
  return sorted(changed)
127
136
 
128
- def _task_produced_changes(self, task_id: str) -> bool:
129
- """True when the task has a footprint beyond the current working-tree diff.
137
+ def _committed_task_diff(self, task_id: str) -> str:
138
+ """Diff of work committed since the task's ``before`` checkpoint, or "".
130
139
 
131
- Used so the empty-diff guard does not misfire on already-committed work: in
132
- ``dev go`` each task is committed and then re-verified by the reconciliation
133
- pass, at which point ``git diff HEAD`` is empty even though the task was fully
134
- implemented. We detect that via the task's ``before`` checkpoint ref (work
135
- committed since the task started) and a non-empty ``after`` patch. A genuine
136
- no-op run has neither, so it is still correctly flagged as empty.
140
+ When ``dev go`` commits a task's work (e.g. between self-repair attempts, or
141
+ before the reconciliation pass), the working-tree diff (``git diff HEAD``) is
142
+ empty even though the task is fully implemented. This recovers that committed
143
+ change so acceptance compilation/review still have something to reason about
144
+ instead of seeing an empty diff and skipping which would mark every criterion
145
+ unproven and wrongly block correct, committed code.
137
146
  """
138
147
  # Literal of CheckpointService.REF_BEFORE (kept inline to avoid a circular
139
148
  # import: checkpoints.py imports Verifier).
@@ -146,15 +155,27 @@ class Verifier:
146
155
  stderr=subprocess.DEVNULL,
147
156
  ).returncode == 0
148
157
  if has_ref:
149
- diff = subprocess.check_output(
158
+ return subprocess.check_output(
150
159
  ["git", "diff", before_ref],
151
160
  cwd=self.project_root,
152
161
  stderr=subprocess.DEVNULL,
153
162
  ).decode("utf-8", errors="replace")
154
- if diff.strip():
155
- return True
156
163
  except Exception:
157
164
  pass
165
+ return ""
166
+
167
+ def _task_produced_changes(self, task_id: str) -> bool:
168
+ """True when the task has a footprint beyond the current working-tree diff.
169
+
170
+ Used so the empty-diff guard does not misfire on already-committed work: in
171
+ ``dev go`` each task is committed and then re-verified by the reconciliation
172
+ pass, at which point ``git diff HEAD`` is empty even though the task was fully
173
+ implemented. We detect that via the task's ``before`` checkpoint ref (work
174
+ committed since the task started) and a non-empty ``after`` patch. A genuine
175
+ no-op run has neither, so it is still correctly flagged as empty.
176
+ """
177
+ if self._committed_task_diff(task_id).strip():
178
+ return True
158
179
  after_patch = self.project_root / ".devcouncil" / "checkpoints" / f"{task_id}-after.patch"
159
180
  try:
160
181
  return after_patch.exists() and bool(after_patch.read_text(encoding="utf-8", errors="replace").strip())
@@ -209,6 +230,11 @@ class Verifier:
209
230
  return self._filter_change_paths(sorted(files))
210
231
 
211
232
  def _get_untracked_files(self) -> List[str]:
233
+ # Per-verify_task memo: git ls-files is otherwise re-run via get_changed_files,
234
+ # _get_untracked_files_diff, and _classify_change_paths. verify_task primes this
235
+ # once; it is None for every other caller, so they recompute fresh as before.
236
+ if self._untracked_cache is not None:
237
+ return self._untracked_cache
212
238
  try:
213
239
  output = subprocess.check_output(
214
240
  ["git", "ls-files", "--others", "--exclude-standard"],
@@ -415,11 +441,17 @@ class Verifier:
415
441
  return summary[: budget + len(salient) + 8]
416
442
 
417
443
  def _run_command(self, command: str, task_id: str = "verify") -> CommandResult:
418
- try:
419
- config = load_config(self.project_root)
420
- timeout = config.execution.command_timeout
421
- except Exception:
422
- timeout = 300
444
+ # Per-verify_task memo: avoid re-loading config for the timeout on every command
445
+ # in the expected_tests / allowed_commands / compiled-check loops. Falls back to
446
+ # loading config when called outside verify_task (cache is None).
447
+ if self._command_timeout_cache is not None:
448
+ timeout = self._command_timeout_cache
449
+ else:
450
+ try:
451
+ config = load_config(self.project_root)
452
+ timeout = config.execution.command_timeout
453
+ except Exception:
454
+ timeout = 300
423
455
 
424
456
  env = self._verification_env()
425
457
  argv = self._split_command(command)
@@ -664,553 +696,729 @@ class Verifier:
664
696
  pass
665
697
 
666
698
  async def verify_task(self, task: Task, requirements: List[Requirement]) -> Tuple[List[Gap], List[Any]]:
699
+ logger.info("verify_task: task=%s requirements=%d", task.id, len(requirements))
667
700
  self._gap_counter = 0
668
701
  gaps: List[Gap] = []
669
702
  evidence_to_save: List[Any] = []
703
+ # Prime the per-call memos: compute the untracked-file list once (otherwise
704
+ # re-run by get_changed_files, get_diff, and _classify_change_paths) and load the
705
+ # command timeout once (otherwise re-loaded by _run_command on every command).
706
+ # Both are cleared before this method returns.
707
+ self._untracked_cache = self._get_untracked_files()
708
+ ac_samples, ac_repair_attempts = 1, 1
709
+ try:
710
+ _cfg = load_config(self.project_root)
711
+ self._command_timeout_cache = _cfg.execution.command_timeout
712
+ ac_samples = max(1, _cfg.verification.acceptance_checks.samples)
713
+ ac_repair_attempts = max(0, _cfg.verification.acceptance_checks.repair_attempts)
714
+ ac_per_criterion = bool(_cfg.verification.acceptance_checks.per_criterion)
715
+ except Exception:
716
+ self._command_timeout_cache = 300
717
+ ac_per_criterion = False
670
718
  changed_files = self.get_task_changed_files(task.id)
671
719
  diff_content = self.get_diff()
720
+ # When the working tree is clean but the task's work was committed (dev go commits
721
+ # between repair attempts and before reconciliation), fall back to the committed
722
+ # checkpoint diff. Otherwise acceptance compilation/review below — gated on a
723
+ # non-empty diff_content — would be skipped, leaving every criterion unproven and
724
+ # wrongly blocking correct, already-committed code.
725
+ if not diff_content.strip():
726
+ committed_diff = self._committed_task_diff(task.id)
727
+ if committed_diff.strip():
728
+ diff_content = committed_diff
672
729
  diff_empty = not bool(diff_content.strip())
673
- # "Work present" is broader than the current working-tree diff: a task whose
674
- # changes were already committed (e.g. `dev go`'s per-task commit, then the
675
- # final reconciliation pass where `git diff HEAD` is empty) still counts as
676
- # implemented. A genuine no-op run has neither a working diff nor committed
677
- # changes since the task's checkpoint.
678
- work_present = (not diff_empty) or self._task_produced_changes(task.id)
679
-
680
- # Empty-diff guard. If the task declares files to create or modify but produced
681
- # NO work at all, there is nothing to prove an agent must not be able to
682
- # declare victory having written nothing (or after a transient git error that
683
- # degraded the diff to ""). This is the single most dangerous false-pass for
684
- # autonomy, so it blocks regardless of which commands ran.
685
- expects_change = any(pf.allowed_change != "read_only" for pf in task.planned_files)
686
- if not work_present and expects_change:
687
- gaps.append(Gap(
688
- id=self._next_gap_id(task.id, "NODIFF"),
689
- severity="high",
690
- gap_type="task_not_implemented",
691
- task_id=task.id,
692
- description=(
693
- f"Task {task.id} declares files to create or modify, but produced no "
694
- "changes. Verification cannot prove work that does not exist."
695
- ),
696
- evidence=[f"planned files expecting change: {sorted(p.path for p in task.planned_files if p.allowed_change != 'read_only')}"],
697
- recommended_fix=(
698
- "Implement the planned changes so the diff is non-empty, then re-verify. "
699
- "If you did make changes, ensure they are saved and visible to git "
700
- "(not reverted, stashed, or written outside the project root)."
701
- ),
702
- blocking=True,
703
- ))
704
-
705
- if diff_content:
706
- added_files, deleted_files = self._classify_change_paths(changed_files)
707
- diff_ev = DiffEvidence(
708
- task_id=task.id,
709
- changed_files=changed_files,
710
- added_files=added_files,
711
- deleted_files=deleted_files,
712
- diff_summary=f"Diff captured for {len(changed_files)} files."
730
+ # Launch the two independent LLM passes acceptance compilation and the advisory
731
+ # implementation review concurrently as soon as the diff is available, instead
732
+ # of awaiting them sequentially later. Each depends only on (task, requirements,
733
+ # diff_content), so there is no data hazard; each result is awaited (with its
734
+ # existing try/except) at the point it is consumed below. The create-time guards
735
+ # match the consume-time guards exactly, so every task created is always awaited.
736
+ compile_future: Optional["asyncio.Task[Dict[str, List[str]]]"] = None
737
+ if self.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
738
+ # Prefer the self-consistency interface; fall back to single-shot ``compile`` so
739
+ # older compiler doubles/implementations keep working.
740
+ if hasattr(self.acceptance_compiler, "compile_candidates"):
741
+ _compile_coro = self.acceptance_compiler.compile_candidates(
742
+ task, requirements, diff_content, samples=ac_samples,
743
+ per_criterion=ac_per_criterion,
744
+ )
745
+ else:
746
+ _compile_coro = self.acceptance_compiler.compile(task, requirements, diff_content)
747
+ compile_future = asyncio.create_task(_compile_coro)
748
+ review_future: Optional["asyncio.Task[Any]"] = None
749
+ if self.reviewer and diff_content:
750
+ review_future = asyncio.create_task(
751
+ self.reviewer.review_changes(task, requirements, diff_content)
713
752
  )
714
- evidence_to_save.append(diff_ev)
715
-
716
- # 1. Planned-file coverage check
717
- planned_paths = {pf.path for pf in task.planned_files}
718
- changed_set = set(changed_files)
719
- for pf in task.planned_files:
720
- if pf.path not in changed_set and pf.allowed_change != "read_only":
721
- gaps.append(Gap(
722
- id=self._next_gap_id(task.id, "FILE"),
723
- severity="medium",
724
- gap_type="planned_file_not_changed",
725
- task_id=task.id,
726
- description=f"Planned file {pf.path} was not modified.",
727
- recommended_fix=f"Modify {pf.path} as planned or update the task.",
728
- blocking=False,
729
- file=pf.path,
730
- ))
731
-
732
- # 2. Orphan-diff detection
733
- for cf in changed_files:
734
- if cf not in planned_paths:
753
+ try:
754
+ # "Work present" is broader than the current working-tree diff: a task whose
755
+ # changes were already committed (e.g. `dev go`'s per-task commit, then the
756
+ # final reconciliation pass where `git diff HEAD` is empty) still counts as
757
+ # implemented. A genuine no-op run has neither a working diff nor committed
758
+ # changes since the task's checkpoint.
759
+ work_present = (not diff_empty) or self._task_produced_changes(task.id)
760
+
761
+ # Empty-diff guard. If the task declares files to create or modify but produced
762
+ # NO work at all, there is nothing to prove — an agent must not be able to
763
+ # declare victory having written nothing (or after a transient git error that
764
+ # degraded the diff to ""). This is the single most dangerous false-pass for
765
+ # autonomy, so it blocks regardless of which commands ran.
766
+ expects_change = any(pf.allowed_change != "read_only" for pf in task.planned_files)
767
+ if not work_present and expects_change:
735
768
  gaps.append(Gap(
736
- id=self._next_gap_id(task.id, "ORPHAN"),
769
+ id=self._next_gap_id(task.id, "NODIFF"),
737
770
  severity="high",
738
- gap_type="orphan_diff",
771
+ gap_type="task_not_implemented",
739
772
  task_id=task.id,
740
- description=f"File {cf} was modified but not planned for this task.",
741
- evidence=[cf],
742
- recommended_fix=f"Revert changes to {cf} or add it to the task's planned files.",
773
+ description=(
774
+ f"Task {task.id} declares files to create or modify, but produced no "
775
+ "changes. Verification cannot prove work that does not exist."
776
+ ),
777
+ evidence=[f"planned files expecting change: {sorted(p.path for p in task.planned_files if p.allowed_change != 'read_only')}"],
778
+ recommended_fix=(
779
+ "Implement the planned changes so the diff is non-empty, then re-verify. "
780
+ "If you did make changes, ensure they are saved and visible to git "
781
+ "(not reverted, stashed, or written outside the project root)."
782
+ ),
743
783
  blocking=True,
744
- file=cf,
745
784
  ))
746
785
 
747
- gaps.extend(self._check_semantic_diff(task))
748
-
749
- # 3. Dependency change detection
750
- dep_changes = self._check_dependency_changes(changed_files)
751
- for dep_file in dep_changes:
752
- if dep_file not in planned_paths:
753
- gaps.append(Gap(
754
- id=self._next_gap_id(task.id, "DEP"),
755
- severity="high",
756
- gap_type="dependency_risk",
786
+ if diff_content:
787
+ added_files, deleted_files = self._classify_change_paths(changed_files)
788
+ diff_ev = DiffEvidence(
757
789
  task_id=task.id,
758
- description=f"Dependency file {dep_file} was modified without being in planned files.",
759
- evidence=[dep_file],
760
- recommended_fix=f"Justify the dependency change or revert {dep_file}.",
761
- blocking=True,
762
- file=dep_file,
763
- ))
790
+ changed_files=changed_files,
791
+ added_files=added_files,
792
+ deleted_files=deleted_files,
793
+ diff_summary=f"Diff captured for {len(changed_files)} files."
794
+ )
795
+ evidence_to_save.append(diff_ev)
764
796
 
765
- # When DevCouncil can compile its own per-criterion checks, THOSE are the
766
- # authority and the planner's expected_tests are demoted to advisory — so a
767
- # bogus planner command (irrelevant linters, npm on a Python project, tests
768
- # that reference missing files) can no longer block correct work.
769
- compiler_active = bool(self.acceptance_compiler and diff_content and task.acceptance_criterion_ids)
770
-
771
- # 4. Run verification commands
772
- command_results: List[CommandResult] = []
773
- evidence_results: List[CommandResult] = []
774
- genuine_failure = False # a command that actually ran and failed (real defect signal)
775
- had_unrunnable = False # a command that could not run (missing tool / missing tests)
776
- # Genuine test failures demoted to non-blocking only because a compiler is active.
777
- # That demotion is legitimate ONLY if the compiler actually produces per-criterion
778
- # checks to take authority; re-promoted below if it produces none.
779
- demoted_failures: List[Gap] = []
780
- for cmd_type, cmds in self._commands_for_task(task).items():
781
- for cmd in cmds:
782
- applicable, skip_reason = self._command_applicable(cmd)
783
- if not applicable:
784
- # Wrong-stack command (e.g. `npm test` on a Python repo): skip it
785
- # entirely rather than running and failing for a stack reason — an
786
- # advisory note so the skip is visible (no silent drop).
797
+ # 1. Planned-file coverage check
798
+ planned_paths = {pf.path for pf in task.planned_files}
799
+ changed_set = set(changed_files)
800
+ for pf in task.planned_files:
801
+ if pf.path not in changed_set and pf.allowed_change != "read_only":
787
802
  gaps.append(Gap(
788
- id=self._next_gap_id(task.id, "SKIP"),
789
- severity="low",
790
- gap_type="skipped_verification_command",
803
+ id=self._next_gap_id(task.id, "FILE"),
804
+ severity="medium",
805
+ gap_type="planned_file_not_changed",
791
806
  task_id=task.id,
792
- description=f"Skipped verification command '{cmd}': {skip_reason}.",
793
- evidence=[skip_reason],
794
- recommended_fix=(
795
- "Replace it with a command for this repo's stack, or remove it "
796
- "from .devcouncil/config.yaml / the task's expected_tests."
797
- ),
807
+ description=f"Planned file {pf.path} was not modified.",
808
+ recommended_fix=f"Modify {pf.path} as planned or update the task.",
798
809
  blocking=False,
799
- suggested_command=cmd,
810
+ file=pf.path,
800
811
  ))
801
- continue
802
- result = self._run_command(cmd, task_id=task.id)
803
- command_results.append(result)
804
- evidence_to_save.append(result)
805
- if self._command_can_prove_acceptance(cmd_type, cmd):
806
- evidence_results.append(result)
807
- if result.exit_code != 0:
808
- if self._command_is_malformed(result):
809
- had_unrunnable = True
810
- # The verification command itself could not run (e.g. a
811
- # SyntaxError in a `python -c` one-liner, or a missing test
812
- # tool). This proves nothing about the implementation, so do
813
- # not report it as a code failure — surface it as a plan/
814
- # command defect the user can regenerate instead.
812
+
813
+ # 2. Orphan-diff detection
814
+ for cf in changed_files:
815
+ if cf not in planned_paths:
816
+ gaps.append(Gap(
817
+ id=self._next_gap_id(task.id, "ORPHAN"),
818
+ severity="high",
819
+ gap_type="orphan_diff",
820
+ task_id=task.id,
821
+ description=f"File {cf} was modified but not planned for this task.",
822
+ evidence=[cf],
823
+ recommended_fix=f"Revert changes to {cf} or add it to the task's planned files.",
824
+ blocking=True,
825
+ file=cf,
826
+ ))
827
+
828
+ gaps.extend(self._check_semantic_diff(task, requirements))
829
+
830
+ # 3. Dependency change detection
831
+ dep_changes = self._check_dependency_changes(changed_files)
832
+ for dep_file in dep_changes:
833
+ if dep_file not in planned_paths:
834
+ gaps.append(Gap(
835
+ id=self._next_gap_id(task.id, "DEP"),
836
+ severity="high",
837
+ gap_type="dependency_risk",
838
+ task_id=task.id,
839
+ description=f"Dependency file {dep_file} was modified without being in planned files.",
840
+ evidence=[dep_file],
841
+ recommended_fix=f"Justify the dependency change or revert {dep_file}.",
842
+ blocking=True,
843
+ file=dep_file,
844
+ ))
845
+
846
+ # When DevCouncil can compile its own per-criterion checks, THOSE are the
847
+ # authority and the planner's expected_tests are demoted to advisory — so a
848
+ # bogus planner command (irrelevant linters, npm on a Python project, tests
849
+ # that reference missing files) can no longer block correct work.
850
+ compiler_active = bool(self.acceptance_compiler and diff_content and task.acceptance_criterion_ids)
851
+
852
+ # 4. Run verification commands
853
+ command_results: List[CommandResult] = []
854
+ evidence_results: List[CommandResult] = []
855
+ genuine_failure = False # a command that actually ran and failed (real defect signal)
856
+ had_unrunnable = False # a command that could not run (missing tool / missing tests)
857
+ # Genuine test failures demoted to non-blocking only because a compiler is active.
858
+ # That demotion is legitimate ONLY if the compiler actually produces per-criterion
859
+ # checks to take authority; re-promoted below if it produces none.
860
+ demoted_failures: List[Gap] = []
861
+ for cmd_type, cmds in self._commands_for_task(task).items():
862
+ for cmd in cmds:
863
+ applicable, skip_reason = self._command_applicable(cmd)
864
+ if not applicable:
865
+ # Wrong-stack command (e.g. `npm test` on a Python repo): skip it
866
+ # entirely rather than running and failing for a stack reason — an
867
+ # advisory note so the skip is visible (no silent drop).
815
868
  gaps.append(Gap(
816
- id=self._next_gap_id(task.id, "BADCMD"),
817
- severity="medium",
818
- gap_type="invalid_verification_command",
869
+ id=self._next_gap_id(task.id, "SKIP"),
870
+ severity="low",
871
+ gap_type="skipped_verification_command",
819
872
  task_id=task.id,
820
- description=(
821
- f"Verification command could not run (not a code failure): '{cmd}'. "
822
- "It appears malformed or its tooling is unavailable, so this command "
823
- "proves nothing either way."
824
- ),
825
- evidence=[result.summary[:500]],
873
+ description=f"Skipped verification command '{cmd}': {skip_reason}.",
874
+ evidence=[skip_reason],
826
875
  recommended_fix=(
827
- "Regenerate the task's verification commands with 'dev repair', or edit "
828
- "them to be a single runnable command (e.g. 'python -m pytest <file>')."
876
+ "Replace it with a command for this repo's stack, or remove it "
877
+ "from .devcouncil/config.yaml / the task's expected_tests."
829
878
  ),
830
- # Non-blocking: a command that cannot run is not evidence of a
831
- # defect. If it was the *only* check for an acceptance criterion,
832
- # that criterion is independently caught as unproven (blocking).
833
879
  blocking=False,
834
880
  suggested_command=cmd,
835
- stdout_path=result.stdout_path or None,
836
- stderr_path=result.stderr_path or None,
837
881
  ))
838
- else:
839
- # A verification command that genuinely failed. Lint/typecheck
840
- # commands (from the config fallback) report style/type opinion,
841
- # not a correctness defect, so they are ADVISORY — blocking a
842
- # behaviorally-correct task on `flake8`/`mypy`/`ruff` is the
843
- # false-block the benchmark surfaced. A real test failure still
844
- # gates (unless compiled checks supersede it).
845
- is_quality_gate = cmd_type in {"lint", "typecheck"} or self._is_quality_only_command(cmd)
846
- blocking = (not compiler_active) and not is_quality_gate
847
- if blocking:
848
- genuine_failure = True
849
- fail_file, fail_line = self._failure_location(result)
850
- gap = Gap(
851
- id=self._next_gap_id(task.id, cmd_type.upper()),
852
- severity="high" if blocking else "medium",
853
- gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
854
- task_id=task.id,
855
- description=(
856
- f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
857
- f"failed with exit code {result.exit_code}"
858
- + (" (advisory: style/type, not a correctness gate)." if is_quality_gate else ".")
859
- ),
860
- evidence=[result.summary[:500]],
861
- recommended_fix=f"Fix the issues reported by '{cmd}'.",
862
- blocking=blocking,
863
- suggested_command=cmd,
864
- file=fail_file,
865
- line=fail_line,
866
- stdout_path=result.stdout_path or None,
867
- stderr_path=result.stderr_path or None,
868
- )
869
- gaps.append(gap)
870
- # A real test failure demoted only because the compiler is active:
871
- # remember it so we can re-promote if the compiler yields no checks.
872
- if compiler_active and not is_quality_gate and not blocking:
873
- demoted_failures.append(gap)
874
-
875
- # 4b. Compiled acceptance checks — precise, DevCouncil-owned per-criterion
876
- # evidence. Derive one runnable check per acceptance criterion from the
877
- # criterion text + the diff, instead of trusting planner-authored
878
- # expected_tests (which the benchmark showed often reference absent tools or
879
- # test files). Each check maps 1:1 to its criterion, replacing the coarse
880
- # "any command passed -> every criterion proven" mapping.
881
- compiled_pass: Dict[str, bool] = {}
882
- # Per-AC bookkeeping so the unproven-AC gap can attach ONLY the check(s) that
883
- # targeted that criterion (and the specific failing result), instead of dumping
884
- # every command summary. Keys are AC ids; values track the compiled command(s)
885
- # and any failing CommandResults for that AC.
886
- compiled_cmds_by_ac: Dict[str, List[str]] = {}
887
- failing_results_by_ac: Dict[str, List[CommandResult]] = {}
888
- if self.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
889
- try:
890
- compiled = await self.acceptance_compiler.compile(task, requirements, diff_content)
891
- except Exception as exc: # pragma: no cover - best effort
892
- logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
893
- compiled = {}
894
- for ac_id, cmds in compiled.items():
895
- # Defensive: drop any wrong-stack compiled check so it can't fail an AC
896
- # for a stack reason (the compiler is told not to emit these).
897
- cmds = [c for c in cmds if self._command_applicable(c)[0]]
898
- ac_ok = bool(cmds)
899
- compiled_cmds_by_ac[ac_id] = list(cmds)
900
- for cmd in cmds:
882
+ continue
901
883
  result = self._run_command(cmd, task_id=task.id)
902
884
  command_results.append(result)
903
885
  evidence_to_save.append(result)
886
+ if self._command_can_prove_acceptance(cmd_type, cmd):
887
+ evidence_results.append(result)
904
888
  if result.exit_code != 0:
905
- ac_ok = False
906
- failing_results_by_ac.setdefault(ac_id, []).append(result)
907
889
  if self._command_is_malformed(result):
908
890
  had_unrunnable = True
891
+ # The verification command itself could not run (e.g. a
892
+ # SyntaxError in a `python -c` one-liner, or a missing test
893
+ # tool). This proves nothing about the implementation, so do
894
+ # not report it as a code failure — surface it as a plan/
895
+ # command defect the user can regenerate instead.
896
+ gaps.append(Gap(
897
+ id=self._next_gap_id(task.id, "BADCMD"),
898
+ severity="medium",
899
+ gap_type="invalid_verification_command",
900
+ task_id=task.id,
901
+ description=(
902
+ f"Verification command could not run (not a code failure): '{cmd}'. "
903
+ "It appears malformed or its tooling is unavailable, so this command "
904
+ "proves nothing either way."
905
+ ),
906
+ evidence=[result.summary[:500]],
907
+ recommended_fix=(
908
+ "Regenerate the task's verification commands with 'dev repair', or edit "
909
+ "them to be a single runnable command (e.g. 'python -m pytest <file>')."
910
+ ),
911
+ # Non-blocking: a command that cannot run is not evidence of a
912
+ # defect. If it was the *only* check for an acceptance criterion,
913
+ # that criterion is independently caught as unproven (blocking).
914
+ blocking=False,
915
+ suggested_command=cmd,
916
+ stdout_path=result.stdout_path or None,
917
+ stderr_path=result.stderr_path or None,
918
+ ))
909
919
  else:
910
- genuine_failure = True
920
+ # A verification command that genuinely failed. Lint/typecheck
921
+ # commands (from the config fallback) report style/type opinion,
922
+ # not a correctness defect, so they are ADVISORY — blocking a
923
+ # behaviorally-correct task on `flake8`/`mypy`/`ruff` is the
924
+ # false-block the benchmark surfaced. A real test failure still
925
+ # gates (unless compiled checks supersede it).
926
+ is_quality_gate = cmd_type in {"lint", "typecheck"} or self._is_quality_only_command(cmd)
927
+ blocking = (not compiler_active) and not is_quality_gate
928
+ if blocking:
929
+ genuine_failure = True
911
930
  fail_file, fail_line = self._failure_location(result)
912
- gaps.append(Gap(
913
- id=self._next_gap_id(task.id, "ACCHK"),
914
- severity="high",
915
- gap_type="test_failed",
931
+ gap = Gap(
932
+ id=self._next_gap_id(task.id, cmd_type.upper()),
933
+ severity="high" if blocking else "medium",
934
+ gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
916
935
  task_id=task.id,
917
- description=f"Acceptance check for {ac_id} failed: '{cmd}' (exit {result.exit_code}).",
936
+ description=(
937
+ f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
938
+ f"failed with exit code {result.exit_code}"
939
+ + (" (advisory: style/type, not a correctness gate)." if is_quality_gate else ".")
940
+ ),
918
941
  evidence=[result.summary[:500]],
919
- recommended_fix=f"Fix the implementation so acceptance criterion {ac_id} holds.",
920
- blocking=True,
921
- acceptance_criterion_id=ac_id,
942
+ recommended_fix=f"Fix the issues reported by '{cmd}'.",
943
+ blocking=blocking,
922
944
  suggested_command=cmd,
923
945
  file=fail_file,
924
946
  line=fail_line,
925
947
  stdout_path=result.stdout_path or None,
926
948
  stderr_path=result.stderr_path or None,
927
- ))
928
- compiled_pass[ac_id] = ac_ok
929
-
930
- # The compiler only earns the authority to demote a genuinely-failing planner
931
- # test if it produced a per-criterion check for EVERY targeted AC. A partial
932
- # compile is not enough: the uncovered ACs fall back to the coarse signal, so a
933
- # demoted real failure + coarse-proven remainder would otherwise slip past the
934
- # gate. If coverage is incomplete (or zero empty compile / all-wrong-stack /
935
- # a compile exception swallowed to {}), re-promote the demoted failures.
936
- compiler_covered_all = bool(task.acceptance_criterion_ids) and all(
937
- compiled_cmds_by_ac.get(ac_id) for ac_id in task.acceptance_criterion_ids
938
- )
939
- if compiler_active and not compiler_covered_all and demoted_failures:
940
- for gap in demoted_failures:
941
- gap.blocking = True
942
- gap.severity = "high"
943
- genuine_failure = True
944
- logger.info(
945
- "Re-promoted demoted test failure %s to blocking: acceptance compiler "
946
- "did not produce a check for every criterion of task %s.",
947
- gap.id, task.id,
948
- )
949
-
950
- # 5. Acceptance-criteria evidence mapping (precise, per criterion).
951
- # Quality-only commands (lint/typecheck) are excluded: a passing `mypy`/`ruff
952
- # check`/`tsc` exercises no behavior, so it must not coarse-prove a behavioral AC
953
- # — the same false-confidence the per-criterion checks exist to prevent.
954
- successful_commands = [
955
- result for result in evidence_results
956
- if result.exit_code == 0 and not self._is_quality_only_command(result.command)
957
- ]
958
- # Coarse fallback (used only when no compiled per-criterion check exists for an
959
- # AC): a criterion may be marked proven by a passing acceptance-capable command
960
- # ONLY when the task actually produced work. Without this guard a no-op run
961
- # whose unrelated command happens to pass would "prove" every criterion against
962
- # zero changes.
963
- coarse_proof_available = work_present and bool(successful_commands)
964
- if task.acceptance_criterion_ids:
965
- req_by_ac = {ac.id: req.id for req in requirements for ac in req.acceptance_criteria}
966
- unproven_acs: List[str] = []
967
- coarse_proven_acs: List[str] = []
968
- for ac_id in task.acceptance_criterion_ids:
969
- # An AC is proven if its compiled check passed; if no compiled check
970
- # exists for it, fall back to the coarse signal (any expected_test passed).
971
- proven = compiled_pass.get(ac_id)
972
- coarse = False
973
- if proven is None:
974
- proven = coarse_proof_available
975
- coarse = proven # proven only by the coarse, not-AC-specific signal
976
- if proven:
977
- if coarse:
978
- coarse_proven_acs.append(ac_id)
979
- # Don't persist a "passed" record for a coarse-proven criterion during a
980
- # run that also has a genuine blocking failure — the gate already fails,
981
- # and a stored "passed" would mislead audits that read evidence directly.
982
- if not (coarse and genuine_failure):
983
- evidence_to_save.append(TestEvidence(
984
- requirement_id=req_by_ac.get(ac_id, task.requirement_ids[0] if task.requirement_ids else ""),
949
+ )
950
+ gaps.append(gap)
951
+ # A real test failure demoted only because the compiler is active:
952
+ # remember it so we can re-promote if the compiler yields no checks.
953
+ if compiler_active and not is_quality_gate and not blocking:
954
+ demoted_failures.append(gap)
955
+
956
+ # 4b. Compiled acceptance checksprecise, DevCouncil-owned per-criterion
957
+ # evidence. Derive one runnable check per acceptance criterion from the
958
+ # criterion text + the diff, instead of trusting planner-authored
959
+ # expected_tests (which the benchmark showed often reference absent tools or
960
+ # test files). Each check maps 1:1 to its criterion, replacing the coarse
961
+ # "any command passed -> every criterion proven" mapping.
962
+ compiled_pass: Dict[str, bool] = {}
963
+ # Per-AC bookkeeping so the unproven-AC gap can attach ONLY the check(s) that
964
+ # targeted that criterion (and the specific failing result), instead of dumping
965
+ # every command summary. Keys are AC ids; values track the compiled command(s)
966
+ # and any failing CommandResults for that AC.
967
+ compiled_cmds_by_ac: Dict[str, List[str]] = {}
968
+ failing_results_by_ac: Dict[str, List[CommandResult]] = {}
969
+ # Per-AC vote tally for proven criteria: {ac_id: (passes, decisive, repaired)}.
970
+ # Recorded into the stored TestEvidence so an audit can see HOW a criterion was
971
+ # proven (single check vs. majority of independent checks; whether a check had to
972
+ # be repaired to run) instead of just "passed".
973
+ compiled_vote: Dict[str, Tuple[int, int, bool]] = {}
974
+ # ACs whose independently-generated checks split (some pass, some fail) with no
975
+ # majority. Per policy this is inconclusive neither proof nor a defect so the
976
+ # AC is surfaced NON-blocking below instead of false-blocking on a lone bad check.
977
+ inconclusive_acs: set[str] = set()
978
+ if compile_future is not None:
979
+ try:
980
+ compiled = await compile_future
981
+ except Exception as exc: # pragma: no cover - best effort
982
+ logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
983
+ compiled = {}
984
+ ac_meta = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
985
+ for ac_id, raw_cmds in compiled.items():
986
+ # Defensive: drop any wrong-stack candidate so it can't fail an AC for a
987
+ # stack reason (the compiler is told not to emit these).
988
+ candidates = [c for c in raw_cmds if self._command_applicable(c)[0]]
989
+ compiled_cmds_by_ac[ac_id] = list(candidates)
990
+ if not candidates:
991
+ compiled_pass[ac_id] = False
992
+ continue
993
+ # Run each INDEPENDENT candidate; a check that merely failed to RUN
994
+ # (malformed/unrunnable) is regenerated from the launcher error up to
995
+ # ``ac_repair_attempts`` times — safe, because a check that never ran
996
+ # proves nothing, so repairing it cannot weaken the gate.
997
+ passes = 0
998
+ genuine_fails = 0
999
+ repaired = False # a check had to be regenerated before it ran
1000
+ fail_results: List[Tuple[str, CommandResult]] = []
1001
+ for cmd in candidates:
1002
+ result = self._run_command(cmd, task_id=task.id)
1003
+ command_results.append(result)
1004
+ evidence_to_save.append(result)
1005
+ attempts = 0
1006
+ _repair = getattr(self.acceptance_compiler, "repair", None)
1007
+ while (
1008
+ result.exit_code != 0
1009
+ and self._command_is_malformed(result)
1010
+ and attempts < ac_repair_attempts
1011
+ and _repair is not None
1012
+ ):
1013
+ attempts += 1
1014
+ ac_desc = ac_meta[ac_id].description if ac_id in ac_meta else ac_id
1015
+ try:
1016
+ fixed = await _repair(
1017
+ ac_id, ac_desc, cmd, result.summary[:800], diff_content
1018
+ )
1019
+ except Exception:
1020
+ fixed = None
1021
+ if not fixed or not self._command_applicable(fixed)[0]:
1022
+ break
1023
+ cmd = fixed
1024
+ compiled_cmds_by_ac[ac_id].append(cmd)
1025
+ result = self._run_command(cmd, task_id=task.id)
1026
+ command_results.append(result)
1027
+ evidence_to_save.append(result)
1028
+ if result.exit_code == 0:
1029
+ passes += 1
1030
+ if attempts > 0:
1031
+ repaired = True
1032
+ elif self._command_is_malformed(result):
1033
+ # Still couldn't run after repair: proves nothing either way.
1034
+ had_unrunnable = True
1035
+ failing_results_by_ac.setdefault(ac_id, []).append(result)
1036
+ else:
1037
+ genuine_fails += 1
1038
+ fail_results.append((cmd, result))
1039
+ failing_results_by_ac.setdefault(ac_id, []).append(result)
1040
+ decisive = passes + genuine_fails
1041
+ # Majority vote over the checks that actually ran. Proven iff a strict
1042
+ # majority pass; unanimous failure of independent checks is strong evidence
1043
+ # of a real defect and blocks; a split is inconclusive (handled below).
1044
+ ac_proven = decisive > 0 and passes > genuine_fails
1045
+ compiled_pass[ac_id] = ac_proven
1046
+ if ac_proven:
1047
+ compiled_vote[ac_id] = (passes, decisive, repaired)
1048
+ continue
1049
+ if passes == 0 and genuine_fails > 0:
1050
+ genuine_failure = True
1051
+ cmd, result = fail_results[0]
1052
+ fail_file, fail_line = self._failure_location(result)
1053
+ agree = (
1054
+ f" {genuine_fails}/{decisive} independent checks agreed it fails."
1055
+ if decisive > 1 else ""
1056
+ )
1057
+ gaps.append(Gap(
1058
+ id=self._next_gap_id(task.id, "ACCHK"),
1059
+ severity="high",
1060
+ gap_type="test_failed",
1061
+ task_id=task.id,
1062
+ description=f"Acceptance check for {ac_id} failed: '{cmd}' (exit {result.exit_code}).{agree}",
1063
+ evidence=[result.summary[:500]],
1064
+ recommended_fix=f"Fix the implementation so acceptance criterion {ac_id} holds.",
1065
+ blocking=True,
985
1066
  acceptance_criterion_id=ac_id,
986
- command="(devcouncil acceptance check)",
987
- status="passed",
988
- evidence_summary=(
989
- "Acceptance criterion proven only by a COARSE signal (a passing "
990
- "acceptance-capable command, not a per-criterion check); behavior "
991
- "not precisely verified."
992
- if coarse else
993
- "Acceptance criterion proven by a per-criterion compiled check."
994
- ),
1067
+ suggested_command=cmd,
1068
+ file=fail_file,
1069
+ line=fail_line,
1070
+ stdout_path=result.stdout_path or None,
1071
+ stderr_path=result.stderr_path or None,
995
1072
  ))
996
- else:
997
- unproven_acs.append(ac_id)
998
- # Surface coarse proof as a first-class advisory: these criteria passed only
999
- # because some acceptance-capable command exited 0, not because a check tied
1000
- # to the criterion passed. Non-blocking, but no longer invisible.
1001
- if coarse_proven_acs:
1002
- gaps.append(Gap(
1003
- id=self._next_gap_id(task.id, "COARSE"),
1004
- severity="low",
1005
- gap_type="coarse_acceptance_proof",
1006
- task_id=task.id,
1007
- description=(
1008
- "Verification mode = COARSE for "
1009
- f"{', '.join(coarse_proven_acs)}: proven by a passing acceptance-capable "
1010
- "command, not a per-criterion check. Behavior is not precisely verified."
1011
- ),
1012
- evidence=[f"coarse-proven: {', '.join(coarse_proven_acs)}"],
1013
- recommended_fix=(
1014
- "Add a verification command (or test) that exercises each listed criterion "
1015
- "specifically, so DevCouncil can compile a per-criterion check instead of "
1016
- "relying on the coarse fallback."
1017
- ),
1018
- blocking=False,
1019
- ))
1020
- if unproven_acs:
1021
- # Block only on positive evidence of a problem. If verification was
1022
- # attempted but every failure was unrunnable (missing tooling / tests)
1023
- # and nothing genuinely failed, that is a verification defect, not a
1024
- # code defect surface it as a non-blocking "could not verify".
1025
- couldnt_verify = had_unrunnable and not genuine_failure and work_present
1026
- ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
1027
- # Methods that can be proven by running code; only these block the gate
1028
- # when unproven. Inherently-manual criteria (manual/llm_review) and
1029
- # optional ones are surfaced for human review instead of false-blocking
1030
- # the autonomous loop the gate still demands evidence for BEHAVIOR.
1031
- automatable_methods = {"unit_test", "integration_test", "static_check"}
1032
- for ac_id in unproven_acs:
1033
- ac = ac_by_id.get(ac_id)
1034
- method = ac.verification_method if ac else "unit_test"
1035
- is_automatable = (ac.required if ac else True) and method in automatable_methods
1036
- if not is_automatable:
1037
- blocks = False
1038
- optional = "" if (ac is None or ac.required) else " optional"
1039
- fix = (
1040
- f"This{optional} criterion's verification method is '{method}'; it cannot be "
1041
- "proven by running code. Review it manually (it does not block the gate)."
1042
- )
1043
- suffix = f" (non-blocking: {method})"
1044
- elif couldnt_verify:
1045
- blocks = False
1046
- fix = ("Could not verify this criterion: the verification commands did not run "
1047
- "(missing tooling or tests). Regenerate them with 'dev repair' to confirm the work.")
1048
- suffix = " (verification commands could not run)"
1049
- else:
1050
- blocks = True
1051
- fix = "Add or fix a verification command that proves this acceptance criterion."
1052
- suffix = ""
1053
- # Concrete, AC-scoped evidence instead of "all command summaries":
1054
- # * if a compiled check targeted this AC, attach its command(s) and
1055
- # the specific failing result;
1056
- # * otherwise an explicit "no check compiled" marker so the agent
1057
- # knows it must author one, not hunt through unrelated output.
1058
- ac_compiled = compiled_cmds_by_ac.get(ac_id, [])
1059
- ac_failures = failing_results_by_ac.get(ac_id, [])
1060
- ac_evidence: List[str] = []
1061
- suggested_cmd: Optional[str] = None
1062
- if ac_compiled:
1063
- suggested_cmd = ac_compiled[0]
1064
- ac_evidence.extend(f"compiled check: {c}" for c in ac_compiled)
1065
- ac_evidence.extend(r.summary[:500] for r in ac_failures)
1073
+ elif passes > 0 and genuine_fails > 0:
1074
+ # Independent checks disagree with no majority: neither proof nor a
1075
+ # defect. Mark inconclusive so the unproven-AC gap below is NON-blocking
1076
+ # (never false-block on a lone bad check, never auto-pass a real bug).
1077
+ inconclusive_acs.add(ac_id)
1078
+
1079
+ # The compiler only earns the authority to demote a genuinely-failing planner
1080
+ # test if it produced a per-criterion check for EVERY targeted AC. A partial
1081
+ # compile is not enough: the uncovered ACs fall back to the coarse signal, so a
1082
+ # demoted real failure + coarse-proven remainder would otherwise slip past the
1083
+ # gate. If coverage is incomplete (or zero — empty compile / all-wrong-stack /
1084
+ # a compile exception swallowed to {}), re-promote the demoted failures.
1085
+ compiler_covered_all = bool(task.acceptance_criterion_ids) and all(
1086
+ compiled_cmds_by_ac.get(ac_id) for ac_id in task.acceptance_criterion_ids
1087
+ )
1088
+ if compiler_active and not compiler_covered_all and demoted_failures:
1089
+ for gap in demoted_failures:
1090
+ gap.blocking = True
1091
+ gap.severity = "high"
1092
+ genuine_failure = True
1093
+ logger.info(
1094
+ "Re-promoted demoted test failure %s to blocking: acceptance compiler "
1095
+ "did not produce a check for every criterion of task %s.",
1096
+ gap.id, task.id,
1097
+ )
1098
+
1099
+ # 5. Acceptance-criteria evidence mapping (precise, per criterion).
1100
+ # Quality-only commands (lint/typecheck) are excluded: a passing `mypy`/`ruff
1101
+ # check`/`tsc` exercises no behavior, so it must not coarse-prove a behavioral AC
1102
+ # the same false-confidence the per-criterion checks exist to prevent.
1103
+ successful_commands = [
1104
+ result for result in evidence_results
1105
+ if result.exit_code == 0 and not self._is_quality_only_command(result.command)
1106
+ ]
1107
+ # Coarse fallback (used only when no compiled per-criterion check exists for an
1108
+ # AC): a criterion may be marked proven by a passing acceptance-capable command
1109
+ # ONLY when the task actually produced work. Without this guard a no-op run
1110
+ # whose unrelated command happens to pass would "prove" every criterion against
1111
+ # zero changes.
1112
+ coarse_proof_available = work_present and bool(successful_commands)
1113
+ if task.acceptance_criterion_ids:
1114
+ req_by_ac = {ac.id: req.id for req in requirements for ac in req.acceptance_criteria}
1115
+ unproven_acs: List[str] = []
1116
+ coarse_proven_acs: List[str] = []
1117
+ for ac_id in task.acceptance_criterion_ids:
1118
+ # An AC is proven if its compiled check passed; if no compiled check
1119
+ # exists for it, fall back to the coarse signal (any expected_test passed).
1120
+ proven: Optional[bool] = compiled_pass.get(ac_id)
1121
+ coarse = False
1122
+ if proven is None:
1123
+ proven = coarse_proof_available
1124
+ coarse = proven # proven only by the coarse, not-AC-specific signal
1125
+ if proven:
1126
+ if coarse:
1127
+ coarse_proven_acs.append(ac_id)
1128
+ # Don't persist a "passed" record for a coarse-proven criterion during a
1129
+ # run that also has a genuine blocking failure — the gate already fails,
1130
+ # and a stored "passed" would mislead audits that read evidence directly.
1131
+ if not (coarse and genuine_failure):
1132
+ proof_mode: Literal["compiled", "vote", "coarse", ""]
1133
+ if coarse:
1134
+ proof_summary = (
1135
+ "Acceptance criterion proven only by a COARSE signal (a passing "
1136
+ "acceptance-capable command, not a per-criterion check); behavior "
1137
+ "not precisely verified."
1138
+ )
1139
+ proof_mode = "coarse"
1140
+ else:
1141
+ # Make the per-criterion proof auditable: single check vs. majority
1142
+ # of independent checks, and whether a check had to be repaired to run.
1143
+ passes_n, decisive_n, was_repaired = compiled_vote.get(ac_id, (1, 1, False))
1144
+ proof_mode = "vote" if decisive_n > 1 else "compiled"
1145
+ how = (
1146
+ f"a majority vote of independent compiled checks ({passes_n}/{decisive_n} passed)"
1147
+ if decisive_n > 1 else
1148
+ "a per-criterion compiled check"
1149
+ )
1150
+ repaired_note = " (one check was regenerated from its launcher error to run)" if was_repaired else ""
1151
+ proof_summary = f"Acceptance criterion proven by {how}.{repaired_note}"
1152
+ evidence_to_save.append(TestEvidence(
1153
+ requirement_id=req_by_ac.get(ac_id, task.requirement_ids[0] if task.requirement_ids else ""),
1154
+ acceptance_criterion_id=ac_id,
1155
+ command="(devcouncil acceptance check)",
1156
+ status="passed",
1157
+ evidence_summary=proof_summary,
1158
+ mode=proof_mode,
1159
+ ))
1066
1160
  else:
1067
- ac_evidence.append(
1068
- f"no DevCouncil check compiled for {ac_id} "
1069
- f"(expected verification method: {method})"
1070
- )
1161
+ unproven_acs.append(ac_id)
1162
+ # Surface coarse proof as a first-class advisory: these criteria passed only
1163
+ # because some acceptance-capable command exited 0, not because a check tied
1164
+ # to the criterion passed. Non-blocking, but no longer invisible.
1165
+ if coarse_proven_acs:
1071
1166
  gaps.append(Gap(
1072
- id=self._next_gap_id(task.id, "AC"),
1073
- severity="high" if blocks else "medium",
1074
- gap_type="acceptance_criteria_unproven",
1075
- requirement_id=self._requirement_id_for_ac(requirements, ac_id),
1167
+ id=self._next_gap_id(task.id, "COARSE"),
1168
+ severity="low",
1169
+ gap_type="coarse_acceptance_proof",
1076
1170
  task_id=task.id,
1077
1171
  description=(
1078
- f"Acceptance criterion {ac_id} has no passing verification evidence "
1079
- f"for task {task.id}.{suffix}"
1172
+ "Verification mode = COARSE for "
1173
+ f"{', '.join(coarse_proven_acs)}: proven by a passing acceptance-capable "
1174
+ "command, not a per-criterion check. Behavior is not precisely verified."
1080
1175
  ),
1081
- evidence=ac_evidence,
1082
- recommended_fix=fix,
1083
- blocking=blocks,
1084
- acceptance_criterion_id=ac_id,
1085
- expected_verification_method=method,
1086
- suggested_command=suggested_cmd,
1176
+ evidence=[f"coarse-proven: {', '.join(coarse_proven_acs)}"],
1177
+ recommended_fix=(
1178
+ "Add a verification command (or test) that exercises each listed criterion "
1179
+ "specifically, so DevCouncil can compile a per-criterion check instead of "
1180
+ "relying on the coarse fallback."
1181
+ ),
1182
+ blocking=False,
1087
1183
  ))
1088
- elif task.requirement_ids:
1089
- gaps.append(Gap(
1090
- id=self._next_gap_id(task.id, "NOAC"),
1091
- severity="high",
1092
- gap_type="acceptance_criteria_unproven",
1093
- requirement_id=task.requirement_ids[0],
1094
- task_id=task.id,
1095
- description=f"Task {task.id} is linked to requirements but no acceptance criteria.",
1096
- recommended_fix="Link the task to specific acceptance_criterion_ids before verification.",
1097
- blocking=True,
1098
- ))
1099
-
1100
- # 5b. Diff↔coverage gate. A green suite is only acceptance evidence if it
1101
- # exercised the lines the diff changed. This catches the failure the README
1102
- # promises to stop: tests "pass" while the new logic is never run (unrelated
1103
- # suite, code never imported, untouched branch). Measured only when the target
1104
- # repo has coverage tooling and the diff has measurable Python changes; absent
1105
- # that, it degrades silently rather than blocking correct work.
1106
- measure_cov, enforce_cov, min_ratio = self._diff_coverage_settings()
1107
- any_passing = bool(successful_commands) or any(compiled_pass.values())
1108
- coverage_measured = False
1109
- coverage_skipped_reason: Optional[str] = None
1110
- if not measure_cov:
1111
- coverage_skipped_reason = "diff coverage disabled in config"
1112
- elif not diff_content:
1113
- coverage_skipped_reason = "no diff to measure"
1114
- elif not task.acceptance_criterion_ids:
1115
- coverage_skipped_reason = "task has no acceptance criteria"
1116
- elif not any_passing:
1117
- coverage_skipped_reason = "no passing verification command to instrument"
1118
- if measure_cov and diff_content and task.acceptance_criterion_ids and any_passing:
1119
- cov = self.measure_diff_coverage(task, diff_content)
1120
- if not cov.measured:
1121
- coverage_skipped_reason = cov.reason or "diff coverage could not be measured"
1122
- if cov.measured:
1123
- coverage_measured = True
1124
- coverage_skipped_reason = None
1125
- evidence_to_save.append(DiffCoverageEvidence(
1184
+ if unproven_acs:
1185
+ # Block only on positive evidence of a problem. If verification was
1186
+ # attempted but every failure was unrunnable (missing tooling / tests)
1187
+ # and nothing genuinely failed, that is a verification defect, not a
1188
+ # code defect — surface it as a non-blocking "could not verify".
1189
+ couldnt_verify = had_unrunnable and not genuine_failure and work_present
1190
+ ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
1191
+ # Methods whose criteria HARD-BLOCK the gate when unproven: only those
1192
+ # that assert BEHAVIOR. Inherently-manual criteria (manual/llm_review),
1193
+ # optional ones, and quality-only `static_check` criteria (PEP 8 /
1194
+ # docstring / formatting) are surfaced for review instead of
1195
+ # false-blocking the autonomous loop. static_check is a quality gate,
1196
+ # not a correctness gate mirroring how lint/type COMMAND failures are
1197
+ # already demoted to advisory and the compiler often cannot author a
1198
+ # reliable style check (or the criterion lands on a no-diff process task),
1199
+ # which otherwise blocks correct, style-conforming code.
1200
+ automatable_methods = {"unit_test", "integration_test"}
1201
+ for ac_id in unproven_acs:
1202
+ ac = ac_by_id.get(ac_id)
1203
+ method = ac.verification_method if ac else "unit_test"
1204
+ is_automatable = (ac.required if ac else True) and method in automatable_methods
1205
+ if not is_automatable:
1206
+ blocks = False
1207
+ optional = "" if (ac is None or ac.required) else " optional"
1208
+ fix = (
1209
+ f"This{optional} criterion's verification method is '{method}'; it cannot be "
1210
+ "proven by running code. Review it manually (it does not block the gate)."
1211
+ )
1212
+ suffix = f" (non-blocking: {method})"
1213
+ elif ac_id in inconclusive_acs:
1214
+ # Independently-generated checks split with no majority — inconclusive,
1215
+ # so this does not block (a lone bad check must not fail correct code).
1216
+ blocks = False
1217
+ fix = ("Auto-generated acceptance checks disagreed on this criterion (some "
1218
+ "passed, some failed). Add a precise verification command that "
1219
+ "unambiguously proves it so the result is decisive.")
1220
+ suffix = " (auto-checks inconclusive)"
1221
+ elif couldnt_verify:
1222
+ blocks = False
1223
+ fix = ("Could not verify this criterion: the verification commands did not run "
1224
+ "(missing tooling or tests). Regenerate them with 'dev repair' to confirm the work.")
1225
+ suffix = " (verification commands could not run)"
1226
+ else:
1227
+ blocks = True
1228
+ fix = "Add or fix a verification command that proves this acceptance criterion."
1229
+ suffix = ""
1230
+ # Concrete, AC-scoped evidence instead of "all command summaries":
1231
+ # * if a compiled check targeted this AC, attach its command(s) and
1232
+ # the specific failing result;
1233
+ # * otherwise an explicit "no check compiled" marker so the agent
1234
+ # knows it must author one, not hunt through unrelated output.
1235
+ ac_compiled = compiled_cmds_by_ac.get(ac_id, [])
1236
+ ac_failures = failing_results_by_ac.get(ac_id, [])
1237
+ ac_evidence: List[str] = []
1238
+ suggested_cmd: Optional[str] = None
1239
+ if ac_compiled:
1240
+ suggested_cmd = ac_compiled[0]
1241
+ ac_evidence.extend(f"compiled check: {c}" for c in ac_compiled)
1242
+ ac_evidence.extend(r.summary[:500] for r in ac_failures)
1243
+ else:
1244
+ ac_evidence.append(
1245
+ f"no DevCouncil check compiled for {ac_id} "
1246
+ f"(expected verification method: {method})"
1247
+ )
1248
+ gaps.append(Gap(
1249
+ id=self._next_gap_id(task.id, "AC"),
1250
+ severity="high" if blocks else "medium",
1251
+ gap_type="acceptance_criteria_unproven",
1252
+ requirement_id=self._requirement_id_for_ac(requirements, ac_id),
1253
+ task_id=task.id,
1254
+ description=(
1255
+ f"Acceptance criterion {ac_id} has no passing verification evidence "
1256
+ f"for task {task.id}.{suffix}"
1257
+ ),
1258
+ evidence=ac_evidence,
1259
+ recommended_fix=fix,
1260
+ blocking=blocks,
1261
+ acceptance_criterion_id=ac_id,
1262
+ expected_verification_method=method,
1263
+ suggested_command=suggested_cmd,
1264
+ ))
1265
+ elif task.requirement_ids:
1266
+ gaps.append(Gap(
1267
+ id=self._next_gap_id(task.id, "NOAC"),
1268
+ severity="high",
1269
+ gap_type="acceptance_criteria_unproven",
1270
+ requirement_id=task.requirement_ids[0],
1126
1271
  task_id=task.id,
1127
- tool=cov.tool,
1128
- measured=True,
1129
- changed_lines=cov.changed_executable_lines,
1130
- covered_lines=cov.covered_changed_lines,
1131
- coverage_ratio=cov.ratio,
1132
- uncovered_by_file=cov.uncovered_by_file,
1133
- absent_files=cov.absent_files,
1134
- summary=cov.summary(),
1272
+ description=f"Task {task.id} is linked to requirements but no acceptance criteria.",
1273
+ recommended_fix="Link the task to specific acceptance_criterion_ids before verification.",
1274
+ blocking=True,
1135
1275
  ))
1136
- failing = cov.covered_changed_lines == 0 if min_ratio <= 0 else cov.ratio < min_ratio
1137
- if failing:
1138
- first_file = next(iter(cov.uncovered_by_file), None)
1139
- first_lines = cov.uncovered_by_file.get(first_file or "", [])
1140
- target_cmds = self._coverage_target_commands(task)
1141
- gaps.append(Gap(
1142
- id=self._next_gap_id(task.id, "DIFFCOV"),
1143
- severity="high" if enforce_cov else "medium",
1144
- gap_type="diff_not_exercised",
1276
+
1277
+ # 5b. Diff↔coverage gate. A green suite is only acceptance evidence if it
1278
+ # exercised the lines the diff changed. This catches the failure the README
1279
+ # promises to stop: tests "pass" while the new logic is never run (unrelated
1280
+ # suite, code never imported, untouched branch). Measured only when the target
1281
+ # repo has coverage tooling and the diff has measurable Python changes; absent
1282
+ # that, it degrades silently rather than blocking correct work.
1283
+ measure_cov, enforce_cov, min_ratio = self._diff_coverage_settings()
1284
+ any_passing = bool(successful_commands) or any(compiled_pass.values())
1285
+ coverage_measured = False
1286
+ coverage_skipped_reason: Optional[str] = None
1287
+ if not measure_cov:
1288
+ coverage_skipped_reason = "diff coverage disabled in config"
1289
+ elif not diff_content:
1290
+ coverage_skipped_reason = "no diff to measure"
1291
+ elif not task.acceptance_criterion_ids:
1292
+ coverage_skipped_reason = "task has no acceptance criteria"
1293
+ elif not any_passing:
1294
+ coverage_skipped_reason = "no passing verification command to instrument"
1295
+ if measure_cov and diff_content and task.acceptance_criterion_ids and any_passing:
1296
+ cov = self.measure_diff_coverage(task, diff_content)
1297
+ if not cov.measured:
1298
+ coverage_skipped_reason = cov.reason or "diff coverage could not be measured"
1299
+ if cov.measured:
1300
+ coverage_measured = True
1301
+ coverage_skipped_reason = None
1302
+ evidence_to_save.append(DiffCoverageEvidence(
1145
1303
  task_id=task.id,
1146
- description=(
1147
- f"Verification commands passed but exercised "
1148
- f"{cov.covered_changed_lines}/{cov.changed_executable_lines} changed line(s): "
1149
- f"{cov.summary()}. The acceptance criteria are not proven because the new "
1150
- "logic was never executed by the tests."
1151
- ),
1152
- evidence=[cov.summary()] + [
1153
- f"{path}: lines {lines}" for path, lines in list(cov.uncovered_by_file.items())[:5]
1154
- ],
1155
- recommended_fix=(
1156
- "Add or extend a test that executes the changed lines, then re-verify. "
1157
- "A passing suite that does not run the new code is not acceptance evidence."
1158
- ),
1159
- # Off by default (signal first); teams opt into blocking via
1160
- # verification.diff_coverage.enforce.
1161
- blocking=enforce_cov,
1162
- file=first_file,
1163
- line=first_lines[0] if first_lines else None,
1164
- suggested_command=target_cmds[0] if target_cmds else None,
1304
+ tool=cov.tool,
1305
+ measured=True,
1306
+ changed_lines=cov.changed_executable_lines,
1307
+ covered_lines=cov.covered_changed_lines,
1308
+ coverage_ratio=cov.ratio,
1309
+ uncovered_by_file=cov.uncovered_by_file,
1310
+ absent_files=cov.absent_files,
1311
+ summary=cov.summary(),
1165
1312
  ))
1313
+ failing = cov.covered_changed_lines == 0 if min_ratio <= 0 else cov.ratio < min_ratio
1314
+ if failing:
1315
+ first_file = next(iter(cov.uncovered_by_file), None)
1316
+ first_lines = cov.uncovered_by_file.get(first_file or "", [])
1317
+ target_cmds = self._coverage_target_commands(task)
1318
+ gaps.append(Gap(
1319
+ id=self._next_gap_id(task.id, "DIFFCOV"),
1320
+ severity="high" if enforce_cov else "medium",
1321
+ gap_type="diff_not_exercised",
1322
+ task_id=task.id,
1323
+ description=(
1324
+ f"Verification commands passed but exercised "
1325
+ f"{cov.covered_changed_lines}/{cov.changed_executable_lines} changed line(s): "
1326
+ f"{cov.summary()}. The acceptance criteria are not proven because the new "
1327
+ "logic was never executed by the tests."
1328
+ ),
1329
+ evidence=[cov.summary()] + [
1330
+ f"{path}: lines {lines}" for path, lines in list(cov.uncovered_by_file.items())[:5]
1331
+ ],
1332
+ recommended_fix=(
1333
+ "Add or extend a test that executes the changed lines, then re-verify. "
1334
+ "A passing suite that does not run the new code is not acceptance evidence."
1335
+ ),
1336
+ # Off by default (signal first); teams opt into blocking via
1337
+ # verification.diff_coverage.enforce.
1338
+ blocking=enforce_cov,
1339
+ file=first_file,
1340
+ line=first_lines[0] if first_lines else None,
1341
+ suggested_command=target_cmds[0] if target_cmds else None,
1342
+ ))
1166
1343
 
1167
- # 6. Secret scan
1168
- if diff_content:
1169
- gaps.extend(self.secret_scanner.scan_diff(diff_content, task.id))
1170
-
1171
- # 7. LLM Implementation Review (ADVISORY ONLY).
1172
- # DevCouncil's authority is executable evidence, not model confidence — so
1173
- # an LLM reviewer must never block on its own say-so. Subjective reviewers
1174
- # over-flag correct code (false negatives that erode trust in "blocked"),
1175
- # so review findings are surfaced as non-blocking signals. A genuine
1176
- # requirement gap is caught by the acceptance-criteria evidence checks
1177
- # above; the review just adds human-facing context.
1178
- if self.reviewer and diff_content:
1179
- try:
1180
- review_result = await self.reviewer.review_changes(task, requirements, diff_content)
1181
- for finding in review_result.findings:
1182
- finding.id = self._next_gap_id(task.id, "REVIEW")
1183
- finding.blocking = False
1184
- gaps.append(finding)
1185
- except Exception as e:
1186
- logger.error("Implementation review failed: %s", e)
1187
-
1188
- # 8. Open live-review cards
1189
- for card in unresolved_blocking_cards(self.project_root, task_id=task.id):
1190
- gaps.append(Gap(
1191
- id=self._next_gap_id(task.id, "LIVE"),
1192
- severity="critical",
1193
- gap_type="architecture_drift",
1194
- task_id=task.id,
1195
- description=f"Open critical live-review card remains: {card.summary}",
1196
- evidence=[card.id, card.message_for_agent],
1197
- recommended_fix=(
1198
- f"Address the critique card, then run `dev watch resolve {card.id}` "
1199
- "or mark it ignored with justification outside the verification gate."
1200
- ),
1201
- blocking=True,
1202
- ))
1203
-
1204
- self.last_outcome = VerificationOutcome(
1205
- mode="compiled" if self.acceptance_compiler else "coarse",
1206
- compiler_active=compiler_active,
1207
- diff_empty=diff_empty,
1208
- coverage_measured=coverage_measured,
1209
- coverage_skipped_reason=coverage_skipped_reason,
1210
- )
1211
- return gaps, evidence_to_save
1344
+ # 6. Secret scan
1345
+ if diff_content:
1346
+ gaps.extend(self.secret_scanner.scan_diff(diff_content, task.id))
1347
+
1348
+ # 7. LLM Implementation Review (ADVISORY ONLY).
1349
+ # DevCouncil's authority is executable evidence, not model confidence — so
1350
+ # an LLM reviewer must never block on its own say-so. Subjective reviewers
1351
+ # over-flag correct code (false negatives that erode trust in "blocked"),
1352
+ # so review findings are surfaced as non-blocking signals. A genuine
1353
+ # requirement gap is caught by the acceptance-criteria evidence checks
1354
+ # above; the review just adds human-facing context.
1355
+ if review_future is not None:
1356
+ try:
1357
+ review_result = await review_future
1358
+ for finding in review_result.findings:
1359
+ finding.id = self._next_gap_id(task.id, "REVIEW")
1360
+ finding.blocking = False
1361
+ gaps.append(finding)
1362
+ except Exception as e:
1363
+ logger.error("Implementation review failed: %s", e)
1364
+
1365
+ # 8. Open live-review cards
1366
+ for card in unresolved_blocking_cards(self.project_root, task_id=task.id):
1367
+ gaps.append(Gap(
1368
+ id=self._next_gap_id(task.id, "LIVE"),
1369
+ severity="critical",
1370
+ gap_type="architecture_drift",
1371
+ task_id=task.id,
1372
+ description=f"Open critical live-review card remains: {card.summary}",
1373
+ evidence=[card.id, card.message_for_agent],
1374
+ recommended_fix=(
1375
+ f"Address the critique card, then run `dev watch resolve {card.id}` "
1376
+ "or mark it ignored with justification outside the verification gate."
1377
+ ),
1378
+ blocking=True,
1379
+ ))
1212
1380
 
1213
- def _check_semantic_diff(self, task: Task) -> List[Gap]:
1381
+ self.last_outcome = VerificationOutcome(
1382
+ mode="compiled" if self.acceptance_compiler else "coarse",
1383
+ compiler_active=compiler_active,
1384
+ diff_empty=diff_empty,
1385
+ coverage_measured=coverage_measured,
1386
+ coverage_skipped_reason=coverage_skipped_reason,
1387
+ )
1388
+ return gaps, evidence_to_save
1389
+ finally:
1390
+ # Always drain the two background LLM tasks (even if the body raised
1391
+ # before their await points) so neither is destroyed-while-pending nor
1392
+ # logs 'exception never retrieved', and clear the per-call memos so a
1393
+ # later non-verify_task call on this instance recomputes fresh.
1394
+ for _fut in (compile_future, review_future):
1395
+ if _fut is not None:
1396
+ if not _fut.done():
1397
+ _fut.cancel()
1398
+ try:
1399
+ await _fut
1400
+ except (asyncio.CancelledError, Exception):
1401
+ pass
1402
+ self._untracked_cache = None
1403
+ self._command_timeout_cache = None
1404
+ # Reload project dependencies next run: a reused Verifier may verify a later
1405
+ # task after pyproject/requirements changed on disk.
1406
+ self._project_deps_cache = None
1407
+
1408
+ def _task_intent_text(self, task: Task, requirements: Optional[List[Requirement]]) -> str:
1409
+ """Lowercased text describing what the task is meant to do — its title,
1410
+ description, and the descriptions of its acceptance criteria. Used to tell an
1411
+ INTENDED public-API change ("remove deprecated foo") from silent drift."""
1412
+ parts = [task.title or "", task.description or ""]
1413
+ if requirements:
1414
+ ac_ids = set(task.acceptance_criterion_ids)
1415
+ for req in requirements:
1416
+ for ac in req.acceptance_criteria:
1417
+ if ac.id in ac_ids:
1418
+ parts.append(ac.description or "")
1419
+ return " ".join(parts).lower()
1420
+
1421
+ def _check_semantic_diff(self, task: Task, requirements: Optional[List[Requirement]] = None) -> List[Gap]:
1214
1422
  gaps: List[Gap] = []
1215
1423
  semantic_path = self.project_root / ".devcouncil" / "semantic" / task.id
1216
1424
  after_path = semantic_path / "after.json"
@@ -1225,10 +1433,45 @@ class Verifier:
1225
1433
  return gaps
1226
1434
 
1227
1435
  planned_paths = {pf.path for pf in task.planned_files}
1228
- for item in result.get("classifications", []):
1436
+ classifications = result.get("classifications", [])
1437
+ # Drift signal inputs: a public symbol re-added elsewhere is a move/rename (a
1438
+ # legitimate refactor, not drift); and the task's own intent text lets a removal
1439
+ # the task actually asked for ("remove deprecated foo") pass without false-blocking.
1440
+ readded_public = {
1441
+ item.get("name") for item in classifications
1442
+ if item.get("type") == "exported_symbol_added" and item.get("name")
1443
+ }
1444
+ intent_text = self._task_intent_text(task, requirements)
1445
+ for item in classifications:
1229
1446
  change_type = item.get("type", "")
1230
1447
  path = item.get("path", "")
1231
- if change_type == "public_api_change" and path not in planned_paths:
1448
+ if change_type == "exported_symbol_removed":
1449
+ # An executor deleting/renaming an existing PUBLIC symbol — even inside a
1450
+ # file it is allowed to touch — is scope drift / a regression the focused
1451
+ # task rarely intends. Block it UNLESS the symbol was re-added elsewhere
1452
+ # (a move/rename) or the task text explicitly calls for the removal.
1453
+ name = item.get("name", "")
1454
+ moved = name in readded_public
1455
+ intended = bool(name) and name.lower() in intent_text
1456
+ gaps.append(Gap(
1457
+ id=self._next_gap_id(task.id, "DRIFT"),
1458
+ severity="high",
1459
+ gap_type="architecture_drift",
1460
+ task_id=task.id,
1461
+ description=(
1462
+ f"Public symbol '{name}' was removed from {path} — possible scope "
1463
+ "drift: the executor changed a public API the task did not call for."
1464
+ ),
1465
+ evidence=[f"{path}:{name}"],
1466
+ recommended_fix=(
1467
+ "Restore the removed public symbol. If its removal IS part of this "
1468
+ "task, state that in the task description / acceptance criteria so the "
1469
+ "change is an intended, reviewed decision rather than silent drift."
1470
+ ),
1471
+ blocking=(not moved and not intended),
1472
+ file=path,
1473
+ ))
1474
+ elif change_type == "public_api_change" and path not in planned_paths:
1232
1475
  gaps.append(Gap(
1233
1476
  id=self._next_gap_id(task.id, "SEM"),
1234
1477
  severity="high",
@@ -1239,17 +1482,66 @@ class Verifier:
1239
1482
  recommended_fix="Add file to planned_files and document acceptance criteria.",
1240
1483
  blocking=not bool(task.acceptance_criterion_ids),
1241
1484
  ))
1242
- elif change_type == "import_dependency_change" and path not in planned_paths:
1485
+ elif change_type == "public_api_change" and path in planned_paths:
1486
+ # The file is in scope, but the executor changed the SIGNATURE of an
1487
+ # existing public symbol. Tasks legitimately change signatures of files
1488
+ # they own, so this is ADVISORY only — surfaced so an audit/agent can see
1489
+ # the public contract moved, not silently drifted.
1243
1490
  gaps.append(Gap(
1244
- id=self._next_gap_id(task.id, "IMP"),
1491
+ id=self._next_gap_id(task.id, "SIGDRIFT"),
1245
1492
  severity="medium",
1246
- gap_type="dependency_risk",
1493
+ gap_type="architecture_drift",
1247
1494
  task_id=task.id,
1248
- description=f"Import dependency change in {path}.",
1249
- evidence=[path],
1250
- recommended_fix="Confirm dependency change is intentional.",
1495
+ description=(
1496
+ f"Public API signature change in planned file {path}"
1497
+ + (f" ({item.get('name')})" if item.get("name") else "")
1498
+ + ". Confirm callers are updated and the change is intended."
1499
+ ),
1500
+ evidence=[f"{path}:{item.get('name', '')}"],
1501
+ recommended_fix=(
1502
+ "If the signature change is part of this task, note it in the task "
1503
+ "description / acceptance criteria; otherwise revert it."
1504
+ ),
1251
1505
  blocking=False,
1252
1506
  ))
1507
+ elif change_type == "import_dependency_change":
1508
+ # A NEW third-party top-level package added to the diff is supply-chain
1509
+ # drift — block it. Everything else (stdlib, relative/local, or an
1510
+ # already-declared/available dependency) stays advisory, and only on an
1511
+ # unplanned file (an unplanned file is already orphan-blocked anyway).
1512
+ statement = item.get("statement", "")
1513
+ top = self._import_top_level(statement)
1514
+ new_third_party = self._is_new_third_party_import(top)
1515
+ if new_third_party:
1516
+ gaps.append(Gap(
1517
+ id=self._next_gap_id(task.id, "DEPADD"),
1518
+ severity="high",
1519
+ gap_type="dependency_risk",
1520
+ task_id=task.id,
1521
+ description=(
1522
+ f"New undeclared third-party dependency '{top}' imported in {path} "
1523
+ f"({statement.strip()}). Adding a dependency the task did not plan is "
1524
+ "supply-chain drift."
1525
+ ),
1526
+ evidence=[path, statement.strip()],
1527
+ recommended_fix=(
1528
+ f"Declare '{top}' in the project's dependencies and plan the change, "
1529
+ "or use an existing/standard-library alternative."
1530
+ ),
1531
+ blocking=True,
1532
+ file=path,
1533
+ ))
1534
+ elif path not in planned_paths:
1535
+ gaps.append(Gap(
1536
+ id=self._next_gap_id(task.id, "IMP"),
1537
+ severity="medium",
1538
+ gap_type="dependency_risk",
1539
+ task_id=task.id,
1540
+ description=f"Import dependency change in {path}.",
1541
+ evidence=[path],
1542
+ recommended_fix="Confirm dependency change is intentional.",
1543
+ blocking=False,
1544
+ ))
1253
1545
  elif change_type == "config_schema_dependency_change" and path not in planned_paths:
1254
1546
  gaps.append(Gap(
1255
1547
  id=self._next_gap_id(task.id, "CFG"),
@@ -1263,6 +1555,102 @@ class Verifier:
1263
1555
  ))
1264
1556
  return gaps
1265
1557
 
1558
+ @staticmethod
1559
+ def _import_top_level(statement: str) -> Optional[str]:
1560
+ """Top-level package of an import statement, or None for relative/local/unparseable.
1561
+
1562
+ ``import requests`` / ``import os.path`` -> the first dotted component; ``from x.y
1563
+ import z`` -> ``x``; ``from . import z`` / ``from .mod import z`` -> None (relative).
1564
+ """
1565
+ s = (statement or "").strip()
1566
+ if s.startswith("import "):
1567
+ first = s[len("import "):].split(",")[0].strip()
1568
+ top = first.split(" as ")[0].strip().split(".")[0].strip()
1569
+ return top or None
1570
+ if s.startswith("from "):
1571
+ rest = s[len("from "):].lstrip()
1572
+ if rest.startswith("."): # relative import -> local, never a new dependency
1573
+ return None
1574
+ mod = rest.split(" import ")[0].strip()
1575
+ return (mod.split(".")[0].strip() or None) if mod else None
1576
+ return None
1577
+
1578
+ def _is_new_third_party_import(self, top: Optional[str]) -> bool:
1579
+ """True only when ``top`` is a genuinely new, undeclared third-party package.
1580
+
1581
+ Conservative on purpose (this gates a BLOCK): a module is NOT flagged when it is
1582
+ the standard library, a declared project dependency, or already importable in the
1583
+ environment (so import-name vs distribution-name mismatches like ``yaml``/``pyyaml``
1584
+ never false-block). Only a package that is none of those — i.e. undeclared AND not
1585
+ present — counts as supply-chain drift."""
1586
+ if not top:
1587
+ return False
1588
+ if top in self._stdlib_modules():
1589
+ return False
1590
+ if top.lower() in self._project_dependencies():
1591
+ return False
1592
+ try:
1593
+ import importlib.util
1594
+ if importlib.util.find_spec(top) is not None:
1595
+ return False # already available in the environment; not a new dependency
1596
+ except Exception:
1597
+ # A find_spec error (e.g. a partially-installed parent) is ambiguous; do not
1598
+ # block on ambiguity.
1599
+ return False
1600
+ return True
1601
+
1602
+ @staticmethod
1603
+ def _stdlib_modules() -> frozenset:
1604
+ names = getattr(sys, "stdlib_module_names", None)
1605
+ return frozenset(names) if names else frozenset()
1606
+
1607
+ def _project_dependencies(self) -> set:
1608
+ """Lower-cased distribution names declared by the project (pyproject/requirements/
1609
+ package.json). Cached per Verifier instance; best-effort (parse errors are ignored)."""
1610
+ cached = getattr(self, "_project_deps_cache", None)
1611
+ if cached is not None:
1612
+ return cached
1613
+ deps: set = set()
1614
+ split_re = r"[><=!~;\[\] ]"
1615
+ pyproject = self.project_root / "pyproject.toml"
1616
+ if pyproject.exists():
1617
+ try:
1618
+ import tomllib
1619
+ data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
1620
+ project = data.get("project", {}) or {}
1621
+ for dep in project.get("dependencies", []) or []:
1622
+ pkg = re.split(split_re, dep.strip())[0].strip().lower()
1623
+ if pkg:
1624
+ deps.add(pkg)
1625
+ for group in (project.get("optional-dependencies", {}) or {}).values():
1626
+ for dep in group or []:
1627
+ pkg = re.split(split_re, dep.strip())[0].strip().lower()
1628
+ if pkg:
1629
+ deps.add(pkg)
1630
+ except Exception:
1631
+ pass
1632
+ requirements = self.project_root / "requirements.txt"
1633
+ if requirements.exists():
1634
+ try:
1635
+ for line in requirements.read_text(encoding="utf-8").splitlines():
1636
+ line = line.strip()
1637
+ if line and not line.startswith("#"):
1638
+ pkg = re.split(split_re, line)[0].strip().lower()
1639
+ if pkg:
1640
+ deps.add(pkg)
1641
+ except Exception:
1642
+ pass
1643
+ package_json = self.project_root / "package.json"
1644
+ if package_json.exists():
1645
+ try:
1646
+ data = json.loads(package_json.read_text(encoding="utf-8"))
1647
+ for key in ("dependencies", "devDependencies", "optionalDependencies"):
1648
+ deps.update(k.lower() for k in (data.get(key) or {}).keys())
1649
+ except Exception:
1650
+ pass
1651
+ self._project_deps_cache = deps
1652
+ return deps
1653
+
1266
1654
  # Signatures that mean the verification command itself could not run (or had
1267
1655
  # nothing to run), so its non-zero exit says nothing about whether the
1268
1656
  # implementation is correct — a tooling/plan defect, not a code defect.