devcouncil 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/devcouncil/app/config.py +181 -7
- package/src/devcouncil/app/orchestrator.py +10 -6
- package/src/devcouncil/app/state_machine.py +4 -0
- package/src/devcouncil/artifacts/graph.py +9 -2
- package/src/devcouncil/cli/commands/check.py +12 -1
- package/src/devcouncil/cli/commands/design.py +186 -0
- package/src/devcouncil/cli/commands/doctor.py +160 -3
- package/src/devcouncil/cli/commands/go.py +96 -16
- package/src/devcouncil/cli/commands/hook.py +172 -0
- package/src/devcouncil/cli/commands/init.py +7 -2
- package/src/devcouncil/cli/commands/integrate.py +492 -34
- package/src/devcouncil/cli/commands/logs.py +106 -0
- package/src/devcouncil/cli/commands/okf.py +245 -0
- package/src/devcouncil/cli/commands/plan.py +54 -14
- package/src/devcouncil/cli/commands/repair.py +12 -3
- package/src/devcouncil/cli/commands/run.py +128 -7
- package/src/devcouncil/cli/commands/skills.py +180 -1
- package/src/devcouncil/cli/commands/status.py +7 -16
- package/src/devcouncil/cli/commands/verify.py +16 -10
- package/src/devcouncil/cli/commands/watch.py +24 -4
- package/src/devcouncil/cli/main.py +36 -1
- package/src/devcouncil/domain/evidence.py +7 -0
- package/src/devcouncil/execution/checkpoints.py +12 -2
- package/src/devcouncil/execution/fs_watcher.py +27 -2
- package/src/devcouncil/execution/handoff.py +1 -1
- package/src/devcouncil/execution/patch.py +6 -0
- package/src/devcouncil/execution/permissions.py +7 -0
- package/src/devcouncil/execution/policy_engine.py +12 -5
- package/src/devcouncil/execution/prompt_builder.py +126 -10
- package/src/devcouncil/execution/shell_session.py +6 -0
- package/src/devcouncil/execution/task_runner.py +18 -7
- package/src/devcouncil/executors/agent_registry.py +22 -1
- package/src/devcouncil/executors/coding_cli.py +133 -5
- package/src/devcouncil/executors/mini_swe.py +6 -0
- package/src/devcouncil/executors/native/agent.py +15 -0
- package/src/devcouncil/executors/openhands.py +6 -0
- package/src/devcouncil/gating/checks/secret_scan_check.py +7 -0
- package/src/devcouncil/gating/policy.py +38 -7
- package/src/devcouncil/indexing/ast_matcher.py +16 -6
- package/src/devcouncil/indexing/repo_mapper.py +30 -8
- package/src/devcouncil/indexing/semantic_index.py +42 -26
- package/src/devcouncil/integrations/actions.py +24 -4
- package/src/devcouncil/integrations/check.py +7 -4
- package/src/devcouncil/integrations/claude_assets.py +444 -0
- package/src/devcouncil/integrations/code_review_graph.py +13 -2
- package/src/devcouncil/integrations/github_intent.py +8 -1
- package/src/devcouncil/integrations/gitnexus.py +10 -2
- package/src/devcouncil/integrations/mcp/server.py +404 -15
- package/src/devcouncil/integrations/pr_comments.py +9 -0
- package/src/devcouncil/knowledge/__init__.py +23 -0
- package/src/devcouncil/knowledge/design.py +374 -0
- package/src/devcouncil/knowledge/design_conformance.py +317 -0
- package/src/devcouncil/knowledge/fetch.py +223 -0
- package/src/devcouncil/knowledge/frontmatter.py +51 -0
- package/src/devcouncil/knowledge/okf.py +202 -0
- package/src/devcouncil/knowledge/skill_bridge.py +96 -0
- package/src/devcouncil/knowledge/sources.py +239 -0
- package/src/devcouncil/live/cards.py +20 -6
- package/src/devcouncil/live/repair_prompt.py +29 -6
- package/src/devcouncil/live/reviewer.py +72 -13
- package/src/devcouncil/live/summary.py +18 -8
- package/src/devcouncil/live/transcripts.py +38 -5
- package/src/devcouncil/llm/cache.py +14 -6
- package/src/devcouncil/llm/provider.py +179 -92
- package/src/devcouncil/llm/router.py +122 -23
- package/src/devcouncil/optimization/skillopt.py +673 -0
- package/src/devcouncil/planning/arbiter_service.py +10 -2
- package/src/devcouncil/planning/correction_manifest.py +47 -4
- package/src/devcouncil/planning/critique_service.py +9 -2
- package/src/devcouncil/planning/plan_service.py +69 -3
- package/src/devcouncil/planning/prompt_enhancer_service.py +124 -0
- package/src/devcouncil/planning/repair_service.py +8 -2
- package/src/devcouncil/planning/spec_service.py +10 -2
- package/src/devcouncil/repo/ci_scaffold.py +13 -5
- package/src/devcouncil/repo/sca.py +11 -1
- package/src/devcouncil/reporting/json_report.py +11 -0
- package/src/devcouncil/reporting/markdown_report.py +14 -1
- package/src/devcouncil/reporting/okf_bundle_writer.py +364 -0
- package/src/devcouncil/reporting/okf_html.py +323 -0
- package/src/devcouncil/reporting/report_builder.py +18 -1
- package/src/devcouncil/skills/registry.py +111 -33
- package/src/devcouncil/storage/db.py +58 -2
- package/src/devcouncil/storage/models.py +4 -0
- package/src/devcouncil/storage/native.py +20 -18
- package/src/devcouncil/storage/repositories.py +35 -18
- package/src/devcouncil/telemetry/logging_setup.py +244 -0
- package/src/devcouncil/telemetry/stages.py +141 -0
- package/src/devcouncil/telemetry/tracker.py +12 -1
- package/src/devcouncil/ui/dashboard.py +69 -5
- package/src/devcouncil/verification/acceptance_compiler.py +147 -19
- package/src/devcouncil/verification/ad_hoc_check.py +6 -0
- package/src/devcouncil/verification/implementation_reviewer.py +11 -2
- package/src/devcouncil/verification/sandbox.py +7 -4
- package/src/devcouncil/verification/verifier.py +905 -517
- package/uv.lock +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
import hashlib
|
|
2
3
|
import os
|
|
3
4
|
import shutil
|
|
@@ -11,7 +12,7 @@ import re
|
|
|
11
12
|
import shlex
|
|
12
13
|
from dataclasses import dataclass, asdict
|
|
13
14
|
from pathlib import Path
|
|
14
|
-
from typing import List, Dict, Any, Optional, Tuple
|
|
15
|
+
from typing import List, Dict, Any, Literal, Optional, Tuple
|
|
15
16
|
|
|
16
17
|
from devcouncil.app.config import load_config
|
|
17
18
|
|
|
@@ -86,6 +87,14 @@ class Verifier:
|
|
|
86
87
|
# When set, overrides the (measure, enforce, min_ratio) diff-coverage settings
|
|
87
88
|
# that would otherwise come from config. Used by ad-hoc checks and tests.
|
|
88
89
|
self._diff_coverage_override: Optional[Tuple[bool, bool, float]] = None
|
|
90
|
+
# Per-verify_task memos (primed at verify_task entry, cleared before it returns)
|
|
91
|
+
# so the hot path does not re-run `git ls-files` or re-load config repeatedly.
|
|
92
|
+
# None outside a verify_task call, so all other callers behave exactly as before.
|
|
93
|
+
self._untracked_cache: Optional[List[str]] = None
|
|
94
|
+
self._command_timeout_cache: Optional[int] = None
|
|
95
|
+
# Project dependency names (lower-cased), loaded once per verify_task and cleared
|
|
96
|
+
# in its finally so a reused Verifier re-reads them for a later task.
|
|
97
|
+
self._project_deps_cache: Optional[set] = None
|
|
89
98
|
|
|
90
99
|
def _next_gap_id(self, task_id: str, suffix: str) -> str:
|
|
91
100
|
"""Generate unique gap IDs to prevent SQLite overwrites."""
|
|
@@ -125,15 +134,15 @@ class Verifier:
|
|
|
125
134
|
changed.difference_update(self._load_task_snapshot_files(task_id))
|
|
126
135
|
return sorted(changed)
|
|
127
136
|
|
|
128
|
-
def
|
|
129
|
-
"""
|
|
137
|
+
def _committed_task_diff(self, task_id: str) -> str:
|
|
138
|
+
"""Diff of work committed since the task's ``before`` checkpoint, or "".
|
|
130
139
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
140
|
+
When ``dev go`` commits a task's work (e.g. between self-repair attempts, or
|
|
141
|
+
before the reconciliation pass), the working-tree diff (``git diff HEAD``) is
|
|
142
|
+
empty even though the task is fully implemented. This recovers that committed
|
|
143
|
+
change so acceptance compilation/review still have something to reason about
|
|
144
|
+
instead of seeing an empty diff and skipping — which would mark every criterion
|
|
145
|
+
unproven and wrongly block correct, committed code.
|
|
137
146
|
"""
|
|
138
147
|
# Literal of CheckpointService.REF_BEFORE (kept inline to avoid a circular
|
|
139
148
|
# import: checkpoints.py imports Verifier).
|
|
@@ -146,15 +155,27 @@ class Verifier:
|
|
|
146
155
|
stderr=subprocess.DEVNULL,
|
|
147
156
|
).returncode == 0
|
|
148
157
|
if has_ref:
|
|
149
|
-
|
|
158
|
+
return subprocess.check_output(
|
|
150
159
|
["git", "diff", before_ref],
|
|
151
160
|
cwd=self.project_root,
|
|
152
161
|
stderr=subprocess.DEVNULL,
|
|
153
162
|
).decode("utf-8", errors="replace")
|
|
154
|
-
if diff.strip():
|
|
155
|
-
return True
|
|
156
163
|
except Exception:
|
|
157
164
|
pass
|
|
165
|
+
return ""
|
|
166
|
+
|
|
167
|
+
def _task_produced_changes(self, task_id: str) -> bool:
|
|
168
|
+
"""True when the task has a footprint beyond the current working-tree diff.
|
|
169
|
+
|
|
170
|
+
Used so the empty-diff guard does not misfire on already-committed work: in
|
|
171
|
+
``dev go`` each task is committed and then re-verified by the reconciliation
|
|
172
|
+
pass, at which point ``git diff HEAD`` is empty even though the task was fully
|
|
173
|
+
implemented. We detect that via the task's ``before`` checkpoint ref (work
|
|
174
|
+
committed since the task started) and a non-empty ``after`` patch. A genuine
|
|
175
|
+
no-op run has neither, so it is still correctly flagged as empty.
|
|
176
|
+
"""
|
|
177
|
+
if self._committed_task_diff(task_id).strip():
|
|
178
|
+
return True
|
|
158
179
|
after_patch = self.project_root / ".devcouncil" / "checkpoints" / f"{task_id}-after.patch"
|
|
159
180
|
try:
|
|
160
181
|
return after_patch.exists() and bool(after_patch.read_text(encoding="utf-8", errors="replace").strip())
|
|
@@ -209,6 +230,11 @@ class Verifier:
|
|
|
209
230
|
return self._filter_change_paths(sorted(files))
|
|
210
231
|
|
|
211
232
|
def _get_untracked_files(self) -> List[str]:
|
|
233
|
+
# Per-verify_task memo: git ls-files is otherwise re-run via get_changed_files,
|
|
234
|
+
# _get_untracked_files_diff, and _classify_change_paths. verify_task primes this
|
|
235
|
+
# once; it is None for every other caller, so they recompute fresh as before.
|
|
236
|
+
if self._untracked_cache is not None:
|
|
237
|
+
return self._untracked_cache
|
|
212
238
|
try:
|
|
213
239
|
output = subprocess.check_output(
|
|
214
240
|
["git", "ls-files", "--others", "--exclude-standard"],
|
|
@@ -415,11 +441,17 @@ class Verifier:
|
|
|
415
441
|
return summary[: budget + len(salient) + 8]
|
|
416
442
|
|
|
417
443
|
def _run_command(self, command: str, task_id: str = "verify") -> CommandResult:
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
timeout =
|
|
444
|
+
# Per-verify_task memo: avoid re-loading config for the timeout on every command
|
|
445
|
+
# in the expected_tests / allowed_commands / compiled-check loops. Falls back to
|
|
446
|
+
# loading config when called outside verify_task (cache is None).
|
|
447
|
+
if self._command_timeout_cache is not None:
|
|
448
|
+
timeout = self._command_timeout_cache
|
|
449
|
+
else:
|
|
450
|
+
try:
|
|
451
|
+
config = load_config(self.project_root)
|
|
452
|
+
timeout = config.execution.command_timeout
|
|
453
|
+
except Exception:
|
|
454
|
+
timeout = 300
|
|
423
455
|
|
|
424
456
|
env = self._verification_env()
|
|
425
457
|
argv = self._split_command(command)
|
|
@@ -664,553 +696,729 @@ class Verifier:
|
|
|
664
696
|
pass
|
|
665
697
|
|
|
666
698
|
async def verify_task(self, task: Task, requirements: List[Requirement]) -> Tuple[List[Gap], List[Any]]:
|
|
699
|
+
logger.info("verify_task: task=%s requirements=%d", task.id, len(requirements))
|
|
667
700
|
self._gap_counter = 0
|
|
668
701
|
gaps: List[Gap] = []
|
|
669
702
|
evidence_to_save: List[Any] = []
|
|
703
|
+
# Prime the per-call memos: compute the untracked-file list once (otherwise
|
|
704
|
+
# re-run by get_changed_files, get_diff, and _classify_change_paths) and load the
|
|
705
|
+
# command timeout once (otherwise re-loaded by _run_command on every command).
|
|
706
|
+
# Both are cleared before this method returns.
|
|
707
|
+
self._untracked_cache = self._get_untracked_files()
|
|
708
|
+
ac_samples, ac_repair_attempts = 1, 1
|
|
709
|
+
try:
|
|
710
|
+
_cfg = load_config(self.project_root)
|
|
711
|
+
self._command_timeout_cache = _cfg.execution.command_timeout
|
|
712
|
+
ac_samples = max(1, _cfg.verification.acceptance_checks.samples)
|
|
713
|
+
ac_repair_attempts = max(0, _cfg.verification.acceptance_checks.repair_attempts)
|
|
714
|
+
ac_per_criterion = bool(_cfg.verification.acceptance_checks.per_criterion)
|
|
715
|
+
except Exception:
|
|
716
|
+
self._command_timeout_cache = 300
|
|
717
|
+
ac_per_criterion = False
|
|
670
718
|
changed_files = self.get_task_changed_files(task.id)
|
|
671
719
|
diff_content = self.get_diff()
|
|
720
|
+
# When the working tree is clean but the task's work was committed (dev go commits
|
|
721
|
+
# between repair attempts and before reconciliation), fall back to the committed
|
|
722
|
+
# checkpoint diff. Otherwise acceptance compilation/review below — gated on a
|
|
723
|
+
# non-empty diff_content — would be skipped, leaving every criterion unproven and
|
|
724
|
+
# wrongly blocking correct, already-committed code.
|
|
725
|
+
if not diff_content.strip():
|
|
726
|
+
committed_diff = self._committed_task_diff(task.id)
|
|
727
|
+
if committed_diff.strip():
|
|
728
|
+
diff_content = committed_diff
|
|
672
729
|
diff_empty = not bool(diff_content.strip())
|
|
673
|
-
#
|
|
674
|
-
#
|
|
675
|
-
#
|
|
676
|
-
#
|
|
677
|
-
#
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
),
|
|
696
|
-
evidence=[f"planned files expecting change: {sorted(p.path for p in task.planned_files if p.allowed_change != 'read_only')}"],
|
|
697
|
-
recommended_fix=(
|
|
698
|
-
"Implement the planned changes so the diff is non-empty, then re-verify. "
|
|
699
|
-
"If you did make changes, ensure they are saved and visible to git "
|
|
700
|
-
"(not reverted, stashed, or written outside the project root)."
|
|
701
|
-
),
|
|
702
|
-
blocking=True,
|
|
703
|
-
))
|
|
704
|
-
|
|
705
|
-
if diff_content:
|
|
706
|
-
added_files, deleted_files = self._classify_change_paths(changed_files)
|
|
707
|
-
diff_ev = DiffEvidence(
|
|
708
|
-
task_id=task.id,
|
|
709
|
-
changed_files=changed_files,
|
|
710
|
-
added_files=added_files,
|
|
711
|
-
deleted_files=deleted_files,
|
|
712
|
-
diff_summary=f"Diff captured for {len(changed_files)} files."
|
|
730
|
+
# Launch the two independent LLM passes — acceptance compilation and the advisory
|
|
731
|
+
# implementation review — concurrently as soon as the diff is available, instead
|
|
732
|
+
# of awaiting them sequentially later. Each depends only on (task, requirements,
|
|
733
|
+
# diff_content), so there is no data hazard; each result is awaited (with its
|
|
734
|
+
# existing try/except) at the point it is consumed below. The create-time guards
|
|
735
|
+
# match the consume-time guards exactly, so every task created is always awaited.
|
|
736
|
+
compile_future: Optional["asyncio.Task[Dict[str, List[str]]]"] = None
|
|
737
|
+
if self.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
|
|
738
|
+
# Prefer the self-consistency interface; fall back to single-shot ``compile`` so
|
|
739
|
+
# older compiler doubles/implementations keep working.
|
|
740
|
+
if hasattr(self.acceptance_compiler, "compile_candidates"):
|
|
741
|
+
_compile_coro = self.acceptance_compiler.compile_candidates(
|
|
742
|
+
task, requirements, diff_content, samples=ac_samples,
|
|
743
|
+
per_criterion=ac_per_criterion,
|
|
744
|
+
)
|
|
745
|
+
else:
|
|
746
|
+
_compile_coro = self.acceptance_compiler.compile(task, requirements, diff_content)
|
|
747
|
+
compile_future = asyncio.create_task(_compile_coro)
|
|
748
|
+
review_future: Optional["asyncio.Task[Any]"] = None
|
|
749
|
+
if self.reviewer and diff_content:
|
|
750
|
+
review_future = asyncio.create_task(
|
|
751
|
+
self.reviewer.review_changes(task, requirements, diff_content)
|
|
713
752
|
)
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
file=pf.path,
|
|
730
|
-
))
|
|
731
|
-
|
|
732
|
-
# 2. Orphan-diff detection
|
|
733
|
-
for cf in changed_files:
|
|
734
|
-
if cf not in planned_paths:
|
|
753
|
+
try:
|
|
754
|
+
# "Work present" is broader than the current working-tree diff: a task whose
|
|
755
|
+
# changes were already committed (e.g. `dev go`'s per-task commit, then the
|
|
756
|
+
# final reconciliation pass where `git diff HEAD` is empty) still counts as
|
|
757
|
+
# implemented. A genuine no-op run has neither a working diff nor committed
|
|
758
|
+
# changes since the task's checkpoint.
|
|
759
|
+
work_present = (not diff_empty) or self._task_produced_changes(task.id)
|
|
760
|
+
|
|
761
|
+
# Empty-diff guard. If the task declares files to create or modify but produced
|
|
762
|
+
# NO work at all, there is nothing to prove — an agent must not be able to
|
|
763
|
+
# declare victory having written nothing (or after a transient git error that
|
|
764
|
+
# degraded the diff to ""). This is the single most dangerous false-pass for
|
|
765
|
+
# autonomy, so it blocks regardless of which commands ran.
|
|
766
|
+
expects_change = any(pf.allowed_change != "read_only" for pf in task.planned_files)
|
|
767
|
+
if not work_present and expects_change:
|
|
735
768
|
gaps.append(Gap(
|
|
736
|
-
id=self._next_gap_id(task.id, "
|
|
769
|
+
id=self._next_gap_id(task.id, "NODIFF"),
|
|
737
770
|
severity="high",
|
|
738
|
-
gap_type="
|
|
771
|
+
gap_type="task_not_implemented",
|
|
739
772
|
task_id=task.id,
|
|
740
|
-
description=
|
|
741
|
-
|
|
742
|
-
|
|
773
|
+
description=(
|
|
774
|
+
f"Task {task.id} declares files to create or modify, but produced no "
|
|
775
|
+
"changes. Verification cannot prove work that does not exist."
|
|
776
|
+
),
|
|
777
|
+
evidence=[f"planned files expecting change: {sorted(p.path for p in task.planned_files if p.allowed_change != 'read_only')}"],
|
|
778
|
+
recommended_fix=(
|
|
779
|
+
"Implement the planned changes so the diff is non-empty, then re-verify. "
|
|
780
|
+
"If you did make changes, ensure they are saved and visible to git "
|
|
781
|
+
"(not reverted, stashed, or written outside the project root)."
|
|
782
|
+
),
|
|
743
783
|
blocking=True,
|
|
744
|
-
file=cf,
|
|
745
784
|
))
|
|
746
785
|
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
dep_changes = self._check_dependency_changes(changed_files)
|
|
751
|
-
for dep_file in dep_changes:
|
|
752
|
-
if dep_file not in planned_paths:
|
|
753
|
-
gaps.append(Gap(
|
|
754
|
-
id=self._next_gap_id(task.id, "DEP"),
|
|
755
|
-
severity="high",
|
|
756
|
-
gap_type="dependency_risk",
|
|
786
|
+
if diff_content:
|
|
787
|
+
added_files, deleted_files = self._classify_change_paths(changed_files)
|
|
788
|
+
diff_ev = DiffEvidence(
|
|
757
789
|
task_id=task.id,
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
)
|
|
790
|
+
changed_files=changed_files,
|
|
791
|
+
added_files=added_files,
|
|
792
|
+
deleted_files=deleted_files,
|
|
793
|
+
diff_summary=f"Diff captured for {len(changed_files)} files."
|
|
794
|
+
)
|
|
795
|
+
evidence_to_save.append(diff_ev)
|
|
764
796
|
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
# 4. Run verification commands
|
|
772
|
-
command_results: List[CommandResult] = []
|
|
773
|
-
evidence_results: List[CommandResult] = []
|
|
774
|
-
genuine_failure = False # a command that actually ran and failed (real defect signal)
|
|
775
|
-
had_unrunnable = False # a command that could not run (missing tool / missing tests)
|
|
776
|
-
# Genuine test failures demoted to non-blocking only because a compiler is active.
|
|
777
|
-
# That demotion is legitimate ONLY if the compiler actually produces per-criterion
|
|
778
|
-
# checks to take authority; re-promoted below if it produces none.
|
|
779
|
-
demoted_failures: List[Gap] = []
|
|
780
|
-
for cmd_type, cmds in self._commands_for_task(task).items():
|
|
781
|
-
for cmd in cmds:
|
|
782
|
-
applicable, skip_reason = self._command_applicable(cmd)
|
|
783
|
-
if not applicable:
|
|
784
|
-
# Wrong-stack command (e.g. `npm test` on a Python repo): skip it
|
|
785
|
-
# entirely rather than running and failing for a stack reason — an
|
|
786
|
-
# advisory note so the skip is visible (no silent drop).
|
|
797
|
+
# 1. Planned-file coverage check
|
|
798
|
+
planned_paths = {pf.path for pf in task.planned_files}
|
|
799
|
+
changed_set = set(changed_files)
|
|
800
|
+
for pf in task.planned_files:
|
|
801
|
+
if pf.path not in changed_set and pf.allowed_change != "read_only":
|
|
787
802
|
gaps.append(Gap(
|
|
788
|
-
id=self._next_gap_id(task.id, "
|
|
789
|
-
severity="
|
|
790
|
-
gap_type="
|
|
803
|
+
id=self._next_gap_id(task.id, "FILE"),
|
|
804
|
+
severity="medium",
|
|
805
|
+
gap_type="planned_file_not_changed",
|
|
791
806
|
task_id=task.id,
|
|
792
|
-
description=f"
|
|
793
|
-
|
|
794
|
-
recommended_fix=(
|
|
795
|
-
"Replace it with a command for this repo's stack, or remove it "
|
|
796
|
-
"from .devcouncil/config.yaml / the task's expected_tests."
|
|
797
|
-
),
|
|
807
|
+
description=f"Planned file {pf.path} was not modified.",
|
|
808
|
+
recommended_fix=f"Modify {pf.path} as planned or update the task.",
|
|
798
809
|
blocking=False,
|
|
799
|
-
|
|
810
|
+
file=pf.path,
|
|
800
811
|
))
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
812
|
+
|
|
813
|
+
# 2. Orphan-diff detection
|
|
814
|
+
for cf in changed_files:
|
|
815
|
+
if cf not in planned_paths:
|
|
816
|
+
gaps.append(Gap(
|
|
817
|
+
id=self._next_gap_id(task.id, "ORPHAN"),
|
|
818
|
+
severity="high",
|
|
819
|
+
gap_type="orphan_diff",
|
|
820
|
+
task_id=task.id,
|
|
821
|
+
description=f"File {cf} was modified but not planned for this task.",
|
|
822
|
+
evidence=[cf],
|
|
823
|
+
recommended_fix=f"Revert changes to {cf} or add it to the task's planned files.",
|
|
824
|
+
blocking=True,
|
|
825
|
+
file=cf,
|
|
826
|
+
))
|
|
827
|
+
|
|
828
|
+
gaps.extend(self._check_semantic_diff(task, requirements))
|
|
829
|
+
|
|
830
|
+
# 3. Dependency change detection
|
|
831
|
+
dep_changes = self._check_dependency_changes(changed_files)
|
|
832
|
+
for dep_file in dep_changes:
|
|
833
|
+
if dep_file not in planned_paths:
|
|
834
|
+
gaps.append(Gap(
|
|
835
|
+
id=self._next_gap_id(task.id, "DEP"),
|
|
836
|
+
severity="high",
|
|
837
|
+
gap_type="dependency_risk",
|
|
838
|
+
task_id=task.id,
|
|
839
|
+
description=f"Dependency file {dep_file} was modified without being in planned files.",
|
|
840
|
+
evidence=[dep_file],
|
|
841
|
+
recommended_fix=f"Justify the dependency change or revert {dep_file}.",
|
|
842
|
+
blocking=True,
|
|
843
|
+
file=dep_file,
|
|
844
|
+
))
|
|
845
|
+
|
|
846
|
+
# When DevCouncil can compile its own per-criterion checks, THOSE are the
|
|
847
|
+
# authority and the planner's expected_tests are demoted to advisory — so a
|
|
848
|
+
# bogus planner command (irrelevant linters, npm on a Python project, tests
|
|
849
|
+
# that reference missing files) can no longer block correct work.
|
|
850
|
+
compiler_active = bool(self.acceptance_compiler and diff_content and task.acceptance_criterion_ids)
|
|
851
|
+
|
|
852
|
+
# 4. Run verification commands
|
|
853
|
+
command_results: List[CommandResult] = []
|
|
854
|
+
evidence_results: List[CommandResult] = []
|
|
855
|
+
genuine_failure = False # a command that actually ran and failed (real defect signal)
|
|
856
|
+
had_unrunnable = False # a command that could not run (missing tool / missing tests)
|
|
857
|
+
# Genuine test failures demoted to non-blocking only because a compiler is active.
|
|
858
|
+
# That demotion is legitimate ONLY if the compiler actually produces per-criterion
|
|
859
|
+
# checks to take authority; re-promoted below if it produces none.
|
|
860
|
+
demoted_failures: List[Gap] = []
|
|
861
|
+
for cmd_type, cmds in self._commands_for_task(task).items():
|
|
862
|
+
for cmd in cmds:
|
|
863
|
+
applicable, skip_reason = self._command_applicable(cmd)
|
|
864
|
+
if not applicable:
|
|
865
|
+
# Wrong-stack command (e.g. `npm test` on a Python repo): skip it
|
|
866
|
+
# entirely rather than running and failing for a stack reason — an
|
|
867
|
+
# advisory note so the skip is visible (no silent drop).
|
|
815
868
|
gaps.append(Gap(
|
|
816
|
-
id=self._next_gap_id(task.id, "
|
|
817
|
-
severity="
|
|
818
|
-
gap_type="
|
|
869
|
+
id=self._next_gap_id(task.id, "SKIP"),
|
|
870
|
+
severity="low",
|
|
871
|
+
gap_type="skipped_verification_command",
|
|
819
872
|
task_id=task.id,
|
|
820
|
-
description=
|
|
821
|
-
|
|
822
|
-
"It appears malformed or its tooling is unavailable, so this command "
|
|
823
|
-
"proves nothing either way."
|
|
824
|
-
),
|
|
825
|
-
evidence=[result.summary[:500]],
|
|
873
|
+
description=f"Skipped verification command '{cmd}': {skip_reason}.",
|
|
874
|
+
evidence=[skip_reason],
|
|
826
875
|
recommended_fix=(
|
|
827
|
-
"
|
|
828
|
-
"
|
|
876
|
+
"Replace it with a command for this repo's stack, or remove it "
|
|
877
|
+
"from .devcouncil/config.yaml / the task's expected_tests."
|
|
829
878
|
),
|
|
830
|
-
# Non-blocking: a command that cannot run is not evidence of a
|
|
831
|
-
# defect. If it was the *only* check for an acceptance criterion,
|
|
832
|
-
# that criterion is independently caught as unproven (blocking).
|
|
833
879
|
blocking=False,
|
|
834
880
|
suggested_command=cmd,
|
|
835
|
-
stdout_path=result.stdout_path or None,
|
|
836
|
-
stderr_path=result.stderr_path or None,
|
|
837
881
|
))
|
|
838
|
-
|
|
839
|
-
# A verification command that genuinely failed. Lint/typecheck
|
|
840
|
-
# commands (from the config fallback) report style/type opinion,
|
|
841
|
-
# not a correctness defect, so they are ADVISORY — blocking a
|
|
842
|
-
# behaviorally-correct task on `flake8`/`mypy`/`ruff` is the
|
|
843
|
-
# false-block the benchmark surfaced. A real test failure still
|
|
844
|
-
# gates (unless compiled checks supersede it).
|
|
845
|
-
is_quality_gate = cmd_type in {"lint", "typecheck"} or self._is_quality_only_command(cmd)
|
|
846
|
-
blocking = (not compiler_active) and not is_quality_gate
|
|
847
|
-
if blocking:
|
|
848
|
-
genuine_failure = True
|
|
849
|
-
fail_file, fail_line = self._failure_location(result)
|
|
850
|
-
gap = Gap(
|
|
851
|
-
id=self._next_gap_id(task.id, cmd_type.upper()),
|
|
852
|
-
severity="high" if blocking else "medium",
|
|
853
|
-
gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
|
|
854
|
-
task_id=task.id,
|
|
855
|
-
description=(
|
|
856
|
-
f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
|
|
857
|
-
f"failed with exit code {result.exit_code}"
|
|
858
|
-
+ (" (advisory: style/type, not a correctness gate)." if is_quality_gate else ".")
|
|
859
|
-
),
|
|
860
|
-
evidence=[result.summary[:500]],
|
|
861
|
-
recommended_fix=f"Fix the issues reported by '{cmd}'.",
|
|
862
|
-
blocking=blocking,
|
|
863
|
-
suggested_command=cmd,
|
|
864
|
-
file=fail_file,
|
|
865
|
-
line=fail_line,
|
|
866
|
-
stdout_path=result.stdout_path or None,
|
|
867
|
-
stderr_path=result.stderr_path or None,
|
|
868
|
-
)
|
|
869
|
-
gaps.append(gap)
|
|
870
|
-
# A real test failure demoted only because the compiler is active:
|
|
871
|
-
# remember it so we can re-promote if the compiler yields no checks.
|
|
872
|
-
if compiler_active and not is_quality_gate and not blocking:
|
|
873
|
-
demoted_failures.append(gap)
|
|
874
|
-
|
|
875
|
-
# 4b. Compiled acceptance checks — precise, DevCouncil-owned per-criterion
|
|
876
|
-
# evidence. Derive one runnable check per acceptance criterion from the
|
|
877
|
-
# criterion text + the diff, instead of trusting planner-authored
|
|
878
|
-
# expected_tests (which the benchmark showed often reference absent tools or
|
|
879
|
-
# test files). Each check maps 1:1 to its criterion, replacing the coarse
|
|
880
|
-
# "any command passed -> every criterion proven" mapping.
|
|
881
|
-
compiled_pass: Dict[str, bool] = {}
|
|
882
|
-
# Per-AC bookkeeping so the unproven-AC gap can attach ONLY the check(s) that
|
|
883
|
-
# targeted that criterion (and the specific failing result), instead of dumping
|
|
884
|
-
# every command summary. Keys are AC ids; values track the compiled command(s)
|
|
885
|
-
# and any failing CommandResults for that AC.
|
|
886
|
-
compiled_cmds_by_ac: Dict[str, List[str]] = {}
|
|
887
|
-
failing_results_by_ac: Dict[str, List[CommandResult]] = {}
|
|
888
|
-
if self.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
|
|
889
|
-
try:
|
|
890
|
-
compiled = await self.acceptance_compiler.compile(task, requirements, diff_content)
|
|
891
|
-
except Exception as exc: # pragma: no cover - best effort
|
|
892
|
-
logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
|
|
893
|
-
compiled = {}
|
|
894
|
-
for ac_id, cmds in compiled.items():
|
|
895
|
-
# Defensive: drop any wrong-stack compiled check so it can't fail an AC
|
|
896
|
-
# for a stack reason (the compiler is told not to emit these).
|
|
897
|
-
cmds = [c for c in cmds if self._command_applicable(c)[0]]
|
|
898
|
-
ac_ok = bool(cmds)
|
|
899
|
-
compiled_cmds_by_ac[ac_id] = list(cmds)
|
|
900
|
-
for cmd in cmds:
|
|
882
|
+
continue
|
|
901
883
|
result = self._run_command(cmd, task_id=task.id)
|
|
902
884
|
command_results.append(result)
|
|
903
885
|
evidence_to_save.append(result)
|
|
886
|
+
if self._command_can_prove_acceptance(cmd_type, cmd):
|
|
887
|
+
evidence_results.append(result)
|
|
904
888
|
if result.exit_code != 0:
|
|
905
|
-
ac_ok = False
|
|
906
|
-
failing_results_by_ac.setdefault(ac_id, []).append(result)
|
|
907
889
|
if self._command_is_malformed(result):
|
|
908
890
|
had_unrunnable = True
|
|
891
|
+
# The verification command itself could not run (e.g. a
|
|
892
|
+
# SyntaxError in a `python -c` one-liner, or a missing test
|
|
893
|
+
# tool). This proves nothing about the implementation, so do
|
|
894
|
+
# not report it as a code failure — surface it as a plan/
|
|
895
|
+
# command defect the user can regenerate instead.
|
|
896
|
+
gaps.append(Gap(
|
|
897
|
+
id=self._next_gap_id(task.id, "BADCMD"),
|
|
898
|
+
severity="medium",
|
|
899
|
+
gap_type="invalid_verification_command",
|
|
900
|
+
task_id=task.id,
|
|
901
|
+
description=(
|
|
902
|
+
f"Verification command could not run (not a code failure): '{cmd}'. "
|
|
903
|
+
"It appears malformed or its tooling is unavailable, so this command "
|
|
904
|
+
"proves nothing either way."
|
|
905
|
+
),
|
|
906
|
+
evidence=[result.summary[:500]],
|
|
907
|
+
recommended_fix=(
|
|
908
|
+
"Regenerate the task's verification commands with 'dev repair', or edit "
|
|
909
|
+
"them to be a single runnable command (e.g. 'python -m pytest <file>')."
|
|
910
|
+
),
|
|
911
|
+
# Non-blocking: a command that cannot run is not evidence of a
|
|
912
|
+
# defect. If it was the *only* check for an acceptance criterion,
|
|
913
|
+
# that criterion is independently caught as unproven (blocking).
|
|
914
|
+
blocking=False,
|
|
915
|
+
suggested_command=cmd,
|
|
916
|
+
stdout_path=result.stdout_path or None,
|
|
917
|
+
stderr_path=result.stderr_path or None,
|
|
918
|
+
))
|
|
909
919
|
else:
|
|
910
|
-
|
|
920
|
+
# A verification command that genuinely failed. Lint/typecheck
|
|
921
|
+
# commands (from the config fallback) report style/type opinion,
|
|
922
|
+
# not a correctness defect, so they are ADVISORY — blocking a
|
|
923
|
+
# behaviorally-correct task on `flake8`/`mypy`/`ruff` is the
|
|
924
|
+
# false-block the benchmark surfaced. A real test failure still
|
|
925
|
+
# gates (unless compiled checks supersede it).
|
|
926
|
+
is_quality_gate = cmd_type in {"lint", "typecheck"} or self._is_quality_only_command(cmd)
|
|
927
|
+
blocking = (not compiler_active) and not is_quality_gate
|
|
928
|
+
if blocking:
|
|
929
|
+
genuine_failure = True
|
|
911
930
|
fail_file, fail_line = self._failure_location(result)
|
|
912
|
-
|
|
913
|
-
id=self._next_gap_id(task.id,
|
|
914
|
-
severity="high",
|
|
915
|
-
gap_type="test_failed",
|
|
931
|
+
gap = Gap(
|
|
932
|
+
id=self._next_gap_id(task.id, cmd_type.upper()),
|
|
933
|
+
severity="high" if blocking else "medium",
|
|
934
|
+
gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
|
|
916
935
|
task_id=task.id,
|
|
917
|
-
description=
|
|
936
|
+
description=(
|
|
937
|
+
f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
|
|
938
|
+
f"failed with exit code {result.exit_code}"
|
|
939
|
+
+ (" (advisory: style/type, not a correctness gate)." if is_quality_gate else ".")
|
|
940
|
+
),
|
|
918
941
|
evidence=[result.summary[:500]],
|
|
919
|
-
recommended_fix=f"Fix the
|
|
920
|
-
blocking=
|
|
921
|
-
acceptance_criterion_id=ac_id,
|
|
942
|
+
recommended_fix=f"Fix the issues reported by '{cmd}'.",
|
|
943
|
+
blocking=blocking,
|
|
922
944
|
suggested_command=cmd,
|
|
923
945
|
file=fail_file,
|
|
924
946
|
line=fail_line,
|
|
925
947
|
stdout_path=result.stdout_path or None,
|
|
926
948
|
stderr_path=result.stderr_path or None,
|
|
927
|
-
)
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
if
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
949
|
+
)
|
|
950
|
+
gaps.append(gap)
|
|
951
|
+
# A real test failure demoted only because the compiler is active:
|
|
952
|
+
# remember it so we can re-promote if the compiler yields no checks.
|
|
953
|
+
if compiler_active and not is_quality_gate and not blocking:
|
|
954
|
+
demoted_failures.append(gap)
|
|
955
|
+
|
|
956
|
+
# 4b. Compiled acceptance checks — precise, DevCouncil-owned per-criterion
|
|
957
|
+
# evidence. Derive one runnable check per acceptance criterion from the
|
|
958
|
+
# criterion text + the diff, instead of trusting planner-authored
|
|
959
|
+
# expected_tests (which the benchmark showed often reference absent tools or
|
|
960
|
+
# test files). Each check maps 1:1 to its criterion, replacing the coarse
|
|
961
|
+
# "any command passed -> every criterion proven" mapping.
|
|
962
|
+
compiled_pass: Dict[str, bool] = {}
|
|
963
|
+
# Per-AC bookkeeping so the unproven-AC gap can attach ONLY the check(s) that
|
|
964
|
+
# targeted that criterion (and the specific failing result), instead of dumping
|
|
965
|
+
# every command summary. Keys are AC ids; values track the compiled command(s)
|
|
966
|
+
# and any failing CommandResults for that AC.
|
|
967
|
+
compiled_cmds_by_ac: Dict[str, List[str]] = {}
|
|
968
|
+
failing_results_by_ac: Dict[str, List[CommandResult]] = {}
|
|
969
|
+
# Per-AC vote tally for proven criteria: {ac_id: (passes, decisive, repaired)}.
|
|
970
|
+
# Recorded into the stored TestEvidence so an audit can see HOW a criterion was
|
|
971
|
+
# proven (single check vs. majority of independent checks; whether a check had to
|
|
972
|
+
# be repaired to run) instead of just "passed".
|
|
973
|
+
compiled_vote: Dict[str, Tuple[int, int, bool]] = {}
|
|
974
|
+
# ACs whose independently-generated checks split (some pass, some fail) with no
|
|
975
|
+
# majority. Per policy this is inconclusive — neither proof nor a defect — so the
|
|
976
|
+
# AC is surfaced NON-blocking below instead of false-blocking on a lone bad check.
|
|
977
|
+
inconclusive_acs: set[str] = set()
|
|
978
|
+
if compile_future is not None:
|
|
979
|
+
try:
|
|
980
|
+
compiled = await compile_future
|
|
981
|
+
except Exception as exc: # pragma: no cover - best effort
|
|
982
|
+
logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
|
|
983
|
+
compiled = {}
|
|
984
|
+
ac_meta = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
|
|
985
|
+
for ac_id, raw_cmds in compiled.items():
|
|
986
|
+
# Defensive: drop any wrong-stack candidate so it can't fail an AC for a
|
|
987
|
+
# stack reason (the compiler is told not to emit these).
|
|
988
|
+
candidates = [c for c in raw_cmds if self._command_applicable(c)[0]]
|
|
989
|
+
compiled_cmds_by_ac[ac_id] = list(candidates)
|
|
990
|
+
if not candidates:
|
|
991
|
+
compiled_pass[ac_id] = False
|
|
992
|
+
continue
|
|
993
|
+
# Run each INDEPENDENT candidate; a check that merely failed to RUN
|
|
994
|
+
# (malformed/unrunnable) is regenerated from the launcher error up to
|
|
995
|
+
# ``ac_repair_attempts`` times — safe, because a check that never ran
|
|
996
|
+
# proves nothing, so repairing it cannot weaken the gate.
|
|
997
|
+
passes = 0
|
|
998
|
+
genuine_fails = 0
|
|
999
|
+
repaired = False # a check had to be regenerated before it ran
|
|
1000
|
+
fail_results: List[Tuple[str, CommandResult]] = []
|
|
1001
|
+
for cmd in candidates:
|
|
1002
|
+
result = self._run_command(cmd, task_id=task.id)
|
|
1003
|
+
command_results.append(result)
|
|
1004
|
+
evidence_to_save.append(result)
|
|
1005
|
+
attempts = 0
|
|
1006
|
+
_repair = getattr(self.acceptance_compiler, "repair", None)
|
|
1007
|
+
while (
|
|
1008
|
+
result.exit_code != 0
|
|
1009
|
+
and self._command_is_malformed(result)
|
|
1010
|
+
and attempts < ac_repair_attempts
|
|
1011
|
+
and _repair is not None
|
|
1012
|
+
):
|
|
1013
|
+
attempts += 1
|
|
1014
|
+
ac_desc = ac_meta[ac_id].description if ac_id in ac_meta else ac_id
|
|
1015
|
+
try:
|
|
1016
|
+
fixed = await _repair(
|
|
1017
|
+
ac_id, ac_desc, cmd, result.summary[:800], diff_content
|
|
1018
|
+
)
|
|
1019
|
+
except Exception:
|
|
1020
|
+
fixed = None
|
|
1021
|
+
if not fixed or not self._command_applicable(fixed)[0]:
|
|
1022
|
+
break
|
|
1023
|
+
cmd = fixed
|
|
1024
|
+
compiled_cmds_by_ac[ac_id].append(cmd)
|
|
1025
|
+
result = self._run_command(cmd, task_id=task.id)
|
|
1026
|
+
command_results.append(result)
|
|
1027
|
+
evidence_to_save.append(result)
|
|
1028
|
+
if result.exit_code == 0:
|
|
1029
|
+
passes += 1
|
|
1030
|
+
if attempts > 0:
|
|
1031
|
+
repaired = True
|
|
1032
|
+
elif self._command_is_malformed(result):
|
|
1033
|
+
# Still couldn't run after repair: proves nothing either way.
|
|
1034
|
+
had_unrunnable = True
|
|
1035
|
+
failing_results_by_ac.setdefault(ac_id, []).append(result)
|
|
1036
|
+
else:
|
|
1037
|
+
genuine_fails += 1
|
|
1038
|
+
fail_results.append((cmd, result))
|
|
1039
|
+
failing_results_by_ac.setdefault(ac_id, []).append(result)
|
|
1040
|
+
decisive = passes + genuine_fails
|
|
1041
|
+
# Majority vote over the checks that actually ran. Proven iff a strict
|
|
1042
|
+
# majority pass; unanimous failure of independent checks is strong evidence
|
|
1043
|
+
# of a real defect and blocks; a split is inconclusive (handled below).
|
|
1044
|
+
ac_proven = decisive > 0 and passes > genuine_fails
|
|
1045
|
+
compiled_pass[ac_id] = ac_proven
|
|
1046
|
+
if ac_proven:
|
|
1047
|
+
compiled_vote[ac_id] = (passes, decisive, repaired)
|
|
1048
|
+
continue
|
|
1049
|
+
if passes == 0 and genuine_fails > 0:
|
|
1050
|
+
genuine_failure = True
|
|
1051
|
+
cmd, result = fail_results[0]
|
|
1052
|
+
fail_file, fail_line = self._failure_location(result)
|
|
1053
|
+
agree = (
|
|
1054
|
+
f" {genuine_fails}/{decisive} independent checks agreed it fails."
|
|
1055
|
+
if decisive > 1 else ""
|
|
1056
|
+
)
|
|
1057
|
+
gaps.append(Gap(
|
|
1058
|
+
id=self._next_gap_id(task.id, "ACCHK"),
|
|
1059
|
+
severity="high",
|
|
1060
|
+
gap_type="test_failed",
|
|
1061
|
+
task_id=task.id,
|
|
1062
|
+
description=f"Acceptance check for {ac_id} failed: '{cmd}' (exit {result.exit_code}).{agree}",
|
|
1063
|
+
evidence=[result.summary[:500]],
|
|
1064
|
+
recommended_fix=f"Fix the implementation so acceptance criterion {ac_id} holds.",
|
|
1065
|
+
blocking=True,
|
|
985
1066
|
acceptance_criterion_id=ac_id,
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
"not precisely verified."
|
|
992
|
-
if coarse else
|
|
993
|
-
"Acceptance criterion proven by a per-criterion compiled check."
|
|
994
|
-
),
|
|
1067
|
+
suggested_command=cmd,
|
|
1068
|
+
file=fail_file,
|
|
1069
|
+
line=fail_line,
|
|
1070
|
+
stdout_path=result.stdout_path or None,
|
|
1071
|
+
stderr_path=result.stderr_path or None,
|
|
995
1072
|
))
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1073
|
+
elif passes > 0 and genuine_fails > 0:
|
|
1074
|
+
# Independent checks disagree with no majority: neither proof nor a
|
|
1075
|
+
# defect. Mark inconclusive so the unproven-AC gap below is NON-blocking
|
|
1076
|
+
# (never false-block on a lone bad check, never auto-pass a real bug).
|
|
1077
|
+
inconclusive_acs.add(ac_id)
|
|
1078
|
+
|
|
1079
|
+
# The compiler only earns the authority to demote a genuinely-failing planner
|
|
1080
|
+
# test if it produced a per-criterion check for EVERY targeted AC. A partial
|
|
1081
|
+
# compile is not enough: the uncovered ACs fall back to the coarse signal, so a
|
|
1082
|
+
# demoted real failure + coarse-proven remainder would otherwise slip past the
|
|
1083
|
+
# gate. If coverage is incomplete (or zero — empty compile / all-wrong-stack /
|
|
1084
|
+
# a compile exception swallowed to {}), re-promote the demoted failures.
|
|
1085
|
+
compiler_covered_all = bool(task.acceptance_criterion_ids) and all(
|
|
1086
|
+
compiled_cmds_by_ac.get(ac_id) for ac_id in task.acceptance_criterion_ids
|
|
1087
|
+
)
|
|
1088
|
+
if compiler_active and not compiler_covered_all and demoted_failures:
|
|
1089
|
+
for gap in demoted_failures:
|
|
1090
|
+
gap.blocking = True
|
|
1091
|
+
gap.severity = "high"
|
|
1092
|
+
genuine_failure = True
|
|
1093
|
+
logger.info(
|
|
1094
|
+
"Re-promoted demoted test failure %s to blocking: acceptance compiler "
|
|
1095
|
+
"did not produce a check for every criterion of task %s.",
|
|
1096
|
+
gap.id, task.id,
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
# 5. Acceptance-criteria evidence mapping (precise, per criterion).
|
|
1100
|
+
# Quality-only commands (lint/typecheck) are excluded: a passing `mypy`/`ruff
|
|
1101
|
+
# check`/`tsc` exercises no behavior, so it must not coarse-prove a behavioral AC
|
|
1102
|
+
# — the same false-confidence the per-criterion checks exist to prevent.
|
|
1103
|
+
successful_commands = [
|
|
1104
|
+
result for result in evidence_results
|
|
1105
|
+
if result.exit_code == 0 and not self._is_quality_only_command(result.command)
|
|
1106
|
+
]
|
|
1107
|
+
# Coarse fallback (used only when no compiled per-criterion check exists for an
|
|
1108
|
+
# AC): a criterion may be marked proven by a passing acceptance-capable command
|
|
1109
|
+
# ONLY when the task actually produced work. Without this guard a no-op run
|
|
1110
|
+
# whose unrelated command happens to pass would "prove" every criterion against
|
|
1111
|
+
# zero changes.
|
|
1112
|
+
coarse_proof_available = work_present and bool(successful_commands)
|
|
1113
|
+
if task.acceptance_criterion_ids:
|
|
1114
|
+
req_by_ac = {ac.id: req.id for req in requirements for ac in req.acceptance_criteria}
|
|
1115
|
+
unproven_acs: List[str] = []
|
|
1116
|
+
coarse_proven_acs: List[str] = []
|
|
1117
|
+
for ac_id in task.acceptance_criterion_ids:
|
|
1118
|
+
# An AC is proven if its compiled check passed; if no compiled check
|
|
1119
|
+
# exists for it, fall back to the coarse signal (any expected_test passed).
|
|
1120
|
+
proven: Optional[bool] = compiled_pass.get(ac_id)
|
|
1121
|
+
coarse = False
|
|
1122
|
+
if proven is None:
|
|
1123
|
+
proven = coarse_proof_available
|
|
1124
|
+
coarse = proven # proven only by the coarse, not-AC-specific signal
|
|
1125
|
+
if proven:
|
|
1126
|
+
if coarse:
|
|
1127
|
+
coarse_proven_acs.append(ac_id)
|
|
1128
|
+
# Don't persist a "passed" record for a coarse-proven criterion during a
|
|
1129
|
+
# run that also has a genuine blocking failure — the gate already fails,
|
|
1130
|
+
# and a stored "passed" would mislead audits that read evidence directly.
|
|
1131
|
+
if not (coarse and genuine_failure):
|
|
1132
|
+
proof_mode: Literal["compiled", "vote", "coarse", ""]
|
|
1133
|
+
if coarse:
|
|
1134
|
+
proof_summary = (
|
|
1135
|
+
"Acceptance criterion proven only by a COARSE signal (a passing "
|
|
1136
|
+
"acceptance-capable command, not a per-criterion check); behavior "
|
|
1137
|
+
"not precisely verified."
|
|
1138
|
+
)
|
|
1139
|
+
proof_mode = "coarse"
|
|
1140
|
+
else:
|
|
1141
|
+
# Make the per-criterion proof auditable: single check vs. majority
|
|
1142
|
+
# of independent checks, and whether a check had to be repaired to run.
|
|
1143
|
+
passes_n, decisive_n, was_repaired = compiled_vote.get(ac_id, (1, 1, False))
|
|
1144
|
+
proof_mode = "vote" if decisive_n > 1 else "compiled"
|
|
1145
|
+
how = (
|
|
1146
|
+
f"a majority vote of independent compiled checks ({passes_n}/{decisive_n} passed)"
|
|
1147
|
+
if decisive_n > 1 else
|
|
1148
|
+
"a per-criterion compiled check"
|
|
1149
|
+
)
|
|
1150
|
+
repaired_note = " (one check was regenerated from its launcher error to run)" if was_repaired else ""
|
|
1151
|
+
proof_summary = f"Acceptance criterion proven by {how}.{repaired_note}"
|
|
1152
|
+
evidence_to_save.append(TestEvidence(
|
|
1153
|
+
requirement_id=req_by_ac.get(ac_id, task.requirement_ids[0] if task.requirement_ids else ""),
|
|
1154
|
+
acceptance_criterion_id=ac_id,
|
|
1155
|
+
command="(devcouncil acceptance check)",
|
|
1156
|
+
status="passed",
|
|
1157
|
+
evidence_summary=proof_summary,
|
|
1158
|
+
mode=proof_mode,
|
|
1159
|
+
))
|
|
1066
1160
|
else:
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1161
|
+
unproven_acs.append(ac_id)
|
|
1162
|
+
# Surface coarse proof as a first-class advisory: these criteria passed only
|
|
1163
|
+
# because some acceptance-capable command exited 0, not because a check tied
|
|
1164
|
+
# to the criterion passed. Non-blocking, but no longer invisible.
|
|
1165
|
+
if coarse_proven_acs:
|
|
1071
1166
|
gaps.append(Gap(
|
|
1072
|
-
id=self._next_gap_id(task.id, "
|
|
1073
|
-
severity="
|
|
1074
|
-
gap_type="
|
|
1075
|
-
requirement_id=self._requirement_id_for_ac(requirements, ac_id),
|
|
1167
|
+
id=self._next_gap_id(task.id, "COARSE"),
|
|
1168
|
+
severity="low",
|
|
1169
|
+
gap_type="coarse_acceptance_proof",
|
|
1076
1170
|
task_id=task.id,
|
|
1077
1171
|
description=(
|
|
1078
|
-
|
|
1079
|
-
f"
|
|
1172
|
+
"Verification mode = COARSE for "
|
|
1173
|
+
f"{', '.join(coarse_proven_acs)}: proven by a passing acceptance-capable "
|
|
1174
|
+
"command, not a per-criterion check. Behavior is not precisely verified."
|
|
1080
1175
|
),
|
|
1081
|
-
evidence=
|
|
1082
|
-
recommended_fix=
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1176
|
+
evidence=[f"coarse-proven: {', '.join(coarse_proven_acs)}"],
|
|
1177
|
+
recommended_fix=(
|
|
1178
|
+
"Add a verification command (or test) that exercises each listed criterion "
|
|
1179
|
+
"specifically, so DevCouncil can compile a per-criterion check instead of "
|
|
1180
|
+
"relying on the coarse fallback."
|
|
1181
|
+
),
|
|
1182
|
+
blocking=False,
|
|
1087
1183
|
))
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1184
|
+
if unproven_acs:
|
|
1185
|
+
# Block only on positive evidence of a problem. If verification was
|
|
1186
|
+
# attempted but every failure was unrunnable (missing tooling / tests)
|
|
1187
|
+
# and nothing genuinely failed, that is a verification defect, not a
|
|
1188
|
+
# code defect — surface it as a non-blocking "could not verify".
|
|
1189
|
+
couldnt_verify = had_unrunnable and not genuine_failure and work_present
|
|
1190
|
+
ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
|
|
1191
|
+
# Methods whose criteria HARD-BLOCK the gate when unproven: only those
|
|
1192
|
+
# that assert BEHAVIOR. Inherently-manual criteria (manual/llm_review),
|
|
1193
|
+
# optional ones, and quality-only `static_check` criteria (PEP 8 /
|
|
1194
|
+
# docstring / formatting) are surfaced for review instead of
|
|
1195
|
+
# false-blocking the autonomous loop. static_check is a quality gate,
|
|
1196
|
+
# not a correctness gate — mirroring how lint/type COMMAND failures are
|
|
1197
|
+
# already demoted to advisory — and the compiler often cannot author a
|
|
1198
|
+
# reliable style check (or the criterion lands on a no-diff process task),
|
|
1199
|
+
# which otherwise blocks correct, style-conforming code.
|
|
1200
|
+
automatable_methods = {"unit_test", "integration_test"}
|
|
1201
|
+
for ac_id in unproven_acs:
|
|
1202
|
+
ac = ac_by_id.get(ac_id)
|
|
1203
|
+
method = ac.verification_method if ac else "unit_test"
|
|
1204
|
+
is_automatable = (ac.required if ac else True) and method in automatable_methods
|
|
1205
|
+
if not is_automatable:
|
|
1206
|
+
blocks = False
|
|
1207
|
+
optional = "" if (ac is None or ac.required) else " optional"
|
|
1208
|
+
fix = (
|
|
1209
|
+
f"This{optional} criterion's verification method is '{method}'; it cannot be "
|
|
1210
|
+
"proven by running code. Review it manually (it does not block the gate)."
|
|
1211
|
+
)
|
|
1212
|
+
suffix = f" (non-blocking: {method})"
|
|
1213
|
+
elif ac_id in inconclusive_acs:
|
|
1214
|
+
# Independently-generated checks split with no majority — inconclusive,
|
|
1215
|
+
# so this does not block (a lone bad check must not fail correct code).
|
|
1216
|
+
blocks = False
|
|
1217
|
+
fix = ("Auto-generated acceptance checks disagreed on this criterion (some "
|
|
1218
|
+
"passed, some failed). Add a precise verification command that "
|
|
1219
|
+
"unambiguously proves it so the result is decisive.")
|
|
1220
|
+
suffix = " (auto-checks inconclusive)"
|
|
1221
|
+
elif couldnt_verify:
|
|
1222
|
+
blocks = False
|
|
1223
|
+
fix = ("Could not verify this criterion: the verification commands did not run "
|
|
1224
|
+
"(missing tooling or tests). Regenerate them with 'dev repair' to confirm the work.")
|
|
1225
|
+
suffix = " (verification commands could not run)"
|
|
1226
|
+
else:
|
|
1227
|
+
blocks = True
|
|
1228
|
+
fix = "Add or fix a verification command that proves this acceptance criterion."
|
|
1229
|
+
suffix = ""
|
|
1230
|
+
# Concrete, AC-scoped evidence instead of "all command summaries":
|
|
1231
|
+
# * if a compiled check targeted this AC, attach its command(s) and
|
|
1232
|
+
# the specific failing result;
|
|
1233
|
+
# * otherwise an explicit "no check compiled" marker so the agent
|
|
1234
|
+
# knows it must author one, not hunt through unrelated output.
|
|
1235
|
+
ac_compiled = compiled_cmds_by_ac.get(ac_id, [])
|
|
1236
|
+
ac_failures = failing_results_by_ac.get(ac_id, [])
|
|
1237
|
+
ac_evidence: List[str] = []
|
|
1238
|
+
suggested_cmd: Optional[str] = None
|
|
1239
|
+
if ac_compiled:
|
|
1240
|
+
suggested_cmd = ac_compiled[0]
|
|
1241
|
+
ac_evidence.extend(f"compiled check: {c}" for c in ac_compiled)
|
|
1242
|
+
ac_evidence.extend(r.summary[:500] for r in ac_failures)
|
|
1243
|
+
else:
|
|
1244
|
+
ac_evidence.append(
|
|
1245
|
+
f"no DevCouncil check compiled for {ac_id} "
|
|
1246
|
+
f"(expected verification method: {method})"
|
|
1247
|
+
)
|
|
1248
|
+
gaps.append(Gap(
|
|
1249
|
+
id=self._next_gap_id(task.id, "AC"),
|
|
1250
|
+
severity="high" if blocks else "medium",
|
|
1251
|
+
gap_type="acceptance_criteria_unproven",
|
|
1252
|
+
requirement_id=self._requirement_id_for_ac(requirements, ac_id),
|
|
1253
|
+
task_id=task.id,
|
|
1254
|
+
description=(
|
|
1255
|
+
f"Acceptance criterion {ac_id} has no passing verification evidence "
|
|
1256
|
+
f"for task {task.id}.{suffix}"
|
|
1257
|
+
),
|
|
1258
|
+
evidence=ac_evidence,
|
|
1259
|
+
recommended_fix=fix,
|
|
1260
|
+
blocking=blocks,
|
|
1261
|
+
acceptance_criterion_id=ac_id,
|
|
1262
|
+
expected_verification_method=method,
|
|
1263
|
+
suggested_command=suggested_cmd,
|
|
1264
|
+
))
|
|
1265
|
+
elif task.requirement_ids:
|
|
1266
|
+
gaps.append(Gap(
|
|
1267
|
+
id=self._next_gap_id(task.id, "NOAC"),
|
|
1268
|
+
severity="high",
|
|
1269
|
+
gap_type="acceptance_criteria_unproven",
|
|
1270
|
+
requirement_id=task.requirement_ids[0],
|
|
1126
1271
|
task_id=task.id,
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
covered_lines=cov.covered_changed_lines,
|
|
1131
|
-
coverage_ratio=cov.ratio,
|
|
1132
|
-
uncovered_by_file=cov.uncovered_by_file,
|
|
1133
|
-
absent_files=cov.absent_files,
|
|
1134
|
-
summary=cov.summary(),
|
|
1272
|
+
description=f"Task {task.id} is linked to requirements but no acceptance criteria.",
|
|
1273
|
+
recommended_fix="Link the task to specific acceptance_criterion_ids before verification.",
|
|
1274
|
+
blocking=True,
|
|
1135
1275
|
))
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1276
|
+
|
|
1277
|
+
# 5b. Diff↔coverage gate. A green suite is only acceptance evidence if it
|
|
1278
|
+
# exercised the lines the diff changed. This catches the failure the README
|
|
1279
|
+
# promises to stop: tests "pass" while the new logic is never run (unrelated
|
|
1280
|
+
# suite, code never imported, untouched branch). Measured only when the target
|
|
1281
|
+
# repo has coverage tooling and the diff has measurable Python changes; absent
|
|
1282
|
+
# that, it degrades silently rather than blocking correct work.
|
|
1283
|
+
measure_cov, enforce_cov, min_ratio = self._diff_coverage_settings()
|
|
1284
|
+
any_passing = bool(successful_commands) or any(compiled_pass.values())
|
|
1285
|
+
coverage_measured = False
|
|
1286
|
+
coverage_skipped_reason: Optional[str] = None
|
|
1287
|
+
if not measure_cov:
|
|
1288
|
+
coverage_skipped_reason = "diff coverage disabled in config"
|
|
1289
|
+
elif not diff_content:
|
|
1290
|
+
coverage_skipped_reason = "no diff to measure"
|
|
1291
|
+
elif not task.acceptance_criterion_ids:
|
|
1292
|
+
coverage_skipped_reason = "task has no acceptance criteria"
|
|
1293
|
+
elif not any_passing:
|
|
1294
|
+
coverage_skipped_reason = "no passing verification command to instrument"
|
|
1295
|
+
if measure_cov and diff_content and task.acceptance_criterion_ids and any_passing:
|
|
1296
|
+
cov = self.measure_diff_coverage(task, diff_content)
|
|
1297
|
+
if not cov.measured:
|
|
1298
|
+
coverage_skipped_reason = cov.reason or "diff coverage could not be measured"
|
|
1299
|
+
if cov.measured:
|
|
1300
|
+
coverage_measured = True
|
|
1301
|
+
coverage_skipped_reason = None
|
|
1302
|
+
evidence_to_save.append(DiffCoverageEvidence(
|
|
1145
1303
|
task_id=task.id,
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
],
|
|
1155
|
-
recommended_fix=(
|
|
1156
|
-
"Add or extend a test that executes the changed lines, then re-verify. "
|
|
1157
|
-
"A passing suite that does not run the new code is not acceptance evidence."
|
|
1158
|
-
),
|
|
1159
|
-
# Off by default (signal first); teams opt into blocking via
|
|
1160
|
-
# verification.diff_coverage.enforce.
|
|
1161
|
-
blocking=enforce_cov,
|
|
1162
|
-
file=first_file,
|
|
1163
|
-
line=first_lines[0] if first_lines else None,
|
|
1164
|
-
suggested_command=target_cmds[0] if target_cmds else None,
|
|
1304
|
+
tool=cov.tool,
|
|
1305
|
+
measured=True,
|
|
1306
|
+
changed_lines=cov.changed_executable_lines,
|
|
1307
|
+
covered_lines=cov.covered_changed_lines,
|
|
1308
|
+
coverage_ratio=cov.ratio,
|
|
1309
|
+
uncovered_by_file=cov.uncovered_by_file,
|
|
1310
|
+
absent_files=cov.absent_files,
|
|
1311
|
+
summary=cov.summary(),
|
|
1165
1312
|
))
|
|
1313
|
+
failing = cov.covered_changed_lines == 0 if min_ratio <= 0 else cov.ratio < min_ratio
|
|
1314
|
+
if failing:
|
|
1315
|
+
first_file = next(iter(cov.uncovered_by_file), None)
|
|
1316
|
+
first_lines = cov.uncovered_by_file.get(first_file or "", [])
|
|
1317
|
+
target_cmds = self._coverage_target_commands(task)
|
|
1318
|
+
gaps.append(Gap(
|
|
1319
|
+
id=self._next_gap_id(task.id, "DIFFCOV"),
|
|
1320
|
+
severity="high" if enforce_cov else "medium",
|
|
1321
|
+
gap_type="diff_not_exercised",
|
|
1322
|
+
task_id=task.id,
|
|
1323
|
+
description=(
|
|
1324
|
+
f"Verification commands passed but exercised "
|
|
1325
|
+
f"{cov.covered_changed_lines}/{cov.changed_executable_lines} changed line(s): "
|
|
1326
|
+
f"{cov.summary()}. The acceptance criteria are not proven because the new "
|
|
1327
|
+
"logic was never executed by the tests."
|
|
1328
|
+
),
|
|
1329
|
+
evidence=[cov.summary()] + [
|
|
1330
|
+
f"{path}: lines {lines}" for path, lines in list(cov.uncovered_by_file.items())[:5]
|
|
1331
|
+
],
|
|
1332
|
+
recommended_fix=(
|
|
1333
|
+
"Add or extend a test that executes the changed lines, then re-verify. "
|
|
1334
|
+
"A passing suite that does not run the new code is not acceptance evidence."
|
|
1335
|
+
),
|
|
1336
|
+
# Off by default (signal first); teams opt into blocking via
|
|
1337
|
+
# verification.diff_coverage.enforce.
|
|
1338
|
+
blocking=enforce_cov,
|
|
1339
|
+
file=first_file,
|
|
1340
|
+
line=first_lines[0] if first_lines else None,
|
|
1341
|
+
suggested_command=target_cmds[0] if target_cmds else None,
|
|
1342
|
+
))
|
|
1166
1343
|
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
self.last_outcome = VerificationOutcome(
|
|
1205
|
-
mode="compiled" if self.acceptance_compiler else "coarse",
|
|
1206
|
-
compiler_active=compiler_active,
|
|
1207
|
-
diff_empty=diff_empty,
|
|
1208
|
-
coverage_measured=coverage_measured,
|
|
1209
|
-
coverage_skipped_reason=coverage_skipped_reason,
|
|
1210
|
-
)
|
|
1211
|
-
return gaps, evidence_to_save
|
|
1344
|
+
# 6. Secret scan
|
|
1345
|
+
if diff_content:
|
|
1346
|
+
gaps.extend(self.secret_scanner.scan_diff(diff_content, task.id))
|
|
1347
|
+
|
|
1348
|
+
# 7. LLM Implementation Review (ADVISORY ONLY).
|
|
1349
|
+
# DevCouncil's authority is executable evidence, not model confidence — so
|
|
1350
|
+
# an LLM reviewer must never block on its own say-so. Subjective reviewers
|
|
1351
|
+
# over-flag correct code (false negatives that erode trust in "blocked"),
|
|
1352
|
+
# so review findings are surfaced as non-blocking signals. A genuine
|
|
1353
|
+
# requirement gap is caught by the acceptance-criteria evidence checks
|
|
1354
|
+
# above; the review just adds human-facing context.
|
|
1355
|
+
if review_future is not None:
|
|
1356
|
+
try:
|
|
1357
|
+
review_result = await review_future
|
|
1358
|
+
for finding in review_result.findings:
|
|
1359
|
+
finding.id = self._next_gap_id(task.id, "REVIEW")
|
|
1360
|
+
finding.blocking = False
|
|
1361
|
+
gaps.append(finding)
|
|
1362
|
+
except Exception as e:
|
|
1363
|
+
logger.error("Implementation review failed: %s", e)
|
|
1364
|
+
|
|
1365
|
+
# 8. Open live-review cards
|
|
1366
|
+
for card in unresolved_blocking_cards(self.project_root, task_id=task.id):
|
|
1367
|
+
gaps.append(Gap(
|
|
1368
|
+
id=self._next_gap_id(task.id, "LIVE"),
|
|
1369
|
+
severity="critical",
|
|
1370
|
+
gap_type="architecture_drift",
|
|
1371
|
+
task_id=task.id,
|
|
1372
|
+
description=f"Open critical live-review card remains: {card.summary}",
|
|
1373
|
+
evidence=[card.id, card.message_for_agent],
|
|
1374
|
+
recommended_fix=(
|
|
1375
|
+
f"Address the critique card, then run `dev watch resolve {card.id}` "
|
|
1376
|
+
"or mark it ignored with justification outside the verification gate."
|
|
1377
|
+
),
|
|
1378
|
+
blocking=True,
|
|
1379
|
+
))
|
|
1212
1380
|
|
|
1213
|
-
|
|
1381
|
+
self.last_outcome = VerificationOutcome(
|
|
1382
|
+
mode="compiled" if self.acceptance_compiler else "coarse",
|
|
1383
|
+
compiler_active=compiler_active,
|
|
1384
|
+
diff_empty=diff_empty,
|
|
1385
|
+
coverage_measured=coverage_measured,
|
|
1386
|
+
coverage_skipped_reason=coverage_skipped_reason,
|
|
1387
|
+
)
|
|
1388
|
+
return gaps, evidence_to_save
|
|
1389
|
+
finally:
|
|
1390
|
+
# Always drain the two background LLM tasks (even if the body raised
|
|
1391
|
+
# before their await points) so neither is destroyed-while-pending nor
|
|
1392
|
+
# logs 'exception never retrieved', and clear the per-call memos so a
|
|
1393
|
+
# later non-verify_task call on this instance recomputes fresh.
|
|
1394
|
+
for _fut in (compile_future, review_future):
|
|
1395
|
+
if _fut is not None:
|
|
1396
|
+
if not _fut.done():
|
|
1397
|
+
_fut.cancel()
|
|
1398
|
+
try:
|
|
1399
|
+
await _fut
|
|
1400
|
+
except (asyncio.CancelledError, Exception):
|
|
1401
|
+
pass
|
|
1402
|
+
self._untracked_cache = None
|
|
1403
|
+
self._command_timeout_cache = None
|
|
1404
|
+
# Reload project dependencies next run: a reused Verifier may verify a later
|
|
1405
|
+
# task after pyproject/requirements changed on disk.
|
|
1406
|
+
self._project_deps_cache = None
|
|
1407
|
+
|
|
1408
|
+
def _task_intent_text(self, task: Task, requirements: Optional[List[Requirement]]) -> str:
|
|
1409
|
+
"""Lowercased text describing what the task is meant to do — its title,
|
|
1410
|
+
description, and the descriptions of its acceptance criteria. Used to tell an
|
|
1411
|
+
INTENDED public-API change ("remove deprecated foo") from silent drift."""
|
|
1412
|
+
parts = [task.title or "", task.description or ""]
|
|
1413
|
+
if requirements:
|
|
1414
|
+
ac_ids = set(task.acceptance_criterion_ids)
|
|
1415
|
+
for req in requirements:
|
|
1416
|
+
for ac in req.acceptance_criteria:
|
|
1417
|
+
if ac.id in ac_ids:
|
|
1418
|
+
parts.append(ac.description or "")
|
|
1419
|
+
return " ".join(parts).lower()
|
|
1420
|
+
|
|
1421
|
+
def _check_semantic_diff(self, task: Task, requirements: Optional[List[Requirement]] = None) -> List[Gap]:
|
|
1214
1422
|
gaps: List[Gap] = []
|
|
1215
1423
|
semantic_path = self.project_root / ".devcouncil" / "semantic" / task.id
|
|
1216
1424
|
after_path = semantic_path / "after.json"
|
|
@@ -1225,10 +1433,45 @@ class Verifier:
|
|
|
1225
1433
|
return gaps
|
|
1226
1434
|
|
|
1227
1435
|
planned_paths = {pf.path for pf in task.planned_files}
|
|
1228
|
-
|
|
1436
|
+
classifications = result.get("classifications", [])
|
|
1437
|
+
# Drift signal inputs: a public symbol re-added elsewhere is a move/rename (a
|
|
1438
|
+
# legitimate refactor, not drift); and the task's own intent text lets a removal
|
|
1439
|
+
# the task actually asked for ("remove deprecated foo") pass without false-blocking.
|
|
1440
|
+
readded_public = {
|
|
1441
|
+
item.get("name") for item in classifications
|
|
1442
|
+
if item.get("type") == "exported_symbol_added" and item.get("name")
|
|
1443
|
+
}
|
|
1444
|
+
intent_text = self._task_intent_text(task, requirements)
|
|
1445
|
+
for item in classifications:
|
|
1229
1446
|
change_type = item.get("type", "")
|
|
1230
1447
|
path = item.get("path", "")
|
|
1231
|
-
if change_type == "
|
|
1448
|
+
if change_type == "exported_symbol_removed":
|
|
1449
|
+
# An executor deleting/renaming an existing PUBLIC symbol — even inside a
|
|
1450
|
+
# file it is allowed to touch — is scope drift / a regression the focused
|
|
1451
|
+
# task rarely intends. Block it UNLESS the symbol was re-added elsewhere
|
|
1452
|
+
# (a move/rename) or the task text explicitly calls for the removal.
|
|
1453
|
+
name = item.get("name", "")
|
|
1454
|
+
moved = name in readded_public
|
|
1455
|
+
intended = bool(name) and name.lower() in intent_text
|
|
1456
|
+
gaps.append(Gap(
|
|
1457
|
+
id=self._next_gap_id(task.id, "DRIFT"),
|
|
1458
|
+
severity="high",
|
|
1459
|
+
gap_type="architecture_drift",
|
|
1460
|
+
task_id=task.id,
|
|
1461
|
+
description=(
|
|
1462
|
+
f"Public symbol '{name}' was removed from {path} — possible scope "
|
|
1463
|
+
"drift: the executor changed a public API the task did not call for."
|
|
1464
|
+
),
|
|
1465
|
+
evidence=[f"{path}:{name}"],
|
|
1466
|
+
recommended_fix=(
|
|
1467
|
+
"Restore the removed public symbol. If its removal IS part of this "
|
|
1468
|
+
"task, state that in the task description / acceptance criteria so the "
|
|
1469
|
+
"change is an intended, reviewed decision rather than silent drift."
|
|
1470
|
+
),
|
|
1471
|
+
blocking=(not moved and not intended),
|
|
1472
|
+
file=path,
|
|
1473
|
+
))
|
|
1474
|
+
elif change_type == "public_api_change" and path not in planned_paths:
|
|
1232
1475
|
gaps.append(Gap(
|
|
1233
1476
|
id=self._next_gap_id(task.id, "SEM"),
|
|
1234
1477
|
severity="high",
|
|
@@ -1239,17 +1482,66 @@ class Verifier:
|
|
|
1239
1482
|
recommended_fix="Add file to planned_files and document acceptance criteria.",
|
|
1240
1483
|
blocking=not bool(task.acceptance_criterion_ids),
|
|
1241
1484
|
))
|
|
1242
|
-
elif change_type == "
|
|
1485
|
+
elif change_type == "public_api_change" and path in planned_paths:
|
|
1486
|
+
# The file is in scope, but the executor changed the SIGNATURE of an
|
|
1487
|
+
# existing public symbol. Tasks legitimately change signatures of files
|
|
1488
|
+
# they own, so this is ADVISORY only — surfaced so an audit/agent can see
|
|
1489
|
+
# the public contract moved, not silently drifted.
|
|
1243
1490
|
gaps.append(Gap(
|
|
1244
|
-
id=self._next_gap_id(task.id, "
|
|
1491
|
+
id=self._next_gap_id(task.id, "SIGDRIFT"),
|
|
1245
1492
|
severity="medium",
|
|
1246
|
-
gap_type="
|
|
1493
|
+
gap_type="architecture_drift",
|
|
1247
1494
|
task_id=task.id,
|
|
1248
|
-
description=
|
|
1249
|
-
|
|
1250
|
-
|
|
1495
|
+
description=(
|
|
1496
|
+
f"Public API signature change in planned file {path}"
|
|
1497
|
+
+ (f" ({item.get('name')})" if item.get("name") else "")
|
|
1498
|
+
+ ". Confirm callers are updated and the change is intended."
|
|
1499
|
+
),
|
|
1500
|
+
evidence=[f"{path}:{item.get('name', '')}"],
|
|
1501
|
+
recommended_fix=(
|
|
1502
|
+
"If the signature change is part of this task, note it in the task "
|
|
1503
|
+
"description / acceptance criteria; otherwise revert it."
|
|
1504
|
+
),
|
|
1251
1505
|
blocking=False,
|
|
1252
1506
|
))
|
|
1507
|
+
elif change_type == "import_dependency_change":
|
|
1508
|
+
# A NEW third-party top-level package added to the diff is supply-chain
|
|
1509
|
+
# drift — block it. Everything else (stdlib, relative/local, or an
|
|
1510
|
+
# already-declared/available dependency) stays advisory, and only on an
|
|
1511
|
+
# unplanned file (an unplanned file is already orphan-blocked anyway).
|
|
1512
|
+
statement = item.get("statement", "")
|
|
1513
|
+
top = self._import_top_level(statement)
|
|
1514
|
+
new_third_party = self._is_new_third_party_import(top)
|
|
1515
|
+
if new_third_party:
|
|
1516
|
+
gaps.append(Gap(
|
|
1517
|
+
id=self._next_gap_id(task.id, "DEPADD"),
|
|
1518
|
+
severity="high",
|
|
1519
|
+
gap_type="dependency_risk",
|
|
1520
|
+
task_id=task.id,
|
|
1521
|
+
description=(
|
|
1522
|
+
f"New undeclared third-party dependency '{top}' imported in {path} "
|
|
1523
|
+
f"({statement.strip()}). Adding a dependency the task did not plan is "
|
|
1524
|
+
"supply-chain drift."
|
|
1525
|
+
),
|
|
1526
|
+
evidence=[path, statement.strip()],
|
|
1527
|
+
recommended_fix=(
|
|
1528
|
+
f"Declare '{top}' in the project's dependencies and plan the change, "
|
|
1529
|
+
"or use an existing/standard-library alternative."
|
|
1530
|
+
),
|
|
1531
|
+
blocking=True,
|
|
1532
|
+
file=path,
|
|
1533
|
+
))
|
|
1534
|
+
elif path not in planned_paths:
|
|
1535
|
+
gaps.append(Gap(
|
|
1536
|
+
id=self._next_gap_id(task.id, "IMP"),
|
|
1537
|
+
severity="medium",
|
|
1538
|
+
gap_type="dependency_risk",
|
|
1539
|
+
task_id=task.id,
|
|
1540
|
+
description=f"Import dependency change in {path}.",
|
|
1541
|
+
evidence=[path],
|
|
1542
|
+
recommended_fix="Confirm dependency change is intentional.",
|
|
1543
|
+
blocking=False,
|
|
1544
|
+
))
|
|
1253
1545
|
elif change_type == "config_schema_dependency_change" and path not in planned_paths:
|
|
1254
1546
|
gaps.append(Gap(
|
|
1255
1547
|
id=self._next_gap_id(task.id, "CFG"),
|
|
@@ -1263,6 +1555,102 @@ class Verifier:
|
|
|
1263
1555
|
))
|
|
1264
1556
|
return gaps
|
|
1265
1557
|
|
|
1558
|
+
@staticmethod
|
|
1559
|
+
def _import_top_level(statement: str) -> Optional[str]:
|
|
1560
|
+
"""Top-level package of an import statement, or None for relative/local/unparseable.
|
|
1561
|
+
|
|
1562
|
+
``import requests`` / ``import os.path`` -> the first dotted component; ``from x.y
|
|
1563
|
+
import z`` -> ``x``; ``from . import z`` / ``from .mod import z`` -> None (relative).
|
|
1564
|
+
"""
|
|
1565
|
+
s = (statement or "").strip()
|
|
1566
|
+
if s.startswith("import "):
|
|
1567
|
+
first = s[len("import "):].split(",")[0].strip()
|
|
1568
|
+
top = first.split(" as ")[0].strip().split(".")[0].strip()
|
|
1569
|
+
return top or None
|
|
1570
|
+
if s.startswith("from "):
|
|
1571
|
+
rest = s[len("from "):].lstrip()
|
|
1572
|
+
if rest.startswith("."): # relative import -> local, never a new dependency
|
|
1573
|
+
return None
|
|
1574
|
+
mod = rest.split(" import ")[0].strip()
|
|
1575
|
+
return (mod.split(".")[0].strip() or None) if mod else None
|
|
1576
|
+
return None
|
|
1577
|
+
|
|
1578
|
+
def _is_new_third_party_import(self, top: Optional[str]) -> bool:
|
|
1579
|
+
"""True only when ``top`` is a genuinely new, undeclared third-party package.
|
|
1580
|
+
|
|
1581
|
+
Conservative on purpose (this gates a BLOCK): a module is NOT flagged when it is
|
|
1582
|
+
the standard library, a declared project dependency, or already importable in the
|
|
1583
|
+
environment (so import-name vs distribution-name mismatches like ``yaml``/``pyyaml``
|
|
1584
|
+
never false-block). Only a package that is none of those — i.e. undeclared AND not
|
|
1585
|
+
present — counts as supply-chain drift."""
|
|
1586
|
+
if not top:
|
|
1587
|
+
return False
|
|
1588
|
+
if top in self._stdlib_modules():
|
|
1589
|
+
return False
|
|
1590
|
+
if top.lower() in self._project_dependencies():
|
|
1591
|
+
return False
|
|
1592
|
+
try:
|
|
1593
|
+
import importlib.util
|
|
1594
|
+
if importlib.util.find_spec(top) is not None:
|
|
1595
|
+
return False # already available in the environment; not a new dependency
|
|
1596
|
+
except Exception:
|
|
1597
|
+
# A find_spec error (e.g. a partially-installed parent) is ambiguous; do not
|
|
1598
|
+
# block on ambiguity.
|
|
1599
|
+
return False
|
|
1600
|
+
return True
|
|
1601
|
+
|
|
1602
|
+
@staticmethod
|
|
1603
|
+
def _stdlib_modules() -> frozenset:
|
|
1604
|
+
names = getattr(sys, "stdlib_module_names", None)
|
|
1605
|
+
return frozenset(names) if names else frozenset()
|
|
1606
|
+
|
|
1607
|
+
def _project_dependencies(self) -> set:
|
|
1608
|
+
"""Lower-cased distribution names declared by the project (pyproject/requirements/
|
|
1609
|
+
package.json). Cached per Verifier instance; best-effort (parse errors are ignored)."""
|
|
1610
|
+
cached = getattr(self, "_project_deps_cache", None)
|
|
1611
|
+
if cached is not None:
|
|
1612
|
+
return cached
|
|
1613
|
+
deps: set = set()
|
|
1614
|
+
split_re = r"[><=!~;\[\] ]"
|
|
1615
|
+
pyproject = self.project_root / "pyproject.toml"
|
|
1616
|
+
if pyproject.exists():
|
|
1617
|
+
try:
|
|
1618
|
+
import tomllib
|
|
1619
|
+
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
|
|
1620
|
+
project = data.get("project", {}) or {}
|
|
1621
|
+
for dep in project.get("dependencies", []) or []:
|
|
1622
|
+
pkg = re.split(split_re, dep.strip())[0].strip().lower()
|
|
1623
|
+
if pkg:
|
|
1624
|
+
deps.add(pkg)
|
|
1625
|
+
for group in (project.get("optional-dependencies", {}) or {}).values():
|
|
1626
|
+
for dep in group or []:
|
|
1627
|
+
pkg = re.split(split_re, dep.strip())[0].strip().lower()
|
|
1628
|
+
if pkg:
|
|
1629
|
+
deps.add(pkg)
|
|
1630
|
+
except Exception:
|
|
1631
|
+
pass
|
|
1632
|
+
requirements = self.project_root / "requirements.txt"
|
|
1633
|
+
if requirements.exists():
|
|
1634
|
+
try:
|
|
1635
|
+
for line in requirements.read_text(encoding="utf-8").splitlines():
|
|
1636
|
+
line = line.strip()
|
|
1637
|
+
if line and not line.startswith("#"):
|
|
1638
|
+
pkg = re.split(split_re, line)[0].strip().lower()
|
|
1639
|
+
if pkg:
|
|
1640
|
+
deps.add(pkg)
|
|
1641
|
+
except Exception:
|
|
1642
|
+
pass
|
|
1643
|
+
package_json = self.project_root / "package.json"
|
|
1644
|
+
if package_json.exists():
|
|
1645
|
+
try:
|
|
1646
|
+
data = json.loads(package_json.read_text(encoding="utf-8"))
|
|
1647
|
+
for key in ("dependencies", "devDependencies", "optionalDependencies"):
|
|
1648
|
+
deps.update(k.lower() for k in (data.get(key) or {}).keys())
|
|
1649
|
+
except Exception:
|
|
1650
|
+
pass
|
|
1651
|
+
self._project_deps_cache = deps
|
|
1652
|
+
return deps
|
|
1653
|
+
|
|
1266
1654
|
# Signatures that mean the verification command itself could not run (or had
|
|
1267
1655
|
# nothing to run), so its non-zero exit says nothing about whether the
|
|
1268
1656
|
# implementation is correct — a tooling/plan defect, not a code defect.
|