devcouncil 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -30
- package/package.json +6 -2
- package/packages/codeintel-grammars/hatch_build.py +43 -0
- package/packages/codeintel-grammars/pyproject.toml +16 -0
- package/packages/codeintel-grammars/src/devcouncil_codeintel_grammars/__init__.py +93 -0
- package/pyproject.toml +99 -4
- package/src/devcouncil/app/config.py +512 -20
- package/src/devcouncil/app/events.py +4 -23
- package/src/devcouncil/app/orchestrator.py +5 -0
- package/src/devcouncil/app/run_context.py +3 -3
- package/src/devcouncil/assets/__init__.py +4 -1
- package/src/devcouncil/assets/vendor/force-graph.min.js +5 -0
- package/src/devcouncil/campaign/__init__.py +71 -0
- package/src/devcouncil/campaign/bloom.py +137 -0
- package/src/devcouncil/campaign/dashboard.py +123 -0
- package/src/devcouncil/campaign/mailbox.py +305 -0
- package/src/devcouncil/campaign/notify.py +91 -0
- package/src/devcouncil/campaign/orchestrator.py +592 -0
- package/src/devcouncil/campaign/prompts/coordinator.md +29 -0
- package/src/devcouncil/campaign/prompts/director.md +21 -0
- package/src/devcouncil/campaign/prompts/protocol.md +46 -0
- package/src/devcouncil/campaign/prompts/reviewer.md +24 -0
- package/src/devcouncil/campaign/prompts/worker.md +24 -0
- package/src/devcouncil/campaign/roles.py +202 -0
- package/src/devcouncil/campaign/watcher.py +153 -0
- package/src/devcouncil/cli/commands/agents.py +24 -17
- package/src/devcouncil/cli/commands/artifacts.py +36 -27
- package/src/devcouncil/cli/commands/ast.py +12 -3
- package/src/devcouncil/cli/commands/baseline.py +21 -12
- package/src/devcouncil/cli/commands/boot.py +218 -0
- package/src/devcouncil/cli/commands/campaign.py +302 -0
- package/src/devcouncil/cli/commands/check.py +225 -12
- package/src/devcouncil/cli/commands/config.py +221 -74
- package/src/devcouncil/cli/commands/cost.py +137 -28
- package/src/devcouncil/cli/commands/dashboard.py +12 -4
- package/src/devcouncil/cli/commands/debug_cmd.py +249 -0
- package/src/devcouncil/cli/commands/design.py +27 -17
- package/src/devcouncil/cli/commands/doctor.py +790 -8
- package/src/devcouncil/cli/commands/evidence.py +41 -20
- package/src/devcouncil/cli/commands/export.py +73 -0
- package/src/devcouncil/cli/commands/gaps.py +175 -0
- package/src/devcouncil/cli/commands/gated_write.py +76 -0
- package/src/devcouncil/cli/commands/go.py +220 -68
- package/src/devcouncil/cli/commands/graph_cmd.py +1192 -0
- package/src/devcouncil/cli/commands/handoff.py +45 -34
- package/src/devcouncil/cli/commands/hook.py +630 -85
- package/src/devcouncil/cli/commands/init.py +89 -30
- package/src/devcouncil/cli/commands/integrate.py +296 -1385
- package/src/devcouncil/cli/commands/lease.py +120 -0
- package/src/devcouncil/cli/commands/logs.py +12 -5
- package/src/devcouncil/cli/commands/lsp.py +40 -5
- package/src/devcouncil/cli/commands/map.py +317 -74
- package/src/devcouncil/cli/commands/mcp_server.py +12 -2
- package/src/devcouncil/cli/commands/okf.py +44 -6
- package/src/devcouncil/cli/commands/plan.py +184 -69
- package/src/devcouncil/cli/commands/prompt.py +26 -17
- package/src/devcouncil/cli/commands/provenance.py +79 -0
- package/src/devcouncil/cli/commands/repair.py +60 -49
- package/src/devcouncil/cli/commands/report.py +148 -40
- package/src/devcouncil/cli/commands/requirements.py +104 -0
- package/src/devcouncil/cli/commands/reset_demo_state.py +13 -4
- package/src/devcouncil/cli/commands/rollback.py +46 -35
- package/src/devcouncil/cli/commands/run.py +173 -8
- package/src/devcouncil/cli/commands/runs.py +298 -68
- package/src/devcouncil/cli/commands/scaffold.py +33 -12
- package/src/devcouncil/cli/commands/semantic.py +29 -14
- package/src/devcouncil/cli/commands/setup.py +103 -93
- package/src/devcouncil/cli/commands/shell.py +51 -42
- package/src/devcouncil/cli/commands/show.py +56 -42
- package/src/devcouncil/cli/commands/skills.py +29 -20
- package/src/devcouncil/cli/commands/status.py +80 -67
- package/src/devcouncil/cli/commands/task_gate.py +295 -0
- package/src/devcouncil/cli/commands/tasks.py +248 -19
- package/src/devcouncil/cli/commands/trace.py +14 -8
- package/src/devcouncil/cli/commands/verify.py +33 -8
- package/src/devcouncil/cli/commands/version.py +14 -6
- package/src/devcouncil/cli/commands/watch.py +49 -30
- package/src/devcouncil/cli/commands/watch_fs.py +30 -19
- package/src/devcouncil/cli/commands/wiki.py +278 -0
- package/src/devcouncil/cli/main.py +58 -1
- package/src/devcouncil/codeintel/__init__.py +16 -0
- package/src/devcouncil/codeintel/build_control.py +429 -0
- package/src/devcouncil/codeintel/build_worker.py +78 -0
- package/src/devcouncil/codeintel/debug/__init__.py +17 -0
- package/src/devcouncil/codeintel/debug/broker.py +114 -0
- package/src/devcouncil/codeintel/debug/broker_client.py +61 -0
- package/src/devcouncil/codeintel/debug/consent.py +36 -0
- package/src/devcouncil/codeintel/debug/discovery.py +132 -0
- package/src/devcouncil/codeintel/debug/fingerprint.py +85 -0
- package/src/devcouncil/codeintel/debug/protocol.py +259 -0
- package/src/devcouncil/codeintel/debug/python_trace_runner.py +81 -0
- package/src/devcouncil/codeintel/debug/session.py +238 -0
- package/src/devcouncil/codeintel/debug/tracing.py +201 -0
- package/src/devcouncil/codeintel/languages/__init__.py +17 -0
- package/src/devcouncil/codeintel/languages/generic_extractor.py +236 -0
- package/src/devcouncil/codeintel/languages/registry.py +149 -0
- package/src/devcouncil/codeintel/languages/workers.py +245 -0
- package/src/devcouncil/codeintel/query/__init__.py +5 -0
- package/src/devcouncil/codeintel/query/engine.py +289 -0
- package/src/devcouncil/codeintel/resolution/__init__.py +6 -0
- package/src/devcouncil/codeintel/resolution/abstract_state.py +301 -0
- package/src/devcouncil/codeintel/resolution/frameworks/__init__.py +33 -0
- package/src/devcouncil/codeintel/resolution/frameworks/base.py +46 -0
- package/src/devcouncil/codeintel/resolution/frameworks/di.py +56 -0
- package/src/devcouncil/codeintel/resolution/frameworks/events.py +45 -0
- package/src/devcouncil/codeintel/resolution/frameworks/routes.py +88 -0
- package/src/devcouncil/codeintel/resolution/semantic.py +887 -0
- package/src/devcouncil/codeintel/service.py +104 -0
- package/src/devcouncil/codeintel/store/__init__.py +15 -0
- package/src/devcouncil/codeintel/store/sqlite.py +1565 -0
- package/src/devcouncil/codeintel/sync/__init__.py +19 -0
- package/src/devcouncil/codeintel/sync/coordinator.py +430 -0
- package/src/devcouncil/codeintel/sync/incremental.py +484 -0
- package/src/devcouncil/codeintel/sync/lease.py +96 -0
- package/src/devcouncil/codeintel/sync/scope.py +98 -0
- package/src/devcouncil/council/__init__.py +4 -0
- package/src/devcouncil/council/prompts/__init__.py +4 -0
- package/src/devcouncil/domain/checkpoint_refs.py +17 -0
- package/src/devcouncil/domain/evidence.py +1 -0
- package/src/devcouncil/domain/gap.py +10 -0
- package/src/devcouncil/domain/requirement.py +5 -1
- package/src/devcouncil/domain/task.py +43 -2
- package/src/devcouncil/execution/checkpoints.py +25 -31
- package/src/devcouncil/execution/context_builder.py +15 -44
- package/src/devcouncil/execution/fs_watcher.py +64 -0
- package/src/devcouncil/execution/gated_write.py +203 -0
- package/src/devcouncil/execution/handoff.py +2 -1
- package/src/devcouncil/execution/hook_policy.py +19 -5
- package/src/devcouncil/execution/lease_ops.py +177 -0
- package/src/devcouncil/execution/lease_validation.py +71 -0
- package/src/devcouncil/execution/patch.py +3 -0
- package/src/devcouncil/execution/permissions.py +1 -0
- package/src/devcouncil/execution/policy_engine.py +205 -10
- package/src/devcouncil/execution/prompt_builder.py +278 -33
- package/src/devcouncil/execution/run_trace.py +356 -0
- package/src/devcouncil/execution/shell_session.py +46 -5
- package/src/devcouncil/execution/stop_gate.py +746 -0
- package/src/devcouncil/execution/stop_gate_history.py +113 -0
- package/src/devcouncil/execution/stop_gate_state.py +54 -0
- package/src/devcouncil/execution/stop_gate_verify_cache.py +69 -0
- package/src/devcouncil/execution/task_gate_ops.py +590 -0
- package/src/devcouncil/execution/task_runner.py +19 -0
- package/src/devcouncil/executors/advisor_tool.py +315 -0
- package/src/devcouncil/executors/agent_registry.py +125 -17
- package/src/devcouncil/executors/claude_sdk.py +376 -0
- package/src/devcouncil/executors/coding_cli.py +724 -25
- package/src/devcouncil/executors/mini_swe.py +50 -8
- package/src/devcouncil/executors/native/agent.py +224 -19
- package/src/devcouncil/executors/openhands.py +50 -8
- package/src/devcouncil/executors/transient_retry.py +99 -0
- package/src/devcouncil/gating/checks/clean_git.py +5 -2
- package/src/devcouncil/gating/checks/planned_files_check.py +38 -11
- package/src/devcouncil/gating/checks/secret_scan_check.py +2 -2
- package/src/devcouncil/gating/policy.py +46 -2
- package/src/devcouncil/indexing/ast_matcher.py +41 -4
- package/src/devcouncil/indexing/graph/__init__.py +78 -0
- package/src/devcouncil/indexing/graph/api_routes.py +522 -0
- package/src/devcouncil/indexing/graph/build.py +862 -0
- package/src/devcouncil/indexing/graph/cache.py +329 -0
- package/src/devcouncil/indexing/graph/communities.py +28 -0
- package/src/devcouncil/indexing/graph/cypher.py +107 -0
- package/src/devcouncil/indexing/graph/embeddings.py +194 -0
- package/src/devcouncil/indexing/graph/export.py +381 -0
- package/src/devcouncil/indexing/graph/export_links.py +81 -0
- package/src/devcouncil/indexing/graph/extract_python.py +307 -0
- package/src/devcouncil/indexing/graph/extract_ts.py +1205 -0
- package/src/devcouncil/indexing/graph/intel.py +668 -0
- package/src/devcouncil/indexing/graph/liveness.py +992 -0
- package/src/devcouncil/indexing/graph/okf_export.py +65 -0
- package/src/devcouncil/indexing/graph/pdg/__init__.py +67 -0
- package/src/devcouncil/indexing/graph/pdg/build.py +11 -0
- package/src/devcouncil/indexing/graph/pdg/cdg.py +41 -0
- package/src/devcouncil/indexing/graph/pdg/cfg.py +199 -0
- package/src/devcouncil/indexing/graph/pdg/query.py +21 -0
- package/src/devcouncil/indexing/graph/pdg/reaching_def.py +126 -0
- package/src/devcouncil/indexing/graph/pdg/schema.py +253 -0
- package/src/devcouncil/indexing/graph/pdg/taint.py +154 -0
- package/src/devcouncil/indexing/graph/query.py +302 -0
- package/src/devcouncil/indexing/graph/resolve.py +1020 -0
- package/src/devcouncil/indexing/graph/schema.py +103 -0
- package/src/devcouncil/indexing/graph_index.py +20 -29
- package/src/devcouncil/indexing/lsp.py +57 -25
- package/src/devcouncil/indexing/lsp_client.py +577 -0
- package/src/devcouncil/indexing/map_artifacts.py +355 -0
- package/src/devcouncil/indexing/map_refresh.py +141 -0
- package/src/devcouncil/indexing/repo_mapper.py +1509 -138
- package/src/devcouncil/indexing/semantic_index.py +12 -6
- package/src/devcouncil/indexing/subsystem_map.py +163 -0
- package/src/devcouncil/indexing/ts_imports.py +343 -0
- package/src/devcouncil/indexing/viz.py +960 -0
- package/src/devcouncil/indexing/walk.py +52 -0
- package/src/devcouncil/indexing/wiring.py +1776 -0
- package/src/devcouncil/integrations/actions.py +27 -4
- package/src/devcouncil/integrations/check.py +211 -16
- package/src/devcouncil/integrations/claude_assets.py +209 -12
- package/src/devcouncil/integrations/clients/__init__.py +1 -0
- package/src/devcouncil/integrations/clients/aider.py +52 -0
- package/src/devcouncil/integrations/clients/antigravity.py +87 -0
- package/src/devcouncil/integrations/clients/claude.py +339 -0
- package/src/devcouncil/integrations/clients/codex.py +39 -0
- package/src/devcouncil/integrations/clients/common.py +332 -0
- package/src/devcouncil/integrations/clients/cursor.py +164 -0
- package/src/devcouncil/integrations/clients/gemini.py +49 -0
- package/src/devcouncil/integrations/clients/grok.py +105 -0
- package/src/devcouncil/integrations/clients/hooks.py +500 -0
- package/src/devcouncil/integrations/clients/opencode.py +96 -0
- package/src/devcouncil/integrations/clients/warp.py +75 -0
- package/src/devcouncil/integrations/code_review_graph.py +2 -2
- package/src/devcouncil/integrations/github.py +73 -7
- package/src/devcouncil/integrations/integration_cli.py +197 -0
- package/src/devcouncil/integrations/mcp/handlers/__init__.py +1 -0
- package/src/devcouncil/integrations/mcp/handlers/ast_lsp.py +77 -0
- package/src/devcouncil/integrations/mcp/handlers/checkout.py +50 -0
- package/src/devcouncil/integrations/mcp/handlers/cli_gate.py +43 -0
- package/src/devcouncil/integrations/mcp/handlers/codeintel.py +182 -0
- package/src/devcouncil/integrations/mcp/handlers/debug.py +236 -0
- package/src/devcouncil/integrations/mcp/handlers/evidence.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/git.py +99 -0
- package/src/devcouncil/integrations/mcp/handlers/graph.py +36 -0
- package/src/devcouncil/integrations/mcp/handlers/handoff.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/knowledge.py +30 -0
- package/src/devcouncil/integrations/mcp/handlers/lease.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/live.py +108 -0
- package/src/devcouncil/integrations/mcp/handlers/map.py +676 -0
- package/src/devcouncil/integrations/mcp/handlers/next_task.py +35 -0
- package/src/devcouncil/integrations/mcp/handlers/policy.py +80 -0
- package/src/devcouncil/integrations/mcp/handlers/prompts.py +168 -0
- package/src/devcouncil/integrations/mcp/handlers/provenance.py +101 -0
- package/src/devcouncil/integrations/mcp/handlers/read.py +74 -0
- package/src/devcouncil/integrations/mcp/handlers/router_cache.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/run.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/runs.py +84 -0
- package/src/devcouncil/integrations/mcp/handlers/scope.py +56 -0
- package/src/devcouncil/integrations/mcp/handlers/status.py +114 -0
- package/src/devcouncil/integrations/mcp/handlers/task.py +100 -0
- package/src/devcouncil/integrations/mcp/handlers/tool_specs.py +904 -0
- package/src/devcouncil/integrations/mcp/handlers/trace.py +65 -0
- package/src/devcouncil/integrations/mcp/handlers/verify.py +45 -0
- package/src/devcouncil/integrations/mcp/handlers/wiki.py +60 -0
- package/src/devcouncil/integrations/mcp/handlers/write.py +69 -0
- package/src/devcouncil/integrations/mcp/server.py +265 -2391
- package/src/devcouncil/integrations/mcp/util.py +303 -0
- package/src/devcouncil/integrations/setup.py +152 -0
- package/src/devcouncil/knowledge/fetch.py +4 -0
- package/src/devcouncil/knowledge/knowledge_select.py +38 -0
- package/src/devcouncil/knowledge/okf.py +2 -1
- package/src/devcouncil/knowledge/resource_discovery.py +40 -0
- package/src/devcouncil/knowledge/wiki.py +643 -0
- package/src/devcouncil/knowledge/wiki_read.py +87 -0
- package/src/devcouncil/live/cards.py +7 -7
- package/src/devcouncil/live/models.py +4 -1
- package/src/devcouncil/live/reviewer.py +90 -11
- package/src/devcouncil/live/signals.py +4 -2
- package/src/devcouncil/live/tasks.py +12 -3
- package/src/devcouncil/live/transcripts.py +69 -2
- package/src/devcouncil/llm/cache.py +5 -6
- package/src/devcouncil/llm/model_defaults.yaml +10 -10
- package/src/devcouncil/llm/provider.py +647 -73
- package/src/devcouncil/llm/router.py +271 -46
- package/src/devcouncil/llm/semantic_bridge.py +614 -0
- package/src/devcouncil/optimization/gepa_agent.py +6 -4
- package/src/devcouncil/optimization/skillopt.py +9 -5
- package/src/devcouncil/planning/arbiter_service.py +12 -3
- package/src/devcouncil/planning/correction_manifest.py +107 -10
- package/src/devcouncil/planning/plan_difficulty.py +69 -0
- package/src/devcouncil/planning/plan_service.py +5 -2
- package/src/devcouncil/planning/planned_files_reconcile.py +191 -0
- package/src/devcouncil/planning/prompt_enhancer_service.py +6 -5
- package/src/devcouncil/planning/question_conversion.py +56 -0
- package/src/devcouncil/planning/spec_service.py +9 -3
- package/src/devcouncil/repo/ci_scaffold.py +197 -1
- package/src/devcouncil/repo/gitignore.py +1 -2
- package/src/devcouncil/reporting/evidence_export.py +124 -0
- package/src/devcouncil/reporting/evidence_html.py +210 -0
- package/src/devcouncil/reporting/json_report.py +16 -12
- package/src/devcouncil/reporting/markdown_report.py +38 -9
- package/src/devcouncil/reporting/mcp_resources.py +142 -0
- package/src/devcouncil/reporting/report_builder.py +40 -4
- package/src/devcouncil/reporting/task_provenance.py +42 -0
- package/src/devcouncil/reporting/verdict.py +75 -0
- package/src/devcouncil/skills/library/README.md +1 -0
- package/src/devcouncil/skills/library/devcouncil-hero-loop.md +109 -0
- package/src/devcouncil/skills/library/devcouncil-verification.md +109 -0
- package/src/devcouncil/skills/library/devcouncil.md +93 -0
- package/src/devcouncil/skills/registry.py +43 -12
- package/src/devcouncil/storage/db.py +57 -11
- package/src/devcouncil/storage/models.py +6 -0
- package/src/devcouncil/storage/native.py +5 -3
- package/src/devcouncil/storage/repositories.py +50 -18
- package/src/devcouncil/telemetry/context.py +28 -0
- package/src/devcouncil/telemetry/cost.py +4 -5
- package/src/devcouncil/telemetry/logging_setup.py +78 -11
- package/src/devcouncil/telemetry/model_pricing.yaml +7 -0
- package/src/devcouncil/telemetry/stages.py +27 -2
- package/src/devcouncil/telemetry/tracker.py +50 -13
- package/src/devcouncil/ui/dashboard.py +120 -8
- package/src/devcouncil/utils/fsio.py +58 -0
- package/src/devcouncil/utils/git_snapshot.py +112 -0
- package/src/devcouncil/utils/json_persist.py +53 -0
- package/src/devcouncil/utils/proc.py +89 -0
- package/src/devcouncil/verification/acceptance_compiler.py +36 -13
- package/src/devcouncil/verification/ad_hoc_check.py +95 -3
- package/src/devcouncil/verification/checks/__init__.py +41 -0
- package/src/devcouncil/verification/checks/acceptance.py +39 -0
- package/src/devcouncil/verification/checks/acceptance_corpus.py +194 -0
- package/src/devcouncil/verification/checks/acceptance_evidence.py +239 -0
- package/src/devcouncil/verification/checks/command_evidence.py +148 -0
- package/src/devcouncil/verification/checks/compiled_acceptance.py +179 -0
- package/src/devcouncil/verification/checks/corpus_stale.py +124 -0
- package/src/devcouncil/verification/checks/corpus_verification.py +9 -0
- package/src/devcouncil/verification/checks/dead_symbols.py +360 -0
- package/src/devcouncil/verification/checks/diff_coverage_gate.py +101 -0
- package/src/devcouncil/verification/checks/doc_code_ref.py +79 -0
- package/src/devcouncil/verification/checks/liveness_ratchet.py +336 -0
- package/src/devcouncil/verification/checks/orphan_diff.py +104 -0
- package/src/devcouncil/verification/checks/planned_files.py +98 -0
- package/src/devcouncil/verification/checks/semantic_diff.py +241 -0
- package/src/devcouncil/verification/checks/stale_map.py +80 -0
- package/src/devcouncil/verification/checks/stub_scan.py +71 -0
- package/src/devcouncil/verification/checks/subsystem_boundary.py +103 -0
- package/src/devcouncil/verification/checks/wiring.py +216 -0
- package/src/devcouncil/verification/claims/__init__.py +23 -0
- package/src/devcouncil/verification/claims/checks.py +395 -0
- package/src/devcouncil/verification/claims/mapper.py +168 -0
- package/src/devcouncil/verification/claims/models.py +39 -0
- package/src/devcouncil/verification/claims/transcript.py +92 -0
- package/src/devcouncil/verification/claims/verdict.py +88 -0
- package/src/devcouncil/verification/command_evidence.py +170 -0
- package/src/devcouncil/verification/command_malformation.py +147 -0
- package/src/devcouncil/verification/command_runner.py +164 -0
- package/src/devcouncil/verification/coverage_measurement.py +292 -0
- package/src/devcouncil/verification/diff_coverage.py +151 -0
- package/src/devcouncil/verification/difficulty.py +296 -0
- package/src/devcouncil/verification/effort_heuristics.py +178 -0
- package/src/devcouncil/verification/gap_ids.py +63 -0
- package/src/devcouncil/verification/gate_cache.py +194 -0
- package/src/devcouncil/verification/gate_selector.py +344 -0
- package/src/devcouncil/verification/git_diff_fallback.py +272 -0
- package/src/devcouncil/verification/implementation_reviewer.py +13 -0
- package/src/devcouncil/verification/incremental_check.py +241 -0
- package/src/devcouncil/verification/next_actions.py +60 -1
- package/src/devcouncil/verification/rigor_analytics.py +130 -0
- package/src/devcouncil/verification/sandbox.py +38 -11
- package/src/devcouncil/verification/stub_detector.py +369 -0
- package/src/devcouncil/verification/test_resolver.py +67 -1
- package/src/devcouncil/verification/verifier.py +137 -1666
- package/src/devcouncil/verification/verify_orchestration.py +610 -0
- package/src/devcouncil/verification/verify_setup.py +176 -0
- package/src/devcouncil/verification/wiki_refresh.py +208 -0
- package/src/semantic_layer/__init__.py +58 -0
- package/src/semantic_layer/benchmark.py +75 -0
- package/src/semantic_layer/cache.py +290 -0
- package/src/semantic_layer/compressor.py +137 -0
- package/src/semantic_layer/config.py +75 -0
- package/src/semantic_layer/embeddings.py +69 -0
- package/src/semantic_layer/llm_backends.py +99 -0
- package/src/semantic_layer/pipeline.py +111 -0
- package/src/semantic_layer/router.py +128 -0
- package/src/semantic_layer/tuner.py +72 -0
- package/uv.lock +973 -9
- package/src/devcouncil/artifacts/migrations.py +0 -20
- package/src/devcouncil/artifacts/schemas.py +0 -23
- package/src/devcouncil/artifacts/serializer.py +0 -21
- package/src/devcouncil/integrations/gitnexus.py +0 -70
- package/src/devcouncil/integrations/graphify.py +0 -34
|
@@ -1,51 +1,47 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import hashlib
|
|
3
|
-
import os
|
|
4
|
-
import shutil
|
|
5
|
-
import subprocess
|
|
6
|
-
import sys
|
|
7
1
|
import logging
|
|
8
|
-
import
|
|
9
|
-
import fnmatch
|
|
10
|
-
import json
|
|
11
|
-
import re
|
|
12
|
-
import shlex
|
|
13
|
-
from dataclasses import dataclass, asdict
|
|
2
|
+
from dataclasses import dataclass, asdict, field
|
|
14
3
|
from pathlib import Path
|
|
15
|
-
from typing import List, Dict, Any,
|
|
4
|
+
from typing import List, Dict, Any, Optional, Tuple
|
|
16
5
|
|
|
17
6
|
from devcouncil.app.config import load_config
|
|
18
7
|
|
|
19
8
|
from devcouncil.domain.task import Task
|
|
20
9
|
from devcouncil.domain.requirement import Requirement
|
|
21
10
|
from devcouncil.domain.gap import Gap
|
|
22
|
-
from devcouncil.domain.evidence import
|
|
11
|
+
from devcouncil.domain.evidence import CommandResult
|
|
23
12
|
from devcouncil.verification import diff_coverage as dc
|
|
13
|
+
from devcouncil.verification.checks.orphan_diff import classify_change_paths
|
|
14
|
+
from devcouncil.verification.checks.semantic_diff import (
|
|
15
|
+
detect_semantic_diff_gaps,
|
|
16
|
+
import_top_level,
|
|
17
|
+
is_new_third_party_import,
|
|
18
|
+
load_project_dependencies,
|
|
19
|
+
task_intent_text,
|
|
20
|
+
)
|
|
21
|
+
from devcouncil.verification.verify_setup import (
|
|
22
|
+
cleanup_verify_futures,
|
|
23
|
+
prime_verify_memos,
|
|
24
|
+
resolve_verify_context,
|
|
25
|
+
start_verify_futures,
|
|
26
|
+
)
|
|
27
|
+
from devcouncil.verification.verify_orchestration import run_verify_orchestration
|
|
28
|
+
from devcouncil.verification.command_evidence import (
|
|
29
|
+
command_has_acceptance_evidence,
|
|
30
|
+
command_is_trivial_evidence,
|
|
31
|
+
)
|
|
24
32
|
from devcouncil.gating.checks.secret_scan_check import SecretScanner
|
|
25
33
|
from devcouncil.verification.implementation_reviewer import ImplementationReviewer
|
|
26
34
|
from devcouncil.verification.acceptance_compiler import AcceptanceTestCompiler
|
|
27
35
|
from devcouncil.llm.router import ModelRouter
|
|
28
|
-
from devcouncil.utils.
|
|
29
|
-
from devcouncil.
|
|
36
|
+
from devcouncil.utils.json_persist import read_json
|
|
37
|
+
from devcouncil.utils.subprocess_env import clean_subprocess_env
|
|
38
|
+
from devcouncil.verification import command_malformation as cmd_malf
|
|
39
|
+
from devcouncil.verification import command_runner as cmd_runner
|
|
40
|
+
from devcouncil.verification.coverage_measurement import DiffCoverageMeasurer
|
|
41
|
+
from devcouncil.verification.git_diff_fallback import GitDiffFallback
|
|
30
42
|
|
|
31
43
|
logger = logging.getLogger(__name__)
|
|
32
44
|
|
|
33
|
-
IGNORED_CHANGE_PATTERNS = (
|
|
34
|
-
"__pycache__/*",
|
|
35
|
-
"*/__pycache__/*",
|
|
36
|
-
"*.pyc",
|
|
37
|
-
"*.pyo",
|
|
38
|
-
".pytest_cache/*",
|
|
39
|
-
".mypy_cache/*",
|
|
40
|
-
".ruff_cache/*",
|
|
41
|
-
".devcouncil/*",
|
|
42
|
-
# DevCouncil manages the root .gitignore itself (ensure_gitignore runs on
|
|
43
|
-
# init and before every task), so its drift is not task work.
|
|
44
|
-
".gitignore",
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
MAX_UNTRACKED_DIFF_BYTES = 256_000
|
|
48
|
-
|
|
49
45
|
|
|
50
46
|
@dataclass
|
|
51
47
|
class VerificationOutcome:
|
|
@@ -64,6 +60,14 @@ class VerificationOutcome:
|
|
|
64
60
|
diff_empty: bool = True
|
|
65
61
|
coverage_measured: bool = False
|
|
66
62
|
coverage_skipped_reason: Optional[str] = None
|
|
63
|
+
# Rigor metadata: the task's estimated (or manually set) difficulty and which
|
|
64
|
+
# anti-laziness escalations actually took effect on this run, so "passed" can
|
|
65
|
+
# be distinguished from "passed under strict gates".
|
|
66
|
+
difficulty: Optional[str] = None
|
|
67
|
+
rigor_applied: List[str] = field(default_factory=list)
|
|
68
|
+
wiki_refresh: Optional[Dict[str, Any]] = None
|
|
69
|
+
# Liveness ratchet baseline status for this verify run: "ok" | "missing".
|
|
70
|
+
liveness_baseline: Optional[str] = None
|
|
67
71
|
|
|
68
72
|
def as_dict(self) -> Dict[str, Any]:
|
|
69
73
|
return asdict(self)
|
|
@@ -72,7 +76,6 @@ class VerificationOutcome:
|
|
|
72
76
|
class Verifier:
|
|
73
77
|
def __init__(self, project_root: Path, router: Optional[ModelRouter] = None):
|
|
74
78
|
self.project_root = project_root
|
|
75
|
-
self._gap_counter = 0
|
|
76
79
|
self.secret_scanner = SecretScanner()
|
|
77
80
|
self.reviewer = ImplementationReviewer(router) if router else None
|
|
78
81
|
self.acceptance_compiler = AcceptanceTestCompiler(router) if router else None
|
|
@@ -90,43 +93,29 @@ class Verifier:
|
|
|
90
93
|
# Per-verify_task memos (primed at verify_task entry, cleared before it returns)
|
|
91
94
|
# so the hot path does not re-run `git ls-files` or re-load config repeatedly.
|
|
92
95
|
# None outside a verify_task call, so all other callers behave exactly as before.
|
|
93
|
-
self.
|
|
96
|
+
self._git_fallback = GitDiffFallback(project_root)
|
|
94
97
|
self._command_timeout_cache: Optional[int] = None
|
|
95
98
|
# Project dependency names (lower-cased), loaded once per verify_task and cleared
|
|
96
99
|
# in its finally so a reused Verifier re-reads them for a later task.
|
|
97
100
|
self._project_deps_cache: Optional[set] = None
|
|
101
|
+
self._coverage_measurer: Optional[DiffCoverageMeasurer] = None
|
|
102
|
+
|
|
103
|
+
def _get_coverage_measurer(self) -> DiffCoverageMeasurer:
|
|
104
|
+
if self._coverage_measurer is None:
|
|
105
|
+
self._coverage_measurer = DiffCoverageMeasurer.from_verifier(self)
|
|
106
|
+
return self._coverage_measurer
|
|
98
107
|
|
|
99
108
|
def _next_gap_id(self, task_id: str, suffix: str) -> str:
|
|
100
|
-
"""Generate
|
|
101
|
-
|
|
102
|
-
|
|
109
|
+
"""Generate stable gap IDs from task + suffix identity (deterministic across runs)."""
|
|
110
|
+
from devcouncil.verification.gap_ids import stable_gap_id
|
|
111
|
+
|
|
112
|
+
return stable_gap_id(task_id, suffix)
|
|
103
113
|
|
|
104
114
|
def get_diff(self) -> str:
|
|
105
|
-
|
|
106
|
-
if not self._has_head():
|
|
107
|
-
return self._get_initial_repo_diff()
|
|
108
|
-
tracked_diff = subprocess.check_output(
|
|
109
|
-
["git", "diff", "HEAD"], cwd=self.project_root
|
|
110
|
-
).decode("utf-8", errors="replace")
|
|
111
|
-
untracked_diff = self._get_untracked_files_diff()
|
|
112
|
-
return "\n".join(part for part in [tracked_diff, untracked_diff] if part)
|
|
113
|
-
except Exception as e:
|
|
114
|
-
logger.warning("Failed to get git diff: %s", e)
|
|
115
|
-
return ""
|
|
115
|
+
return self._git_fallback.get_diff()
|
|
116
116
|
|
|
117
117
|
def get_changed_files(self) -> List[str]:
|
|
118
|
-
|
|
119
|
-
if not self._has_head():
|
|
120
|
-
return self._get_status_files()
|
|
121
|
-
output = subprocess.check_output(
|
|
122
|
-
["git", "diff", "HEAD", "--name-only"], cwd=self.project_root
|
|
123
|
-
).decode("utf-8", errors="replace").splitlines()
|
|
124
|
-
files = set(output)
|
|
125
|
-
files.update(self._get_untracked_files())
|
|
126
|
-
return self._filter_change_paths(sorted(files))
|
|
127
|
-
except Exception as e:
|
|
128
|
-
logger.warning("Failed to get changed files: %s", e)
|
|
129
|
-
return []
|
|
118
|
+
return self._git_fallback.get_changed_files()
|
|
130
119
|
|
|
131
120
|
def get_task_changed_files(self, task_id: str) -> List[str]:
|
|
132
121
|
changed = set(self.get_changed_files())
|
|
@@ -135,34 +124,7 @@ class Verifier:
|
|
|
135
124
|
return sorted(changed)
|
|
136
125
|
|
|
137
126
|
def _committed_task_diff(self, task_id: str) -> str:
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
When ``dev go`` commits a task's work (e.g. between self-repair attempts, or
|
|
141
|
-
before the reconciliation pass), the working-tree diff (``git diff HEAD``) is
|
|
142
|
-
empty even though the task is fully implemented. This recovers that committed
|
|
143
|
-
change so acceptance compilation/review still have something to reason about
|
|
144
|
-
instead of seeing an empty diff and skipping — which would mark every criterion
|
|
145
|
-
unproven and wrongly block correct, committed code.
|
|
146
|
-
"""
|
|
147
|
-
# Literal of CheckpointService.REF_BEFORE (kept inline to avoid a circular
|
|
148
|
-
# import: checkpoints.py imports Verifier).
|
|
149
|
-
before_ref = f"refs/devcouncil/tasks/{task_id}/before"
|
|
150
|
-
try:
|
|
151
|
-
has_ref = subprocess.run(
|
|
152
|
-
["git", "rev-parse", "--verify", before_ref],
|
|
153
|
-
cwd=self.project_root,
|
|
154
|
-
stdout=subprocess.DEVNULL,
|
|
155
|
-
stderr=subprocess.DEVNULL,
|
|
156
|
-
).returncode == 0
|
|
157
|
-
if has_ref:
|
|
158
|
-
return subprocess.check_output(
|
|
159
|
-
["git", "diff", before_ref],
|
|
160
|
-
cwd=self.project_root,
|
|
161
|
-
stderr=subprocess.DEVNULL,
|
|
162
|
-
).decode("utf-8", errors="replace")
|
|
163
|
-
except Exception:
|
|
164
|
-
pass
|
|
165
|
-
return ""
|
|
127
|
+
return self._git_fallback.committed_task_diff(task_id)
|
|
166
128
|
|
|
167
129
|
def _task_produced_changes(self, task_id: str) -> bool:
|
|
168
130
|
"""True when the task has a footprint beyond the current working-tree diff.
|
|
@@ -182,134 +144,11 @@ class Verifier:
|
|
|
182
144
|
except Exception:
|
|
183
145
|
return False
|
|
184
146
|
|
|
185
|
-
def _has_head(self) -> bool:
|
|
186
|
-
return subprocess.run(
|
|
187
|
-
["git", "rev-parse", "--verify", "HEAD"],
|
|
188
|
-
cwd=self.project_root,
|
|
189
|
-
stdout=subprocess.DEVNULL,
|
|
190
|
-
stderr=subprocess.DEVNULL,
|
|
191
|
-
).returncode == 0
|
|
192
|
-
|
|
193
|
-
def _get_initial_repo_diff(self) -> str:
|
|
194
|
-
parts: List[str] = []
|
|
195
|
-
for cmd in (["git", "diff", "--cached"], ["git", "diff"]):
|
|
196
|
-
result = subprocess.run(
|
|
197
|
-
cmd,
|
|
198
|
-
cwd=self.project_root,
|
|
199
|
-
capture_output=True,
|
|
200
|
-
text=True,
|
|
201
|
-
encoding="utf-8",
|
|
202
|
-
errors="replace",
|
|
203
|
-
)
|
|
204
|
-
if result.returncode == 0 and result.stdout:
|
|
205
|
-
parts.append(result.stdout)
|
|
206
|
-
untracked_diff = self._get_untracked_files_diff()
|
|
207
|
-
if untracked_diff:
|
|
208
|
-
parts.append(untracked_diff)
|
|
209
|
-
return "\n".join(parts)
|
|
210
|
-
|
|
211
|
-
def _get_status_files(self) -> List[str]:
|
|
212
|
-
files: set[str] = set()
|
|
213
|
-
commands = (
|
|
214
|
-
["git", "diff", "--cached", "--name-only"],
|
|
215
|
-
["git", "diff", "--name-only"],
|
|
216
|
-
["git", "ls-files", "--others", "--exclude-standard"],
|
|
217
|
-
)
|
|
218
|
-
for cmd in commands:
|
|
219
|
-
try:
|
|
220
|
-
output = subprocess.check_output(
|
|
221
|
-
cmd,
|
|
222
|
-
cwd=self.project_root,
|
|
223
|
-
stderr=subprocess.DEVNULL,
|
|
224
|
-
).decode("utf-8", errors="replace").splitlines()
|
|
225
|
-
files.update(path.replace("\\", "/") for path in output if path.strip())
|
|
226
|
-
except subprocess.CalledProcessError:
|
|
227
|
-
continue
|
|
228
|
-
if not files:
|
|
229
|
-
files.update(self._walk_project_files())
|
|
230
|
-
return self._filter_change_paths(sorted(files))
|
|
231
|
-
|
|
232
147
|
def _get_untracked_files(self) -> List[str]:
|
|
233
|
-
|
|
234
|
-
# _get_untracked_files_diff, and _classify_change_paths. verify_task primes this
|
|
235
|
-
# once; it is None for every other caller, so they recompute fresh as before.
|
|
236
|
-
if self._untracked_cache is not None:
|
|
237
|
-
return self._untracked_cache
|
|
238
|
-
try:
|
|
239
|
-
output = subprocess.check_output(
|
|
240
|
-
["git", "ls-files", "--others", "--exclude-standard"],
|
|
241
|
-
cwd=self.project_root,
|
|
242
|
-
stderr=subprocess.DEVNULL,
|
|
243
|
-
).decode("utf-8", errors="replace").splitlines()
|
|
244
|
-
return self._filter_change_paths(output)
|
|
245
|
-
except Exception as e:
|
|
246
|
-
logger.debug("Failed to list untracked files: %s", e)
|
|
247
|
-
return []
|
|
248
|
-
|
|
249
|
-
def _get_untracked_files_diff(self) -> str:
|
|
250
|
-
parts: List[str] = []
|
|
251
|
-
for rel_path in self._get_untracked_files():
|
|
252
|
-
full_path = self.project_root / rel_path
|
|
253
|
-
if not full_path.is_file():
|
|
254
|
-
continue
|
|
255
|
-
parts.append(self._format_new_file_diff(rel_path, full_path))
|
|
256
|
-
return "\n".join(part for part in parts if part)
|
|
257
|
-
|
|
258
|
-
def _format_new_file_diff(self, rel_path: str, full_path: Path) -> str:
|
|
259
|
-
try:
|
|
260
|
-
raw = full_path.read_bytes()
|
|
261
|
-
except Exception as e:
|
|
262
|
-
logger.debug("Failed to read untracked file %s: %s", rel_path, e)
|
|
263
|
-
return ""
|
|
264
|
-
|
|
265
|
-
header = [
|
|
266
|
-
f"diff --git a/{rel_path} b/{rel_path}",
|
|
267
|
-
"new file mode 100644",
|
|
268
|
-
"--- /dev/null",
|
|
269
|
-
f"+++ b/{rel_path}",
|
|
270
|
-
]
|
|
271
|
-
if b"\0" in raw[:8192]:
|
|
272
|
-
return "\n".join([*header, f"Binary files /dev/null and b/{rel_path} differ"])
|
|
273
|
-
|
|
274
|
-
truncated = len(raw) > MAX_UNTRACKED_DIFF_BYTES
|
|
275
|
-
if truncated:
|
|
276
|
-
raw = raw[:MAX_UNTRACKED_DIFF_BYTES]
|
|
277
|
-
text = raw.decode("utf-8", errors="replace")
|
|
278
|
-
lines = text.splitlines()
|
|
279
|
-
if text.endswith(("\n", "\r")):
|
|
280
|
-
line_count = len(lines)
|
|
281
|
-
else:
|
|
282
|
-
line_count = max(len(lines), 1 if text else 0)
|
|
283
|
-
|
|
284
|
-
diff_lines = [*header, f"@@ -0,0 +1,{line_count} @@"]
|
|
285
|
-
if not text:
|
|
286
|
-
return "\n".join(header) + "\n"
|
|
287
|
-
|
|
288
|
-
diff_lines.extend(f"+{line}" for line in lines)
|
|
289
|
-
if truncated:
|
|
290
|
-
diff_lines.append("+[devcouncil: untracked file diff truncated]")
|
|
291
|
-
return "\n".join(diff_lines)
|
|
292
|
-
|
|
293
|
-
def _walk_project_files(self) -> List[str]:
|
|
294
|
-
files: List[str] = []
|
|
295
|
-
for path in self.project_root.rglob("*"):
|
|
296
|
-
if not path.is_file():
|
|
297
|
-
continue
|
|
298
|
-
rel = path.relative_to(self.project_root).as_posix()
|
|
299
|
-
if rel.startswith(".git/"):
|
|
300
|
-
continue
|
|
301
|
-
files.append(rel)
|
|
302
|
-
return files
|
|
148
|
+
return self._git_fallback.get_untracked_files()
|
|
303
149
|
|
|
304
150
|
def _filter_change_paths(self, paths: List[str]) -> List[str]:
|
|
305
|
-
return
|
|
306
|
-
path
|
|
307
|
-
for path in (p.strip().replace("\\", "/") for p in paths)
|
|
308
|
-
if path and not self._is_ignored_change(path)
|
|
309
|
-
]
|
|
310
|
-
|
|
311
|
-
def _is_ignored_change(self, path: str) -> bool:
|
|
312
|
-
return any(fnmatch.fnmatch(path, pattern) for pattern in IGNORED_CHANGE_PATTERNS)
|
|
151
|
+
return self._git_fallback.filter_change_paths(paths)
|
|
313
152
|
|
|
314
153
|
def _load_baseline_files(self) -> set[str]:
|
|
315
154
|
return self._load_snapshot_files(self.project_root / ".devcouncil" / "baseline.json")
|
|
@@ -323,7 +162,7 @@ class Verifier:
|
|
|
323
162
|
if not path.exists():
|
|
324
163
|
return set()
|
|
325
164
|
try:
|
|
326
|
-
data =
|
|
165
|
+
data = read_json(path)
|
|
327
166
|
return {
|
|
328
167
|
item.replace("\\", "/")
|
|
329
168
|
for item in data.get("changed_files", [])
|
|
@@ -333,6 +172,22 @@ class Verifier:
|
|
|
333
172
|
logger.warning("Failed to load verification snapshot %s: %s", path, e)
|
|
334
173
|
return set()
|
|
335
174
|
|
|
175
|
+
def _load_repo_map(self) -> Optional[dict]:
|
|
176
|
+
"""Parse ``.devcouncil/repo_map.json`` (None if absent/unreadable).
|
|
177
|
+
|
|
178
|
+
Used by structural gates (subsystem-boundary drift) and the wiki post-step.
|
|
179
|
+
Best-effort: any failure degrades to ``None`` so a missing/corrupt map never
|
|
180
|
+
breaks verification."""
|
|
181
|
+
map_path = self.project_root / ".devcouncil" / "repo_map.json"
|
|
182
|
+
if not map_path.exists():
|
|
183
|
+
return None
|
|
184
|
+
try:
|
|
185
|
+
data = read_json(map_path)
|
|
186
|
+
return data if isinstance(data, dict) else None
|
|
187
|
+
except Exception as e:
|
|
188
|
+
logger.debug("Failed to load repo map for verification: %s", e)
|
|
189
|
+
return None
|
|
190
|
+
|
|
336
191
|
def _load_commands(self) -> Dict[str, List[str]]:
|
|
337
192
|
try:
|
|
338
193
|
config = load_config(self.project_root)
|
|
@@ -346,104 +201,16 @@ class Verifier:
|
|
|
346
201
|
return {}
|
|
347
202
|
|
|
348
203
|
def _save_log(self, label: str, command: str, stream: str, content: str) -> str:
|
|
349
|
-
|
|
350
|
-
log_dir = self.project_root / ".devcouncil" / "logs"
|
|
351
|
-
log_dir.mkdir(parents=True, exist_ok=True)
|
|
352
|
-
cmd_hash = hashlib.sha256(command.encode()).hexdigest()[:8]
|
|
353
|
-
filename = f"{label}-{cmd_hash}-{stream}.log"
|
|
354
|
-
log_path = log_dir / filename
|
|
355
|
-
log_path.write_text(redact_string(content), encoding="utf-8")
|
|
356
|
-
return str(log_path)
|
|
204
|
+
return cmd_runner.save_command_log(self.project_root, label, command, stream, content)
|
|
357
205
|
|
|
358
206
|
def _verification_env(self) -> Dict[str, str]:
|
|
359
|
-
|
|
360
|
-
own virtualenv into the target repository.
|
|
361
|
-
|
|
362
|
-
When DevCouncil is installed/run from a venv (e.g. ``uv tool install`` or
|
|
363
|
-
a project ``.venv``), a bare ``python``/``pytest`` in a task's evidence
|
|
364
|
-
command would otherwise resolve to DevCouncil's interpreter, which lacks
|
|
365
|
-
the target project's dependencies — producing false ``No module named
|
|
366
|
-
pytest`` style failures. Strip DevCouncil's venv from ``PATH`` and unset
|
|
367
|
-
the virtualenv markers so commands resolve the project/system interpreter,
|
|
368
|
-
exactly as they would in a plain terminal at the repo root.
|
|
369
|
-
"""
|
|
370
|
-
env = dict(os.environ)
|
|
371
|
-
venv_prefix = Path(sys.prefix).resolve()
|
|
372
|
-
base_prefix = Path(getattr(sys, "base_prefix", sys.prefix)).resolve()
|
|
373
|
-
if venv_prefix == base_prefix:
|
|
374
|
-
return env # Not running inside a venv; nothing to strip.
|
|
375
|
-
|
|
376
|
-
venv_dirs = {
|
|
377
|
-
str(venv_prefix).lower(),
|
|
378
|
-
str((venv_prefix / "Scripts").resolve()).lower(),
|
|
379
|
-
str((venv_prefix / "bin").resolve()).lower(),
|
|
380
|
-
}
|
|
381
|
-
path = env.get("PATH", "")
|
|
382
|
-
kept = []
|
|
383
|
-
for entry in path.split(os.pathsep):
|
|
384
|
-
if not entry:
|
|
385
|
-
continue
|
|
386
|
-
try:
|
|
387
|
-
normalized = str(Path(entry).resolve()).lower()
|
|
388
|
-
except Exception:
|
|
389
|
-
normalized = entry.lower()
|
|
390
|
-
if normalized in venv_dirs:
|
|
391
|
-
continue
|
|
392
|
-
kept.append(entry)
|
|
393
|
-
env["PATH"] = os.pathsep.join(kept)
|
|
394
|
-
|
|
395
|
-
# Drop the virtualenv-activation markers that would pin a freshly-resolved
|
|
396
|
-
# child ``python`` back to DevCouncil's interpreter. VIRTUAL_ENV points at
|
|
397
|
-
# the venv (sys.prefix); PYTHONHOME — set by uv-managed interpreters — points
|
|
398
|
-
# at the base interpreter (sys.base_prefix) and forcibly overrides the stdlib
|
|
399
|
-
# / site-packages location of ANY python the child invokes, which is what
|
|
400
|
-
# makes ``python -m pytest`` fail with "No module named pytest" even when the
|
|
401
|
-
# project's interpreter has pytest installed.
|
|
402
|
-
own_prefixes = {str(venv_prefix), str(base_prefix)}
|
|
403
|
-
for marker in ("VIRTUAL_ENV", "PYTHONHOME"):
|
|
404
|
-
value = env.get(marker)
|
|
405
|
-
if not value:
|
|
406
|
-
continue
|
|
407
|
-
try:
|
|
408
|
-
resolved = str(Path(value).resolve())
|
|
409
|
-
except Exception:
|
|
410
|
-
resolved = value
|
|
411
|
-
if resolved in own_prefixes:
|
|
412
|
-
env.pop(marker, None)
|
|
413
|
-
# uv stashes the same path here and re-applies it to child pythons.
|
|
414
|
-
env.pop("UV_INTERNAL__PYTHONHOME", None)
|
|
415
|
-
return env
|
|
207
|
+
return clean_subprocess_env()
|
|
416
208
|
|
|
417
209
|
@staticmethod
|
|
418
210
|
def _summarize_stream(content: str, budget: int = 360) -> str:
|
|
419
|
-
|
|
420
|
-
error survives downstream truncation.
|
|
421
|
-
|
|
422
|
-
Plain ``content[-500:]`` kept the tail but the combined summary is later clipped
|
|
423
|
-
to its first 500 chars at the gap-evidence sites, which dropped the exception
|
|
424
|
-
line entirely. We hoist the salient error line (the last non-indented line, where
|
|
425
|
-
Python prints the exception) to the front, then append bounded context."""
|
|
426
|
-
if not content or not content.strip():
|
|
427
|
-
return "(empty)"
|
|
428
|
-
lines = [ln.rstrip() for ln in content.splitlines() if ln.strip()]
|
|
429
|
-
markers = ("error", "exception", "assert", "traceback", "failed", "not found", "no module named")
|
|
430
|
-
salient = ""
|
|
431
|
-
for ln in reversed(lines):
|
|
432
|
-
low = ln.lower()
|
|
433
|
-
if any(m in low for m in markers):
|
|
434
|
-
salient = ln.strip()
|
|
435
|
-
break
|
|
436
|
-
if not salient:
|
|
437
|
-
salient = lines[-1].strip()
|
|
438
|
-
salient = salient[:240] # cap a single huge (e.g. minified) line
|
|
439
|
-
tail = content.strip()[-budget:]
|
|
440
|
-
summary = f"{salient} | {tail}" if salient not in tail[: len(salient) + 5] else tail
|
|
441
|
-
return summary[: budget + len(salient) + 8]
|
|
211
|
+
return cmd_runner.summarize_stream(content, budget)
|
|
442
212
|
|
|
443
213
|
def _run_command(self, command: str, task_id: str = "verify") -> CommandResult:
|
|
444
|
-
# Per-verify_task memo: avoid re-loading config for the timeout on every command
|
|
445
|
-
# in the expected_tests / allowed_commands / compiled-check loops. Falls back to
|
|
446
|
-
# loading config when called outside verify_task (cache is None).
|
|
447
214
|
if self._command_timeout_cache is not None:
|
|
448
215
|
timeout = self._command_timeout_cache
|
|
449
216
|
else:
|
|
@@ -452,100 +219,18 @@ class Verifier:
|
|
|
452
219
|
timeout = config.execution.command_timeout
|
|
453
220
|
except Exception:
|
|
454
221
|
timeout = 300
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
# DevCouncil's bundled interpreter (in .venv\Scripts) regardless of PATH.
|
|
462
|
-
# Resolving here pins the command to the project/system interpreter.
|
|
463
|
-
if argv:
|
|
464
|
-
resolved = shutil.which(argv[0], path=env.get("PATH"))
|
|
465
|
-
if resolved:
|
|
466
|
-
argv = [resolved, *argv[1:]]
|
|
467
|
-
|
|
468
|
-
try:
|
|
469
|
-
result = subprocess.run(
|
|
470
|
-
argv,
|
|
471
|
-
shell=False,
|
|
472
|
-
capture_output=True,
|
|
473
|
-
text=True,
|
|
474
|
-
encoding="utf-8",
|
|
475
|
-
errors="replace",
|
|
476
|
-
cwd=self.project_root,
|
|
477
|
-
timeout=timeout,
|
|
478
|
-
env=env,
|
|
479
|
-
)
|
|
480
|
-
stdout = result.stdout or ""
|
|
481
|
-
stderr = result.stderr or ""
|
|
482
|
-
stdout_path = self._save_log(task_id, command, "stdout", stdout)
|
|
483
|
-
stderr_path = self._save_log(task_id, command, "stderr", stderr)
|
|
484
|
-
stdout_summary = redact_string(self._summarize_stream(stdout))
|
|
485
|
-
stderr_summary = redact_string(self._summarize_stream(stderr))
|
|
486
|
-
return CommandResult(
|
|
487
|
-
command=command,
|
|
488
|
-
exit_code=result.returncode,
|
|
489
|
-
stdout_path=stdout_path,
|
|
490
|
-
stderr_path=stderr_path,
|
|
491
|
-
# stderr first: downstream evidence clips summary[:500], so the error
|
|
492
|
-
# line must land in the first 500 chars to stay diagnosable.
|
|
493
|
-
summary=(
|
|
494
|
-
f"Exit code {result.returncode}. "
|
|
495
|
-
f"stderr: {stderr_summary}. "
|
|
496
|
-
f"stdout: {stdout_summary}"
|
|
497
|
-
),
|
|
498
|
-
)
|
|
499
|
-
except Exception as e:
|
|
500
|
-
return CommandResult(
|
|
501
|
-
command=command,
|
|
502
|
-
exit_code=-1,
|
|
503
|
-
stdout_path="",
|
|
504
|
-
stderr_path="",
|
|
505
|
-
summary=f"Failed to run command: {e}",
|
|
506
|
-
)
|
|
222
|
+
return cmd_runner.run_verification_command(
|
|
223
|
+
self.project_root,
|
|
224
|
+
command,
|
|
225
|
+
task_id=task_id,
|
|
226
|
+
timeout=timeout,
|
|
227
|
+
)
|
|
507
228
|
|
|
508
229
|
def _split_command(self, command: str) -> List[str]:
|
|
509
|
-
|
|
510
|
-
# posix=False, `python -c "assert x"` keeps the surrounding quotes, so the
|
|
511
|
-
# interpreter receives the literal string `"assert x"` and treats it as a
|
|
512
|
-
# no-op string expression that exits 0 — every quoted-argument evidence
|
|
513
|
-
# command would then silently "pass" without running, producing false
|
|
514
|
-
# verification. posix=True strips the quotes correctly; planner-generated
|
|
515
|
-
# commands use forward-slash paths, which the interpreter accepts on Windows.
|
|
516
|
-
return shlex.split(command, posix=True)
|
|
517
|
-
|
|
518
|
-
def _check_dependency_changes(self, changed_files: List[str]) -> List[str]:
|
|
519
|
-
dep_files = {
|
|
520
|
-
"package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
|
|
521
|
-
"requirements.txt", "pyproject.toml", "uv.lock", "Pipfile.lock",
|
|
522
|
-
"go.mod", "go.sum", "Cargo.toml", "Cargo.lock",
|
|
523
|
-
}
|
|
524
|
-
return [f for f in changed_files if Path(f).name in dep_files]
|
|
230
|
+
return cmd_runner.split_command(command)
|
|
525
231
|
|
|
526
232
|
def _classify_change_paths(self, changed_files: List[str]) -> Tuple[List[str], List[str]]:
|
|
527
|
-
|
|
528
|
-
added = set(self._get_untracked_files())
|
|
529
|
-
deleted: set[str] = set()
|
|
530
|
-
try:
|
|
531
|
-
output = subprocess.check_output(
|
|
532
|
-
["git", "diff", "HEAD", "--name-status"],
|
|
533
|
-
cwd=self.project_root,
|
|
534
|
-
stderr=subprocess.DEVNULL,
|
|
535
|
-
).decode("utf-8", errors="replace").splitlines()
|
|
536
|
-
for line in output:
|
|
537
|
-
parts = line.split("\t")
|
|
538
|
-
if len(parts) < 2:
|
|
539
|
-
continue
|
|
540
|
-
status = parts[0]
|
|
541
|
-
path = parts[-1].replace("\\", "/")
|
|
542
|
-
if status.startswith("A"):
|
|
543
|
-
added.add(path)
|
|
544
|
-
elif status.startswith("D"):
|
|
545
|
-
deleted.add(path)
|
|
546
|
-
except Exception as e:
|
|
547
|
-
logger.debug("Failed to classify changed files: %s", e)
|
|
548
|
-
return sorted(added & changed_set), sorted(deleted & changed_set)
|
|
233
|
+
return classify_change_paths(self.project_root, changed_files, self._get_untracked_files)
|
|
549
234
|
|
|
550
235
|
def _diff_coverage_settings(self) -> Tuple[bool, bool, float]:
|
|
551
236
|
"""Return (measure, enforce, min_ratio) with safe defaults when unconfigured."""
|
|
@@ -557,1289 +242,78 @@ class Verifier:
|
|
|
557
242
|
except Exception:
|
|
558
243
|
return True, False, 0.0
|
|
559
244
|
|
|
560
|
-
def _resolve_coverage_python(self, env: Dict[str, str]) -> str:
|
|
561
|
-
if self._coverage_python:
|
|
562
|
-
return self._coverage_python
|
|
563
|
-
for name in ("python", "python3", "py"):
|
|
564
|
-
found = shutil.which(name, path=env.get("PATH"))
|
|
565
|
-
if found:
|
|
566
|
-
return found
|
|
567
|
-
return sys.executable
|
|
568
|
-
|
|
569
|
-
def _coverage_available(self, python: str, env: Dict[str, str]) -> bool:
|
|
570
|
-
try:
|
|
571
|
-
result = subprocess.run(
|
|
572
|
-
[python, "-m", "coverage", "--version"],
|
|
573
|
-
cwd=self.project_root,
|
|
574
|
-
capture_output=True,
|
|
575
|
-
text=True,
|
|
576
|
-
encoding="utf-8",
|
|
577
|
-
errors="replace",
|
|
578
|
-
timeout=30,
|
|
579
|
-
env=env,
|
|
580
|
-
)
|
|
581
|
-
return result.returncode == 0
|
|
582
|
-
except Exception:
|
|
583
|
-
return False
|
|
584
|
-
|
|
585
245
|
def _coverage_target_commands(self, task: Task) -> List[str]:
|
|
586
|
-
|
|
587
|
-
if task.expected_tests:
|
|
588
|
-
return list(task.expected_tests)
|
|
589
|
-
test_like = [c for c in task.allowed_commands if self._command_can_prove_acceptance("allowed", c)]
|
|
590
|
-
if test_like:
|
|
591
|
-
return test_like
|
|
592
|
-
return list(self._load_commands().get("test", []))
|
|
246
|
+
return self._get_coverage_measurer().coverage_target_commands(task)
|
|
593
247
|
|
|
594
248
|
def measure_diff_coverage(self, task: Task, diff_content: str) -> dc.DiffCoverageResult:
|
|
595
|
-
"""Run the task's test command(s) under coverage and intersect with the diff.
|
|
596
|
-
|
|
597
|
-
Returns an *unmeasured* result (never a false positive) whenever reliable
|
|
598
|
-
data is unavailable: no measurable Python changes, no instrumentable test
|
|
599
|
-
command, or no coverage tool in the target environment.
|
|
600
|
-
"""
|
|
601
|
-
changed = dc.measurable_python_changes(dc.parse_changed_lines(diff_content))
|
|
602
|
-
if not changed:
|
|
603
|
-
return dc.DiffCoverageResult(measured=False, reason="no measurable Python changes in diff")
|
|
604
|
-
commands = self._coverage_target_commands(task)
|
|
605
|
-
if not commands:
|
|
606
|
-
return dc.DiffCoverageResult(measured=False, reason="no test command to instrument")
|
|
607
|
-
|
|
608
|
-
env = self._verification_env()
|
|
609
|
-
python = self._resolve_coverage_python(env)
|
|
610
|
-
if not self._coverage_available(python, env):
|
|
611
|
-
return dc.DiffCoverageResult(measured=False, reason="coverage tool not available in target environment")
|
|
612
|
-
|
|
613
|
-
try:
|
|
614
|
-
timeout = load_config(self.project_root).execution.command_timeout
|
|
615
|
-
except Exception:
|
|
616
|
-
timeout = 300
|
|
617
|
-
|
|
618
|
-
tmp_dir = self.project_root / ".devcouncil" / "tmp"
|
|
619
|
-
tmp_dir.mkdir(parents=True, exist_ok=True)
|
|
620
|
-
data_file = tmp_dir / f"diffcov-{task.id}.coverage"
|
|
621
|
-
json_file = tmp_dir / f"diffcov-{task.id}.json"
|
|
622
|
-
for stale in (data_file, json_file):
|
|
623
|
-
try:
|
|
624
|
-
stale.unlink()
|
|
625
|
-
except FileNotFoundError:
|
|
626
|
-
pass
|
|
627
|
-
|
|
628
|
-
ran_any = False
|
|
629
|
-
append = False
|
|
630
|
-
inline_scripts: List[Path] = []
|
|
631
|
-
try:
|
|
632
|
-
for idx, cmd in enumerate(commands):
|
|
633
|
-
argv = self._split_command(cmd)
|
|
634
|
-
inline = dc.inline_python_code(argv)
|
|
635
|
-
if inline is not None:
|
|
636
|
-
# Materialise `python -c "CODE"` as a temp script so coverage can
|
|
637
|
-
# instrument it (coverage cannot run a bare -c snippet).
|
|
638
|
-
script = tmp_dir / f"diffcov-inline-{task.id}-{idx}.py"
|
|
639
|
-
try:
|
|
640
|
-
script.write_text(dc.inline_script_content(inline, self.project_root), encoding="utf-8")
|
|
641
|
-
except Exception as exc:
|
|
642
|
-
logger.warning("Diff-coverage inline script write failed for %s: %s", task.id, exc)
|
|
643
|
-
continue
|
|
644
|
-
inline_scripts.append(script)
|
|
645
|
-
cov_argv: Optional[List[str]] = dc.coverage_run_script_argv(
|
|
646
|
-
str(script), python, append=append, data_file=str(data_file)
|
|
647
|
-
)
|
|
648
|
-
else:
|
|
649
|
-
cov_argv = dc.coverage_run_argv(argv, python, append=append, data_file=str(data_file))
|
|
650
|
-
if cov_argv is None:
|
|
651
|
-
continue
|
|
652
|
-
try:
|
|
653
|
-
subprocess.run(
|
|
654
|
-
cov_argv,
|
|
655
|
-
cwd=self.project_root,
|
|
656
|
-
capture_output=True,
|
|
657
|
-
text=True,
|
|
658
|
-
encoding="utf-8",
|
|
659
|
-
errors="replace",
|
|
660
|
-
timeout=timeout,
|
|
661
|
-
env=env,
|
|
662
|
-
)
|
|
663
|
-
except Exception as exc:
|
|
664
|
-
logger.warning("Diff-coverage run failed for %s: %s", task.id, exc)
|
|
665
|
-
continue
|
|
666
|
-
ran_any = True
|
|
667
|
-
append = True
|
|
668
|
-
|
|
669
|
-
if not ran_any:
|
|
670
|
-
return dc.DiffCoverageResult(measured=False, reason="no instrumentable test command")
|
|
671
|
-
if not data_file.exists():
|
|
672
|
-
return dc.DiffCoverageResult(measured=False, reason="coverage produced no data")
|
|
673
|
-
|
|
674
|
-
try:
|
|
675
|
-
subprocess.run(
|
|
676
|
-
[python, "-m", "coverage", "json", f"--data-file={data_file}", "-o", str(json_file)],
|
|
677
|
-
cwd=self.project_root,
|
|
678
|
-
capture_output=True,
|
|
679
|
-
text=True,
|
|
680
|
-
encoding="utf-8",
|
|
681
|
-
errors="replace",
|
|
682
|
-
timeout=120,
|
|
683
|
-
env=env,
|
|
684
|
-
)
|
|
685
|
-
data = json.loads(json_file.read_text(encoding="utf-8"))
|
|
686
|
-
except Exception as exc:
|
|
687
|
-
return dc.DiffCoverageResult(measured=False, reason=f"coverage report unreadable: {exc}")
|
|
688
|
-
|
|
689
|
-
coverage = dc.parse_coverage_json(data, self.project_root)
|
|
690
|
-
return dc.intersect(changed, coverage, tool="coverage.py")
|
|
691
|
-
finally:
|
|
692
|
-
for path in [data_file, json_file, *inline_scripts]:
|
|
693
|
-
try:
|
|
694
|
-
path.unlink()
|
|
695
|
-
except OSError:
|
|
696
|
-
pass
|
|
249
|
+
"""Run the task's test command(s) under coverage and intersect with the diff."""
|
|
250
|
+
return self._get_coverage_measurer().measure(task, diff_content)
|
|
697
251
|
|
|
698
252
|
async def verify_task(self, task: Task, requirements: List[Requirement]) -> Tuple[List[Gap], List[Any]]:
|
|
699
253
|
logger.info("verify_task: task=%s requirements=%d", task.id, len(requirements))
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
254
|
+
from devcouncil.indexing.map_refresh import refresh_stale_map_if_needed
|
|
255
|
+
|
|
256
|
+
refresh_stale_map_if_needed(self.project_root, on_checkout=False, on_verify=True)
|
|
257
|
+
prime_verify_memos(self)
|
|
258
|
+
ctx = resolve_verify_context(self, task, requirements)
|
|
259
|
+
compile_future, review_future = start_verify_futures(
|
|
260
|
+
self,
|
|
261
|
+
task=task,
|
|
262
|
+
requirements=requirements,
|
|
263
|
+
diff_content=ctx.diff_content,
|
|
264
|
+
ac_samples=ctx.ac_samples,
|
|
265
|
+
ac_per_criterion=ctx.ac_per_criterion,
|
|
266
|
+
)
|
|
709
267
|
try:
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
ac_samples = max(1, _cfg.verification.acceptance_checks.samples)
|
|
713
|
-
ac_repair_attempts = max(0, _cfg.verification.acceptance_checks.repair_attempts)
|
|
714
|
-
ac_per_criterion = bool(_cfg.verification.acceptance_checks.per_criterion)
|
|
715
|
-
except Exception:
|
|
716
|
-
self._command_timeout_cache = 300
|
|
717
|
-
ac_per_criterion = False
|
|
718
|
-
changed_files = self.get_task_changed_files(task.id)
|
|
719
|
-
diff_content = self.get_diff()
|
|
720
|
-
# When the working tree is clean but the task's work was committed (dev go commits
|
|
721
|
-
# between repair attempts and before reconciliation), fall back to the committed
|
|
722
|
-
# checkpoint diff. Otherwise acceptance compilation/review below — gated on a
|
|
723
|
-
# non-empty diff_content — would be skipped, leaving every criterion unproven and
|
|
724
|
-
# wrongly blocking correct, already-committed code.
|
|
725
|
-
if not diff_content.strip():
|
|
726
|
-
committed_diff = self._committed_task_diff(task.id)
|
|
727
|
-
if committed_diff.strip():
|
|
728
|
-
diff_content = committed_diff
|
|
729
|
-
diff_empty = not bool(diff_content.strip())
|
|
730
|
-
# Launch the two independent LLM passes — acceptance compilation and the advisory
|
|
731
|
-
# implementation review — concurrently as soon as the diff is available, instead
|
|
732
|
-
# of awaiting them sequentially later. Each depends only on (task, requirements,
|
|
733
|
-
# diff_content), so there is no data hazard; each result is awaited (with its
|
|
734
|
-
# existing try/except) at the point it is consumed below. The create-time guards
|
|
735
|
-
# match the consume-time guards exactly, so every task created is always awaited.
|
|
736
|
-
compile_future: Optional["asyncio.Task[Dict[str, List[str]]]"] = None
|
|
737
|
-
if self.acceptance_compiler and diff_content and task.acceptance_criterion_ids:
|
|
738
|
-
# Prefer the self-consistency interface; fall back to single-shot ``compile`` so
|
|
739
|
-
# older compiler doubles/implementations keep working.
|
|
740
|
-
if hasattr(self.acceptance_compiler, "compile_candidates"):
|
|
741
|
-
_compile_coro = self.acceptance_compiler.compile_candidates(
|
|
742
|
-
task, requirements, diff_content, samples=ac_samples,
|
|
743
|
-
per_criterion=ac_per_criterion,
|
|
744
|
-
)
|
|
745
|
-
else:
|
|
746
|
-
_compile_coro = self.acceptance_compiler.compile(task, requirements, diff_content)
|
|
747
|
-
compile_future = asyncio.create_task(_compile_coro)
|
|
748
|
-
review_future: Optional["asyncio.Task[Any]"] = None
|
|
749
|
-
if self.reviewer and diff_content:
|
|
750
|
-
review_future = asyncio.create_task(
|
|
751
|
-
self.reviewer.review_changes(task, requirements, diff_content)
|
|
268
|
+
return await run_verify_orchestration(
|
|
269
|
+
self, task, requirements, ctx, compile_future, review_future,
|
|
752
270
|
)
|
|
753
|
-
try:
|
|
754
|
-
# "Work present" is broader than the current working-tree diff: a task whose
|
|
755
|
-
# changes were already committed (e.g. `dev go`'s per-task commit, then the
|
|
756
|
-
# final reconciliation pass where `git diff HEAD` is empty) still counts as
|
|
757
|
-
# implemented. A genuine no-op run has neither a working diff nor committed
|
|
758
|
-
# changes since the task's checkpoint.
|
|
759
|
-
work_present = (not diff_empty) or self._task_produced_changes(task.id)
|
|
760
|
-
|
|
761
|
-
# Empty-diff guard. If the task declares files to create or modify but produced
|
|
762
|
-
# NO work at all, there is nothing to prove — an agent must not be able to
|
|
763
|
-
# declare victory having written nothing (or after a transient git error that
|
|
764
|
-
# degraded the diff to ""). This is the single most dangerous false-pass for
|
|
765
|
-
# autonomy, so it blocks regardless of which commands ran.
|
|
766
|
-
expects_change = any(pf.allowed_change != "read_only" for pf in task.planned_files)
|
|
767
|
-
if not work_present and expects_change:
|
|
768
|
-
gaps.append(Gap(
|
|
769
|
-
id=self._next_gap_id(task.id, "NODIFF"),
|
|
770
|
-
severity="high",
|
|
771
|
-
gap_type="task_not_implemented",
|
|
772
|
-
task_id=task.id,
|
|
773
|
-
description=(
|
|
774
|
-
f"Task {task.id} declares files to create or modify, but produced no "
|
|
775
|
-
"changes. Verification cannot prove work that does not exist."
|
|
776
|
-
),
|
|
777
|
-
evidence=[f"planned files expecting change: {sorted(p.path for p in task.planned_files if p.allowed_change != 'read_only')}"],
|
|
778
|
-
recommended_fix=(
|
|
779
|
-
"Implement the planned changes so the diff is non-empty, then re-verify. "
|
|
780
|
-
"If you did make changes, ensure they are saved and visible to git "
|
|
781
|
-
"(not reverted, stashed, or written outside the project root)."
|
|
782
|
-
),
|
|
783
|
-
blocking=True,
|
|
784
|
-
))
|
|
785
|
-
|
|
786
|
-
if diff_content:
|
|
787
|
-
added_files, deleted_files = self._classify_change_paths(changed_files)
|
|
788
|
-
diff_ev = DiffEvidence(
|
|
789
|
-
task_id=task.id,
|
|
790
|
-
changed_files=changed_files,
|
|
791
|
-
added_files=added_files,
|
|
792
|
-
deleted_files=deleted_files,
|
|
793
|
-
diff_summary=f"Diff captured for {len(changed_files)} files."
|
|
794
|
-
)
|
|
795
|
-
evidence_to_save.append(diff_ev)
|
|
796
|
-
|
|
797
|
-
# 1. Planned-file coverage check
|
|
798
|
-
planned_paths = {pf.path for pf in task.planned_files}
|
|
799
|
-
changed_set = set(changed_files)
|
|
800
|
-
for pf in task.planned_files:
|
|
801
|
-
if pf.path not in changed_set and pf.allowed_change != "read_only":
|
|
802
|
-
gaps.append(Gap(
|
|
803
|
-
id=self._next_gap_id(task.id, "FILE"),
|
|
804
|
-
severity="medium",
|
|
805
|
-
gap_type="planned_file_not_changed",
|
|
806
|
-
task_id=task.id,
|
|
807
|
-
description=f"Planned file {pf.path} was not modified.",
|
|
808
|
-
recommended_fix=f"Modify {pf.path} as planned or update the task.",
|
|
809
|
-
blocking=False,
|
|
810
|
-
file=pf.path,
|
|
811
|
-
))
|
|
812
|
-
|
|
813
|
-
# 2. Orphan-diff detection
|
|
814
|
-
for cf in changed_files:
|
|
815
|
-
if cf not in planned_paths:
|
|
816
|
-
gaps.append(Gap(
|
|
817
|
-
id=self._next_gap_id(task.id, "ORPHAN"),
|
|
818
|
-
severity="high",
|
|
819
|
-
gap_type="orphan_diff",
|
|
820
|
-
task_id=task.id,
|
|
821
|
-
description=f"File {cf} was modified but not planned for this task.",
|
|
822
|
-
evidence=[cf],
|
|
823
|
-
recommended_fix=f"Revert changes to {cf} or add it to the task's planned files.",
|
|
824
|
-
blocking=True,
|
|
825
|
-
file=cf,
|
|
826
|
-
))
|
|
827
|
-
|
|
828
|
-
gaps.extend(self._check_semantic_diff(task, requirements))
|
|
829
|
-
|
|
830
|
-
# 3. Dependency change detection
|
|
831
|
-
dep_changes = self._check_dependency_changes(changed_files)
|
|
832
|
-
for dep_file in dep_changes:
|
|
833
|
-
if dep_file not in planned_paths:
|
|
834
|
-
gaps.append(Gap(
|
|
835
|
-
id=self._next_gap_id(task.id, "DEP"),
|
|
836
|
-
severity="high",
|
|
837
|
-
gap_type="dependency_risk",
|
|
838
|
-
task_id=task.id,
|
|
839
|
-
description=f"Dependency file {dep_file} was modified without being in planned files.",
|
|
840
|
-
evidence=[dep_file],
|
|
841
|
-
recommended_fix=f"Justify the dependency change or revert {dep_file}.",
|
|
842
|
-
blocking=True,
|
|
843
|
-
file=dep_file,
|
|
844
|
-
))
|
|
845
|
-
|
|
846
|
-
# When DevCouncil can compile its own per-criterion checks, THOSE are the
|
|
847
|
-
# authority and the planner's expected_tests are demoted to advisory — so a
|
|
848
|
-
# bogus planner command (irrelevant linters, npm on a Python project, tests
|
|
849
|
-
# that reference missing files) can no longer block correct work.
|
|
850
|
-
compiler_active = bool(self.acceptance_compiler and diff_content and task.acceptance_criterion_ids)
|
|
851
|
-
|
|
852
|
-
# 4. Run verification commands
|
|
853
|
-
command_results: List[CommandResult] = []
|
|
854
|
-
evidence_results: List[CommandResult] = []
|
|
855
|
-
genuine_failure = False # a command that actually ran and failed (real defect signal)
|
|
856
|
-
had_unrunnable = False # a command that could not run (missing tool / missing tests)
|
|
857
|
-
# Genuine test failures demoted to non-blocking only because a compiler is active.
|
|
858
|
-
# That demotion is legitimate ONLY if the compiler actually produces per-criterion
|
|
859
|
-
# checks to take authority; re-promoted below if it produces none.
|
|
860
|
-
demoted_failures: List[Gap] = []
|
|
861
|
-
for cmd_type, cmds in self._commands_for_task(task).items():
|
|
862
|
-
for cmd in cmds:
|
|
863
|
-
applicable, skip_reason = self._command_applicable(cmd)
|
|
864
|
-
if not applicable:
|
|
865
|
-
# Wrong-stack command (e.g. `npm test` on a Python repo): skip it
|
|
866
|
-
# entirely rather than running and failing for a stack reason — an
|
|
867
|
-
# advisory note so the skip is visible (no silent drop).
|
|
868
|
-
gaps.append(Gap(
|
|
869
|
-
id=self._next_gap_id(task.id, "SKIP"),
|
|
870
|
-
severity="low",
|
|
871
|
-
gap_type="skipped_verification_command",
|
|
872
|
-
task_id=task.id,
|
|
873
|
-
description=f"Skipped verification command '{cmd}': {skip_reason}.",
|
|
874
|
-
evidence=[skip_reason],
|
|
875
|
-
recommended_fix=(
|
|
876
|
-
"Replace it with a command for this repo's stack, or remove it "
|
|
877
|
-
"from .devcouncil/config.yaml / the task's expected_tests."
|
|
878
|
-
),
|
|
879
|
-
blocking=False,
|
|
880
|
-
suggested_command=cmd,
|
|
881
|
-
))
|
|
882
|
-
continue
|
|
883
|
-
result = self._run_command(cmd, task_id=task.id)
|
|
884
|
-
command_results.append(result)
|
|
885
|
-
evidence_to_save.append(result)
|
|
886
|
-
if self._command_can_prove_acceptance(cmd_type, cmd):
|
|
887
|
-
evidence_results.append(result)
|
|
888
|
-
if result.exit_code != 0:
|
|
889
|
-
if self._command_is_malformed(result):
|
|
890
|
-
had_unrunnable = True
|
|
891
|
-
# The verification command itself could not run (e.g. a
|
|
892
|
-
# SyntaxError in a `python -c` one-liner, or a missing test
|
|
893
|
-
# tool). This proves nothing about the implementation, so do
|
|
894
|
-
# not report it as a code failure — surface it as a plan/
|
|
895
|
-
# command defect the user can regenerate instead.
|
|
896
|
-
gaps.append(Gap(
|
|
897
|
-
id=self._next_gap_id(task.id, "BADCMD"),
|
|
898
|
-
severity="medium",
|
|
899
|
-
gap_type="invalid_verification_command",
|
|
900
|
-
task_id=task.id,
|
|
901
|
-
description=(
|
|
902
|
-
f"Verification command could not run (not a code failure): '{cmd}'. "
|
|
903
|
-
"It appears malformed or its tooling is unavailable, so this command "
|
|
904
|
-
"proves nothing either way."
|
|
905
|
-
),
|
|
906
|
-
evidence=[result.summary[:500]],
|
|
907
|
-
recommended_fix=(
|
|
908
|
-
"Regenerate the task's verification commands with 'dev repair', or edit "
|
|
909
|
-
"them to be a single runnable command (e.g. 'python -m pytest <file>')."
|
|
910
|
-
),
|
|
911
|
-
# Non-blocking: a command that cannot run is not evidence of a
|
|
912
|
-
# defect. If it was the *only* check for an acceptance criterion,
|
|
913
|
-
# that criterion is independently caught as unproven (blocking).
|
|
914
|
-
blocking=False,
|
|
915
|
-
suggested_command=cmd,
|
|
916
|
-
stdout_path=result.stdout_path or None,
|
|
917
|
-
stderr_path=result.stderr_path or None,
|
|
918
|
-
))
|
|
919
|
-
else:
|
|
920
|
-
# A verification command that genuinely failed. Lint/typecheck
|
|
921
|
-
# commands (from the config fallback) report style/type opinion,
|
|
922
|
-
# not a correctness defect, so they are ADVISORY — blocking a
|
|
923
|
-
# behaviorally-correct task on `flake8`/`mypy`/`ruff` is the
|
|
924
|
-
# false-block the benchmark surfaced. A real test failure still
|
|
925
|
-
# gates (unless compiled checks supersede it).
|
|
926
|
-
is_quality_gate = cmd_type in {"lint", "typecheck"} or self._is_quality_only_command(cmd)
|
|
927
|
-
blocking = (not compiler_active) and not is_quality_gate
|
|
928
|
-
if blocking:
|
|
929
|
-
genuine_failure = True
|
|
930
|
-
fail_file, fail_line = self._failure_location(result)
|
|
931
|
-
gap = Gap(
|
|
932
|
-
id=self._next_gap_id(task.id, cmd_type.upper()),
|
|
933
|
-
severity="high" if blocking else "medium",
|
|
934
|
-
gap_type="quality_gate_failed" if is_quality_gate else "test_failed",
|
|
935
|
-
task_id=task.id,
|
|
936
|
-
description=(
|
|
937
|
-
f"{'Quality gate' if is_quality_gate else 'Command'} '{cmd}' "
|
|
938
|
-
f"failed with exit code {result.exit_code}"
|
|
939
|
-
+ (" (advisory: style/type, not a correctness gate)." if is_quality_gate else ".")
|
|
940
|
-
),
|
|
941
|
-
evidence=[result.summary[:500]],
|
|
942
|
-
recommended_fix=f"Fix the issues reported by '{cmd}'.",
|
|
943
|
-
blocking=blocking,
|
|
944
|
-
suggested_command=cmd,
|
|
945
|
-
file=fail_file,
|
|
946
|
-
line=fail_line,
|
|
947
|
-
stdout_path=result.stdout_path or None,
|
|
948
|
-
stderr_path=result.stderr_path or None,
|
|
949
|
-
)
|
|
950
|
-
gaps.append(gap)
|
|
951
|
-
# A real test failure demoted only because the compiler is active:
|
|
952
|
-
# remember it so we can re-promote if the compiler yields no checks.
|
|
953
|
-
if compiler_active and not is_quality_gate and not blocking:
|
|
954
|
-
demoted_failures.append(gap)
|
|
955
|
-
|
|
956
|
-
# 4b. Compiled acceptance checks — precise, DevCouncil-owned per-criterion
|
|
957
|
-
# evidence. Derive one runnable check per acceptance criterion from the
|
|
958
|
-
# criterion text + the diff, instead of trusting planner-authored
|
|
959
|
-
# expected_tests (which the benchmark showed often reference absent tools or
|
|
960
|
-
# test files). Each check maps 1:1 to its criterion, replacing the coarse
|
|
961
|
-
# "any command passed -> every criterion proven" mapping.
|
|
962
|
-
compiled_pass: Dict[str, bool] = {}
|
|
963
|
-
# Per-AC bookkeeping so the unproven-AC gap can attach ONLY the check(s) that
|
|
964
|
-
# targeted that criterion (and the specific failing result), instead of dumping
|
|
965
|
-
# every command summary. Keys are AC ids; values track the compiled command(s)
|
|
966
|
-
# and any failing CommandResults for that AC.
|
|
967
|
-
compiled_cmds_by_ac: Dict[str, List[str]] = {}
|
|
968
|
-
failing_results_by_ac: Dict[str, List[CommandResult]] = {}
|
|
969
|
-
# Per-AC vote tally for proven criteria: {ac_id: (passes, decisive, repaired)}.
|
|
970
|
-
# Recorded into the stored TestEvidence so an audit can see HOW a criterion was
|
|
971
|
-
# proven (single check vs. majority of independent checks; whether a check had to
|
|
972
|
-
# be repaired to run) instead of just "passed".
|
|
973
|
-
compiled_vote: Dict[str, Tuple[int, int, bool]] = {}
|
|
974
|
-
# ACs whose independently-generated checks split (some pass, some fail) with no
|
|
975
|
-
# majority. Per policy this is inconclusive — neither proof nor a defect — so the
|
|
976
|
-
# AC is surfaced NON-blocking below instead of false-blocking on a lone bad check.
|
|
977
|
-
inconclusive_acs: set[str] = set()
|
|
978
|
-
if compile_future is not None:
|
|
979
|
-
try:
|
|
980
|
-
compiled = await compile_future
|
|
981
|
-
except Exception as exc: # pragma: no cover - best effort
|
|
982
|
-
logger.warning("Acceptance compiler failed for %s: %s", task.id, exc)
|
|
983
|
-
compiled = {}
|
|
984
|
-
ac_meta = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
|
|
985
|
-
for ac_id, raw_cmds in compiled.items():
|
|
986
|
-
# Defensive: drop any wrong-stack candidate so it can't fail an AC for a
|
|
987
|
-
# stack reason (the compiler is told not to emit these).
|
|
988
|
-
candidates = [c for c in raw_cmds if self._command_applicable(c)[0]]
|
|
989
|
-
compiled_cmds_by_ac[ac_id] = list(candidates)
|
|
990
|
-
if not candidates:
|
|
991
|
-
compiled_pass[ac_id] = False
|
|
992
|
-
continue
|
|
993
|
-
# Run each INDEPENDENT candidate; a check that merely failed to RUN
|
|
994
|
-
# (malformed/unrunnable) is regenerated from the launcher error up to
|
|
995
|
-
# ``ac_repair_attempts`` times — safe, because a check that never ran
|
|
996
|
-
# proves nothing, so repairing it cannot weaken the gate.
|
|
997
|
-
passes = 0
|
|
998
|
-
genuine_fails = 0
|
|
999
|
-
repaired = False # a check had to be regenerated before it ran
|
|
1000
|
-
fail_results: List[Tuple[str, CommandResult]] = []
|
|
1001
|
-
for cmd in candidates:
|
|
1002
|
-
result = self._run_command(cmd, task_id=task.id)
|
|
1003
|
-
command_results.append(result)
|
|
1004
|
-
evidence_to_save.append(result)
|
|
1005
|
-
attempts = 0
|
|
1006
|
-
_repair = getattr(self.acceptance_compiler, "repair", None)
|
|
1007
|
-
while (
|
|
1008
|
-
result.exit_code != 0
|
|
1009
|
-
and self._command_is_malformed(result)
|
|
1010
|
-
and attempts < ac_repair_attempts
|
|
1011
|
-
and _repair is not None
|
|
1012
|
-
):
|
|
1013
|
-
attempts += 1
|
|
1014
|
-
ac_desc = ac_meta[ac_id].description if ac_id in ac_meta else ac_id
|
|
1015
|
-
try:
|
|
1016
|
-
fixed = await _repair(
|
|
1017
|
-
ac_id, ac_desc, cmd, result.summary[:800], diff_content
|
|
1018
|
-
)
|
|
1019
|
-
except Exception:
|
|
1020
|
-
fixed = None
|
|
1021
|
-
if not fixed or not self._command_applicable(fixed)[0]:
|
|
1022
|
-
break
|
|
1023
|
-
cmd = fixed
|
|
1024
|
-
compiled_cmds_by_ac[ac_id].append(cmd)
|
|
1025
|
-
result = self._run_command(cmd, task_id=task.id)
|
|
1026
|
-
command_results.append(result)
|
|
1027
|
-
evidence_to_save.append(result)
|
|
1028
|
-
if result.exit_code == 0:
|
|
1029
|
-
passes += 1
|
|
1030
|
-
if attempts > 0:
|
|
1031
|
-
repaired = True
|
|
1032
|
-
elif self._command_is_malformed(result):
|
|
1033
|
-
# Still couldn't run after repair: proves nothing either way.
|
|
1034
|
-
had_unrunnable = True
|
|
1035
|
-
failing_results_by_ac.setdefault(ac_id, []).append(result)
|
|
1036
|
-
else:
|
|
1037
|
-
genuine_fails += 1
|
|
1038
|
-
fail_results.append((cmd, result))
|
|
1039
|
-
failing_results_by_ac.setdefault(ac_id, []).append(result)
|
|
1040
|
-
decisive = passes + genuine_fails
|
|
1041
|
-
# Majority vote over the checks that actually ran. Proven iff a strict
|
|
1042
|
-
# majority pass; unanimous failure of independent checks is strong evidence
|
|
1043
|
-
# of a real defect and blocks; a split is inconclusive (handled below).
|
|
1044
|
-
ac_proven = decisive > 0 and passes > genuine_fails
|
|
1045
|
-
compiled_pass[ac_id] = ac_proven
|
|
1046
|
-
if ac_proven:
|
|
1047
|
-
compiled_vote[ac_id] = (passes, decisive, repaired)
|
|
1048
|
-
continue
|
|
1049
|
-
if passes == 0 and genuine_fails > 0:
|
|
1050
|
-
genuine_failure = True
|
|
1051
|
-
cmd, result = fail_results[0]
|
|
1052
|
-
fail_file, fail_line = self._failure_location(result)
|
|
1053
|
-
agree = (
|
|
1054
|
-
f" {genuine_fails}/{decisive} independent checks agreed it fails."
|
|
1055
|
-
if decisive > 1 else ""
|
|
1056
|
-
)
|
|
1057
|
-
gaps.append(Gap(
|
|
1058
|
-
id=self._next_gap_id(task.id, "ACCHK"),
|
|
1059
|
-
severity="high",
|
|
1060
|
-
gap_type="test_failed",
|
|
1061
|
-
task_id=task.id,
|
|
1062
|
-
description=f"Acceptance check for {ac_id} failed: '{cmd}' (exit {result.exit_code}).{agree}",
|
|
1063
|
-
evidence=[result.summary[:500]],
|
|
1064
|
-
recommended_fix=f"Fix the implementation so acceptance criterion {ac_id} holds.",
|
|
1065
|
-
blocking=True,
|
|
1066
|
-
acceptance_criterion_id=ac_id,
|
|
1067
|
-
suggested_command=cmd,
|
|
1068
|
-
file=fail_file,
|
|
1069
|
-
line=fail_line,
|
|
1070
|
-
stdout_path=result.stdout_path or None,
|
|
1071
|
-
stderr_path=result.stderr_path or None,
|
|
1072
|
-
))
|
|
1073
|
-
elif passes > 0 and genuine_fails > 0:
|
|
1074
|
-
# Independent checks disagree with no majority: neither proof nor a
|
|
1075
|
-
# defect. Mark inconclusive so the unproven-AC gap below is NON-blocking
|
|
1076
|
-
# (never false-block on a lone bad check, never auto-pass a real bug).
|
|
1077
|
-
inconclusive_acs.add(ac_id)
|
|
1078
|
-
|
|
1079
|
-
# The compiler only earns the authority to demote a genuinely-failing planner
|
|
1080
|
-
# test if it produced a per-criterion check for EVERY targeted AC. A partial
|
|
1081
|
-
# compile is not enough: the uncovered ACs fall back to the coarse signal, so a
|
|
1082
|
-
# demoted real failure + coarse-proven remainder would otherwise slip past the
|
|
1083
|
-
# gate. If coverage is incomplete (or zero — empty compile / all-wrong-stack /
|
|
1084
|
-
# a compile exception swallowed to {}), re-promote the demoted failures.
|
|
1085
|
-
compiler_covered_all = bool(task.acceptance_criterion_ids) and all(
|
|
1086
|
-
compiled_cmds_by_ac.get(ac_id) for ac_id in task.acceptance_criterion_ids
|
|
1087
|
-
)
|
|
1088
|
-
if compiler_active and not compiler_covered_all and demoted_failures:
|
|
1089
|
-
for gap in demoted_failures:
|
|
1090
|
-
gap.blocking = True
|
|
1091
|
-
gap.severity = "high"
|
|
1092
|
-
genuine_failure = True
|
|
1093
|
-
logger.info(
|
|
1094
|
-
"Re-promoted demoted test failure %s to blocking: acceptance compiler "
|
|
1095
|
-
"did not produce a check for every criterion of task %s.",
|
|
1096
|
-
gap.id, task.id,
|
|
1097
|
-
)
|
|
1098
|
-
|
|
1099
|
-
# 5. Acceptance-criteria evidence mapping (precise, per criterion).
|
|
1100
|
-
# Quality-only commands (lint/typecheck) are excluded: a passing `mypy`/`ruff
|
|
1101
|
-
# check`/`tsc` exercises no behavior, so it must not coarse-prove a behavioral AC
|
|
1102
|
-
# — the same false-confidence the per-criterion checks exist to prevent.
|
|
1103
|
-
successful_commands = [
|
|
1104
|
-
result for result in evidence_results
|
|
1105
|
-
if result.exit_code == 0 and not self._is_quality_only_command(result.command)
|
|
1106
|
-
]
|
|
1107
|
-
# Coarse fallback (used only when no compiled per-criterion check exists for an
|
|
1108
|
-
# AC): a criterion may be marked proven by a passing acceptance-capable command
|
|
1109
|
-
# ONLY when the task actually produced work. Without this guard a no-op run
|
|
1110
|
-
# whose unrelated command happens to pass would "prove" every criterion against
|
|
1111
|
-
# zero changes.
|
|
1112
|
-
coarse_proof_available = work_present and bool(successful_commands)
|
|
1113
|
-
if task.acceptance_criterion_ids:
|
|
1114
|
-
req_by_ac = {ac.id: req.id for req in requirements for ac in req.acceptance_criteria}
|
|
1115
|
-
unproven_acs: List[str] = []
|
|
1116
|
-
coarse_proven_acs: List[str] = []
|
|
1117
|
-
for ac_id in task.acceptance_criterion_ids:
|
|
1118
|
-
# An AC is proven if its compiled check passed; if no compiled check
|
|
1119
|
-
# exists for it, fall back to the coarse signal (any expected_test passed).
|
|
1120
|
-
proven: Optional[bool] = compiled_pass.get(ac_id)
|
|
1121
|
-
coarse = False
|
|
1122
|
-
if proven is None:
|
|
1123
|
-
proven = coarse_proof_available
|
|
1124
|
-
coarse = proven # proven only by the coarse, not-AC-specific signal
|
|
1125
|
-
if proven:
|
|
1126
|
-
if coarse:
|
|
1127
|
-
coarse_proven_acs.append(ac_id)
|
|
1128
|
-
# Don't persist a "passed" record for a coarse-proven criterion during a
|
|
1129
|
-
# run that also has a genuine blocking failure — the gate already fails,
|
|
1130
|
-
# and a stored "passed" would mislead audits that read evidence directly.
|
|
1131
|
-
if not (coarse and genuine_failure):
|
|
1132
|
-
proof_mode: Literal["compiled", "vote", "coarse", ""]
|
|
1133
|
-
if coarse:
|
|
1134
|
-
proof_summary = (
|
|
1135
|
-
"Acceptance criterion proven only by a COARSE signal (a passing "
|
|
1136
|
-
"acceptance-capable command, not a per-criterion check); behavior "
|
|
1137
|
-
"not precisely verified."
|
|
1138
|
-
)
|
|
1139
|
-
proof_mode = "coarse"
|
|
1140
|
-
else:
|
|
1141
|
-
# Make the per-criterion proof auditable: single check vs. majority
|
|
1142
|
-
# of independent checks, and whether a check had to be repaired to run.
|
|
1143
|
-
passes_n, decisive_n, was_repaired = compiled_vote.get(ac_id, (1, 1, False))
|
|
1144
|
-
proof_mode = "vote" if decisive_n > 1 else "compiled"
|
|
1145
|
-
how = (
|
|
1146
|
-
f"a majority vote of independent compiled checks ({passes_n}/{decisive_n} passed)"
|
|
1147
|
-
if decisive_n > 1 else
|
|
1148
|
-
"a per-criterion compiled check"
|
|
1149
|
-
)
|
|
1150
|
-
repaired_note = " (one check was regenerated from its launcher error to run)" if was_repaired else ""
|
|
1151
|
-
proof_summary = f"Acceptance criterion proven by {how}.{repaired_note}"
|
|
1152
|
-
evidence_to_save.append(TestEvidence(
|
|
1153
|
-
requirement_id=req_by_ac.get(ac_id, task.requirement_ids[0] if task.requirement_ids else ""),
|
|
1154
|
-
acceptance_criterion_id=ac_id,
|
|
1155
|
-
command="(devcouncil acceptance check)",
|
|
1156
|
-
status="passed",
|
|
1157
|
-
evidence_summary=proof_summary,
|
|
1158
|
-
mode=proof_mode,
|
|
1159
|
-
))
|
|
1160
|
-
else:
|
|
1161
|
-
unproven_acs.append(ac_id)
|
|
1162
|
-
# Surface coarse proof as a first-class advisory: these criteria passed only
|
|
1163
|
-
# because some acceptance-capable command exited 0, not because a check tied
|
|
1164
|
-
# to the criterion passed. Non-blocking, but no longer invisible.
|
|
1165
|
-
if coarse_proven_acs:
|
|
1166
|
-
gaps.append(Gap(
|
|
1167
|
-
id=self._next_gap_id(task.id, "COARSE"),
|
|
1168
|
-
severity="low",
|
|
1169
|
-
gap_type="coarse_acceptance_proof",
|
|
1170
|
-
task_id=task.id,
|
|
1171
|
-
description=(
|
|
1172
|
-
"Verification mode = COARSE for "
|
|
1173
|
-
f"{', '.join(coarse_proven_acs)}: proven by a passing acceptance-capable "
|
|
1174
|
-
"command, not a per-criterion check. Behavior is not precisely verified."
|
|
1175
|
-
),
|
|
1176
|
-
evidence=[f"coarse-proven: {', '.join(coarse_proven_acs)}"],
|
|
1177
|
-
recommended_fix=(
|
|
1178
|
-
"Add a verification command (or test) that exercises each listed criterion "
|
|
1179
|
-
"specifically, so DevCouncil can compile a per-criterion check instead of "
|
|
1180
|
-
"relying on the coarse fallback."
|
|
1181
|
-
),
|
|
1182
|
-
blocking=False,
|
|
1183
|
-
))
|
|
1184
|
-
if unproven_acs:
|
|
1185
|
-
# Block only on positive evidence of a problem. If verification was
|
|
1186
|
-
# attempted but every failure was unrunnable (missing tooling / tests)
|
|
1187
|
-
# and nothing genuinely failed, that is a verification defect, not a
|
|
1188
|
-
# code defect — surface it as a non-blocking "could not verify".
|
|
1189
|
-
couldnt_verify = had_unrunnable and not genuine_failure and work_present
|
|
1190
|
-
ac_by_id = {ac.id: ac for req in requirements for ac in req.acceptance_criteria}
|
|
1191
|
-
# Methods whose criteria HARD-BLOCK the gate when unproven: only those
|
|
1192
|
-
# that assert BEHAVIOR. Inherently-manual criteria (manual/llm_review),
|
|
1193
|
-
# optional ones, and quality-only `static_check` criteria (PEP 8 /
|
|
1194
|
-
# docstring / formatting) are surfaced for review instead of
|
|
1195
|
-
# false-blocking the autonomous loop. static_check is a quality gate,
|
|
1196
|
-
# not a correctness gate — mirroring how lint/type COMMAND failures are
|
|
1197
|
-
# already demoted to advisory — and the compiler often cannot author a
|
|
1198
|
-
# reliable style check (or the criterion lands on a no-diff process task),
|
|
1199
|
-
# which otherwise blocks correct, style-conforming code.
|
|
1200
|
-
automatable_methods = {"unit_test", "integration_test"}
|
|
1201
|
-
for ac_id in unproven_acs:
|
|
1202
|
-
ac = ac_by_id.get(ac_id)
|
|
1203
|
-
method = ac.verification_method if ac else "unit_test"
|
|
1204
|
-
is_automatable = (ac.required if ac else True) and method in automatable_methods
|
|
1205
|
-
if not is_automatable:
|
|
1206
|
-
blocks = False
|
|
1207
|
-
optional = "" if (ac is None or ac.required) else " optional"
|
|
1208
|
-
fix = (
|
|
1209
|
-
f"This{optional} criterion's verification method is '{method}'; it cannot be "
|
|
1210
|
-
"proven by running code. Review it manually (it does not block the gate)."
|
|
1211
|
-
)
|
|
1212
|
-
suffix = f" (non-blocking: {method})"
|
|
1213
|
-
elif ac_id in inconclusive_acs:
|
|
1214
|
-
# Independently-generated checks split with no majority — inconclusive,
|
|
1215
|
-
# so this does not block (a lone bad check must not fail correct code).
|
|
1216
|
-
blocks = False
|
|
1217
|
-
fix = ("Auto-generated acceptance checks disagreed on this criterion (some "
|
|
1218
|
-
"passed, some failed). Add a precise verification command that "
|
|
1219
|
-
"unambiguously proves it so the result is decisive.")
|
|
1220
|
-
suffix = " (auto-checks inconclusive)"
|
|
1221
|
-
elif couldnt_verify:
|
|
1222
|
-
blocks = False
|
|
1223
|
-
fix = ("Could not verify this criterion: the verification commands did not run "
|
|
1224
|
-
"(missing tooling or tests). Regenerate them with 'dev repair' to confirm the work.")
|
|
1225
|
-
suffix = " (verification commands could not run)"
|
|
1226
|
-
else:
|
|
1227
|
-
blocks = True
|
|
1228
|
-
fix = "Add or fix a verification command that proves this acceptance criterion."
|
|
1229
|
-
suffix = ""
|
|
1230
|
-
# Concrete, AC-scoped evidence instead of "all command summaries":
|
|
1231
|
-
# * if a compiled check targeted this AC, attach its command(s) and
|
|
1232
|
-
# the specific failing result;
|
|
1233
|
-
# * otherwise an explicit "no check compiled" marker so the agent
|
|
1234
|
-
# knows it must author one, not hunt through unrelated output.
|
|
1235
|
-
ac_compiled = compiled_cmds_by_ac.get(ac_id, [])
|
|
1236
|
-
ac_failures = failing_results_by_ac.get(ac_id, [])
|
|
1237
|
-
ac_evidence: List[str] = []
|
|
1238
|
-
suggested_cmd: Optional[str] = None
|
|
1239
|
-
if ac_compiled:
|
|
1240
|
-
suggested_cmd = ac_compiled[0]
|
|
1241
|
-
ac_evidence.extend(f"compiled check: {c}" for c in ac_compiled)
|
|
1242
|
-
ac_evidence.extend(r.summary[:500] for r in ac_failures)
|
|
1243
|
-
else:
|
|
1244
|
-
ac_evidence.append(
|
|
1245
|
-
f"no DevCouncil check compiled for {ac_id} "
|
|
1246
|
-
f"(expected verification method: {method})"
|
|
1247
|
-
)
|
|
1248
|
-
gaps.append(Gap(
|
|
1249
|
-
id=self._next_gap_id(task.id, "AC"),
|
|
1250
|
-
severity="high" if blocks else "medium",
|
|
1251
|
-
gap_type="acceptance_criteria_unproven",
|
|
1252
|
-
requirement_id=self._requirement_id_for_ac(requirements, ac_id),
|
|
1253
|
-
task_id=task.id,
|
|
1254
|
-
description=(
|
|
1255
|
-
f"Acceptance criterion {ac_id} has no passing verification evidence "
|
|
1256
|
-
f"for task {task.id}.{suffix}"
|
|
1257
|
-
),
|
|
1258
|
-
evidence=ac_evidence,
|
|
1259
|
-
recommended_fix=fix,
|
|
1260
|
-
blocking=blocks,
|
|
1261
|
-
acceptance_criterion_id=ac_id,
|
|
1262
|
-
expected_verification_method=method,
|
|
1263
|
-
suggested_command=suggested_cmd,
|
|
1264
|
-
))
|
|
1265
|
-
elif task.requirement_ids:
|
|
1266
|
-
gaps.append(Gap(
|
|
1267
|
-
id=self._next_gap_id(task.id, "NOAC"),
|
|
1268
|
-
severity="high",
|
|
1269
|
-
gap_type="acceptance_criteria_unproven",
|
|
1270
|
-
requirement_id=task.requirement_ids[0],
|
|
1271
|
-
task_id=task.id,
|
|
1272
|
-
description=f"Task {task.id} is linked to requirements but no acceptance criteria.",
|
|
1273
|
-
recommended_fix="Link the task to specific acceptance_criterion_ids before verification.",
|
|
1274
|
-
blocking=True,
|
|
1275
|
-
))
|
|
1276
|
-
|
|
1277
|
-
# 5b. Diff↔coverage gate. A green suite is only acceptance evidence if it
|
|
1278
|
-
# exercised the lines the diff changed. This catches the failure the README
|
|
1279
|
-
# promises to stop: tests "pass" while the new logic is never run (unrelated
|
|
1280
|
-
# suite, code never imported, untouched branch). Measured only when the target
|
|
1281
|
-
# repo has coverage tooling and the diff has measurable Python changes; absent
|
|
1282
|
-
# that, it degrades silently rather than blocking correct work.
|
|
1283
|
-
measure_cov, enforce_cov, min_ratio = self._diff_coverage_settings()
|
|
1284
|
-
any_passing = bool(successful_commands) or any(compiled_pass.values())
|
|
1285
|
-
coverage_measured = False
|
|
1286
|
-
coverage_skipped_reason: Optional[str] = None
|
|
1287
|
-
if not measure_cov:
|
|
1288
|
-
coverage_skipped_reason = "diff coverage disabled in config"
|
|
1289
|
-
elif not diff_content:
|
|
1290
|
-
coverage_skipped_reason = "no diff to measure"
|
|
1291
|
-
elif not task.acceptance_criterion_ids:
|
|
1292
|
-
coverage_skipped_reason = "task has no acceptance criteria"
|
|
1293
|
-
elif not any_passing:
|
|
1294
|
-
coverage_skipped_reason = "no passing verification command to instrument"
|
|
1295
|
-
if measure_cov and diff_content and task.acceptance_criterion_ids and any_passing:
|
|
1296
|
-
cov = self.measure_diff_coverage(task, diff_content)
|
|
1297
|
-
if not cov.measured:
|
|
1298
|
-
coverage_skipped_reason = cov.reason or "diff coverage could not be measured"
|
|
1299
|
-
if cov.measured:
|
|
1300
|
-
coverage_measured = True
|
|
1301
|
-
coverage_skipped_reason = None
|
|
1302
|
-
evidence_to_save.append(DiffCoverageEvidence(
|
|
1303
|
-
task_id=task.id,
|
|
1304
|
-
tool=cov.tool,
|
|
1305
|
-
measured=True,
|
|
1306
|
-
changed_lines=cov.changed_executable_lines,
|
|
1307
|
-
covered_lines=cov.covered_changed_lines,
|
|
1308
|
-
coverage_ratio=cov.ratio,
|
|
1309
|
-
uncovered_by_file=cov.uncovered_by_file,
|
|
1310
|
-
absent_files=cov.absent_files,
|
|
1311
|
-
summary=cov.summary(),
|
|
1312
|
-
))
|
|
1313
|
-
failing = cov.covered_changed_lines == 0 if min_ratio <= 0 else cov.ratio < min_ratio
|
|
1314
|
-
if failing:
|
|
1315
|
-
first_file = next(iter(cov.uncovered_by_file), None)
|
|
1316
|
-
first_lines = cov.uncovered_by_file.get(first_file or "", [])
|
|
1317
|
-
target_cmds = self._coverage_target_commands(task)
|
|
1318
|
-
gaps.append(Gap(
|
|
1319
|
-
id=self._next_gap_id(task.id, "DIFFCOV"),
|
|
1320
|
-
severity="high" if enforce_cov else "medium",
|
|
1321
|
-
gap_type="diff_not_exercised",
|
|
1322
|
-
task_id=task.id,
|
|
1323
|
-
description=(
|
|
1324
|
-
f"Verification commands passed but exercised "
|
|
1325
|
-
f"{cov.covered_changed_lines}/{cov.changed_executable_lines} changed line(s): "
|
|
1326
|
-
f"{cov.summary()}. The acceptance criteria are not proven because the new "
|
|
1327
|
-
"logic was never executed by the tests."
|
|
1328
|
-
),
|
|
1329
|
-
evidence=[cov.summary()] + [
|
|
1330
|
-
f"{path}: lines {lines}" for path, lines in list(cov.uncovered_by_file.items())[:5]
|
|
1331
|
-
],
|
|
1332
|
-
recommended_fix=(
|
|
1333
|
-
"Add or extend a test that executes the changed lines, then re-verify. "
|
|
1334
|
-
"A passing suite that does not run the new code is not acceptance evidence."
|
|
1335
|
-
),
|
|
1336
|
-
# Off by default (signal first); teams opt into blocking via
|
|
1337
|
-
# verification.diff_coverage.enforce.
|
|
1338
|
-
blocking=enforce_cov,
|
|
1339
|
-
file=first_file,
|
|
1340
|
-
line=first_lines[0] if first_lines else None,
|
|
1341
|
-
suggested_command=target_cmds[0] if target_cmds else None,
|
|
1342
|
-
))
|
|
1343
|
-
|
|
1344
|
-
# 6. Secret scan
|
|
1345
|
-
if diff_content:
|
|
1346
|
-
gaps.extend(self.secret_scanner.scan_diff(diff_content, task.id))
|
|
1347
|
-
|
|
1348
|
-
# 7. LLM Implementation Review (ADVISORY ONLY).
|
|
1349
|
-
# DevCouncil's authority is executable evidence, not model confidence — so
|
|
1350
|
-
# an LLM reviewer must never block on its own say-so. Subjective reviewers
|
|
1351
|
-
# over-flag correct code (false negatives that erode trust in "blocked"),
|
|
1352
|
-
# so review findings are surfaced as non-blocking signals. A genuine
|
|
1353
|
-
# requirement gap is caught by the acceptance-criteria evidence checks
|
|
1354
|
-
# above; the review just adds human-facing context.
|
|
1355
|
-
if review_future is not None:
|
|
1356
|
-
try:
|
|
1357
|
-
review_result = await review_future
|
|
1358
|
-
for finding in review_result.findings:
|
|
1359
|
-
finding.id = self._next_gap_id(task.id, "REVIEW")
|
|
1360
|
-
finding.blocking = False
|
|
1361
|
-
gaps.append(finding)
|
|
1362
|
-
except Exception as e:
|
|
1363
|
-
logger.error("Implementation review failed: %s", e)
|
|
1364
|
-
|
|
1365
|
-
# 8. Open live-review cards
|
|
1366
|
-
for card in unresolved_blocking_cards(self.project_root, task_id=task.id):
|
|
1367
|
-
gaps.append(Gap(
|
|
1368
|
-
id=self._next_gap_id(task.id, "LIVE"),
|
|
1369
|
-
severity="critical",
|
|
1370
|
-
gap_type="architecture_drift",
|
|
1371
|
-
task_id=task.id,
|
|
1372
|
-
description=f"Open critical live-review card remains: {card.summary}",
|
|
1373
|
-
evidence=[card.id, card.message_for_agent],
|
|
1374
|
-
recommended_fix=(
|
|
1375
|
-
f"Address the critique card, then run `dev watch resolve {card.id}` "
|
|
1376
|
-
"or mark it ignored with justification outside the verification gate."
|
|
1377
|
-
),
|
|
1378
|
-
blocking=True,
|
|
1379
|
-
))
|
|
1380
|
-
|
|
1381
|
-
self.last_outcome = VerificationOutcome(
|
|
1382
|
-
mode="compiled" if self.acceptance_compiler else "coarse",
|
|
1383
|
-
compiler_active=compiler_active,
|
|
1384
|
-
diff_empty=diff_empty,
|
|
1385
|
-
coverage_measured=coverage_measured,
|
|
1386
|
-
coverage_skipped_reason=coverage_skipped_reason,
|
|
1387
|
-
)
|
|
1388
|
-
return gaps, evidence_to_save
|
|
1389
271
|
finally:
|
|
1390
|
-
|
|
1391
|
-
# before their await points) so neither is destroyed-while-pending nor
|
|
1392
|
-
# logs 'exception never retrieved', and clear the per-call memos so a
|
|
1393
|
-
# later non-verify_task call on this instance recomputes fresh.
|
|
1394
|
-
for _fut in (compile_future, review_future):
|
|
1395
|
-
if _fut is not None:
|
|
1396
|
-
if not _fut.done():
|
|
1397
|
-
_fut.cancel()
|
|
1398
|
-
try:
|
|
1399
|
-
await _fut
|
|
1400
|
-
except (asyncio.CancelledError, Exception):
|
|
1401
|
-
pass
|
|
1402
|
-
self._untracked_cache = None
|
|
1403
|
-
self._command_timeout_cache = None
|
|
1404
|
-
# Reload project dependencies next run: a reused Verifier may verify a later
|
|
1405
|
-
# task after pyproject/requirements changed on disk.
|
|
1406
|
-
self._project_deps_cache = None
|
|
272
|
+
await cleanup_verify_futures(self, compile_future, review_future)
|
|
1407
273
|
|
|
1408
274
|
def _task_intent_text(self, task: Task, requirements: Optional[List[Requirement]]) -> str:
|
|
1409
|
-
|
|
1410
|
-
description, and the descriptions of its acceptance criteria. Used to tell an
|
|
1411
|
-
INTENDED public-API change ("remove deprecated foo") from silent drift."""
|
|
1412
|
-
parts = [task.title or "", task.description or ""]
|
|
1413
|
-
if requirements:
|
|
1414
|
-
ac_ids = set(task.acceptance_criterion_ids)
|
|
1415
|
-
for req in requirements:
|
|
1416
|
-
for ac in req.acceptance_criteria:
|
|
1417
|
-
if ac.id in ac_ids:
|
|
1418
|
-
parts.append(ac.description or "")
|
|
1419
|
-
return " ".join(parts).lower()
|
|
275
|
+
return task_intent_text(task, requirements)
|
|
1420
276
|
|
|
1421
277
|
def _check_semantic_diff(self, task: Task, requirements: Optional[List[Requirement]] = None) -> List[Gap]:
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
return gaps
|
|
1434
|
-
|
|
1435
|
-
planned_paths = {pf.path for pf in task.planned_files}
|
|
1436
|
-
classifications = result.get("classifications", [])
|
|
1437
|
-
# Drift signal inputs: a public symbol re-added elsewhere is a move/rename (a
|
|
1438
|
-
# legitimate refactor, not drift); and the task's own intent text lets a removal
|
|
1439
|
-
# the task actually asked for ("remove deprecated foo") pass without false-blocking.
|
|
1440
|
-
readded_public = {
|
|
1441
|
-
item.get("name") for item in classifications
|
|
1442
|
-
if item.get("type") == "exported_symbol_added" and item.get("name")
|
|
1443
|
-
}
|
|
1444
|
-
intent_text = self._task_intent_text(task, requirements)
|
|
1445
|
-
for item in classifications:
|
|
1446
|
-
change_type = item.get("type", "")
|
|
1447
|
-
path = item.get("path", "")
|
|
1448
|
-
if change_type == "exported_symbol_removed":
|
|
1449
|
-
# An executor deleting/renaming an existing PUBLIC symbol — even inside a
|
|
1450
|
-
# file it is allowed to touch — is scope drift / a regression the focused
|
|
1451
|
-
# task rarely intends. Block it UNLESS the symbol was re-added elsewhere
|
|
1452
|
-
# (a move/rename) or the task text explicitly calls for the removal.
|
|
1453
|
-
name = item.get("name", "")
|
|
1454
|
-
moved = name in readded_public
|
|
1455
|
-
intended = bool(name) and name.lower() in intent_text
|
|
1456
|
-
gaps.append(Gap(
|
|
1457
|
-
id=self._next_gap_id(task.id, "DRIFT"),
|
|
1458
|
-
severity="high",
|
|
1459
|
-
gap_type="architecture_drift",
|
|
1460
|
-
task_id=task.id,
|
|
1461
|
-
description=(
|
|
1462
|
-
f"Public symbol '{name}' was removed from {path} — possible scope "
|
|
1463
|
-
"drift: the executor changed a public API the task did not call for."
|
|
1464
|
-
),
|
|
1465
|
-
evidence=[f"{path}:{name}"],
|
|
1466
|
-
recommended_fix=(
|
|
1467
|
-
"Restore the removed public symbol. If its removal IS part of this "
|
|
1468
|
-
"task, state that in the task description / acceptance criteria so the "
|
|
1469
|
-
"change is an intended, reviewed decision rather than silent drift."
|
|
1470
|
-
),
|
|
1471
|
-
blocking=(not moved and not intended),
|
|
1472
|
-
file=path,
|
|
1473
|
-
))
|
|
1474
|
-
elif change_type == "public_api_change" and path not in planned_paths:
|
|
1475
|
-
gaps.append(Gap(
|
|
1476
|
-
id=self._next_gap_id(task.id, "SEM"),
|
|
1477
|
-
severity="high",
|
|
1478
|
-
gap_type="architecture_drift",
|
|
1479
|
-
task_id=task.id,
|
|
1480
|
-
description=f"Unplanned public API change detected in {path}.",
|
|
1481
|
-
evidence=[path],
|
|
1482
|
-
recommended_fix="Add file to planned_files and document acceptance criteria.",
|
|
1483
|
-
blocking=not bool(task.acceptance_criterion_ids),
|
|
1484
|
-
))
|
|
1485
|
-
elif change_type == "public_api_change" and path in planned_paths:
|
|
1486
|
-
# The file is in scope, but the executor changed the SIGNATURE of an
|
|
1487
|
-
# existing public symbol. Tasks legitimately change signatures of files
|
|
1488
|
-
# they own, so this is ADVISORY only — surfaced so an audit/agent can see
|
|
1489
|
-
# the public contract moved, not silently drifted.
|
|
1490
|
-
gaps.append(Gap(
|
|
1491
|
-
id=self._next_gap_id(task.id, "SIGDRIFT"),
|
|
1492
|
-
severity="medium",
|
|
1493
|
-
gap_type="architecture_drift",
|
|
1494
|
-
task_id=task.id,
|
|
1495
|
-
description=(
|
|
1496
|
-
f"Public API signature change in planned file {path}"
|
|
1497
|
-
+ (f" ({item.get('name')})" if item.get("name") else "")
|
|
1498
|
-
+ ". Confirm callers are updated and the change is intended."
|
|
1499
|
-
),
|
|
1500
|
-
evidence=[f"{path}:{item.get('name', '')}"],
|
|
1501
|
-
recommended_fix=(
|
|
1502
|
-
"If the signature change is part of this task, note it in the task "
|
|
1503
|
-
"description / acceptance criteria; otherwise revert it."
|
|
1504
|
-
),
|
|
1505
|
-
blocking=False,
|
|
1506
|
-
))
|
|
1507
|
-
elif change_type == "import_dependency_change":
|
|
1508
|
-
# A NEW third-party top-level package added to the diff is supply-chain
|
|
1509
|
-
# drift — block it. Everything else (stdlib, relative/local, or an
|
|
1510
|
-
# already-declared/available dependency) stays advisory, and only on an
|
|
1511
|
-
# unplanned file (an unplanned file is already orphan-blocked anyway).
|
|
1512
|
-
statement = item.get("statement", "")
|
|
1513
|
-
top = self._import_top_level(statement)
|
|
1514
|
-
new_third_party = self._is_new_third_party_import(top)
|
|
1515
|
-
if new_third_party:
|
|
1516
|
-
gaps.append(Gap(
|
|
1517
|
-
id=self._next_gap_id(task.id, "DEPADD"),
|
|
1518
|
-
severity="high",
|
|
1519
|
-
gap_type="dependency_risk",
|
|
1520
|
-
task_id=task.id,
|
|
1521
|
-
description=(
|
|
1522
|
-
f"New undeclared third-party dependency '{top}' imported in {path} "
|
|
1523
|
-
f"({statement.strip()}). Adding a dependency the task did not plan is "
|
|
1524
|
-
"supply-chain drift."
|
|
1525
|
-
),
|
|
1526
|
-
evidence=[path, statement.strip()],
|
|
1527
|
-
recommended_fix=(
|
|
1528
|
-
f"Declare '{top}' in the project's dependencies and plan the change, "
|
|
1529
|
-
"or use an existing/standard-library alternative."
|
|
1530
|
-
),
|
|
1531
|
-
blocking=True,
|
|
1532
|
-
file=path,
|
|
1533
|
-
))
|
|
1534
|
-
elif path not in planned_paths:
|
|
1535
|
-
gaps.append(Gap(
|
|
1536
|
-
id=self._next_gap_id(task.id, "IMP"),
|
|
1537
|
-
severity="medium",
|
|
1538
|
-
gap_type="dependency_risk",
|
|
1539
|
-
task_id=task.id,
|
|
1540
|
-
description=f"Import dependency change in {path}.",
|
|
1541
|
-
evidence=[path],
|
|
1542
|
-
recommended_fix="Confirm dependency change is intentional.",
|
|
1543
|
-
blocking=False,
|
|
1544
|
-
))
|
|
1545
|
-
elif change_type == "config_schema_dependency_change" and path not in planned_paths:
|
|
1546
|
-
gaps.append(Gap(
|
|
1547
|
-
id=self._next_gap_id(task.id, "CFG"),
|
|
1548
|
-
severity="high",
|
|
1549
|
-
gap_type="dependency_risk",
|
|
1550
|
-
task_id=task.id,
|
|
1551
|
-
description=f"Config/schema change detected in {path}.",
|
|
1552
|
-
evidence=[path],
|
|
1553
|
-
recommended_fix="Plan the config change or revert it.",
|
|
1554
|
-
blocking=True,
|
|
1555
|
-
))
|
|
1556
|
-
return gaps
|
|
278
|
+
cached = getattr(self, "_project_deps_cache", None)
|
|
279
|
+
if cached is None:
|
|
280
|
+
cached = load_project_dependencies(self.project_root)
|
|
281
|
+
self._project_deps_cache = cached
|
|
282
|
+
return detect_semantic_diff_gaps(
|
|
283
|
+
project_root=self.project_root,
|
|
284
|
+
task=task,
|
|
285
|
+
requirements=requirements,
|
|
286
|
+
next_gap_id=self._next_gap_id,
|
|
287
|
+
project_deps=cached,
|
|
288
|
+
)
|
|
1557
289
|
|
|
1558
290
|
@staticmethod
|
|
1559
291
|
def _import_top_level(statement: str) -> Optional[str]:
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
``import requests`` / ``import os.path`` -> the first dotted component; ``from x.y
|
|
1563
|
-
import z`` -> ``x``; ``from . import z`` / ``from .mod import z`` -> None (relative).
|
|
1564
|
-
"""
|
|
1565
|
-
s = (statement or "").strip()
|
|
1566
|
-
if s.startswith("import "):
|
|
1567
|
-
first = s[len("import "):].split(",")[0].strip()
|
|
1568
|
-
top = first.split(" as ")[0].strip().split(".")[0].strip()
|
|
1569
|
-
return top or None
|
|
1570
|
-
if s.startswith("from "):
|
|
1571
|
-
rest = s[len("from "):].lstrip()
|
|
1572
|
-
if rest.startswith("."): # relative import -> local, never a new dependency
|
|
1573
|
-
return None
|
|
1574
|
-
mod = rest.split(" import ")[0].strip()
|
|
1575
|
-
return (mod.split(".")[0].strip() or None) if mod else None
|
|
1576
|
-
return None
|
|
292
|
+
return import_top_level(statement)
|
|
1577
293
|
|
|
1578
294
|
def _is_new_third_party_import(self, top: Optional[str]) -> bool:
|
|
1579
|
-
|
|
295
|
+
return is_new_third_party_import(top, project_deps=self._project_dependencies())
|
|
1580
296
|
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
never false-block). Only a package that is none of those — i.e. undeclared AND not
|
|
1585
|
-
present — counts as supply-chain drift."""
|
|
1586
|
-
if not top:
|
|
1587
|
-
return False
|
|
1588
|
-
if top in self._stdlib_modules():
|
|
1589
|
-
return False
|
|
1590
|
-
if top.lower() in self._project_dependencies():
|
|
1591
|
-
return False
|
|
1592
|
-
try:
|
|
1593
|
-
import importlib.util
|
|
1594
|
-
if importlib.util.find_spec(top) is not None:
|
|
1595
|
-
return False # already available in the environment; not a new dependency
|
|
1596
|
-
except Exception:
|
|
1597
|
-
# A find_spec error (e.g. a partially-installed parent) is ambiguous; do not
|
|
1598
|
-
# block on ambiguity.
|
|
1599
|
-
return False
|
|
1600
|
-
return True
|
|
1601
|
-
|
|
1602
|
-
@staticmethod
|
|
1603
|
-
def _stdlib_modules() -> frozenset:
|
|
1604
|
-
names = getattr(sys, "stdlib_module_names", None)
|
|
1605
|
-
return frozenset(names) if names else frozenset()
|
|
1606
|
-
|
|
1607
|
-
def _project_dependencies(self) -> set:
|
|
1608
|
-
"""Lower-cased distribution names declared by the project (pyproject/requirements/
|
|
1609
|
-
package.json). Cached per Verifier instance; best-effort (parse errors are ignored)."""
|
|
1610
|
-
cached = getattr(self, "_project_deps_cache", None)
|
|
297
|
+
def _project_dependencies(self) -> set[str]:
|
|
298
|
+
"""Lower-cased distribution names declared by the project."""
|
|
299
|
+
cached: set[str] | None = getattr(self, "_project_deps_cache", None)
|
|
1611
300
|
if cached is not None:
|
|
1612
301
|
return cached
|
|
1613
|
-
deps
|
|
1614
|
-
split_re = r"[><=!~;\[\] ]"
|
|
1615
|
-
pyproject = self.project_root / "pyproject.toml"
|
|
1616
|
-
if pyproject.exists():
|
|
1617
|
-
try:
|
|
1618
|
-
import tomllib
|
|
1619
|
-
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
|
|
1620
|
-
project = data.get("project", {}) or {}
|
|
1621
|
-
for dep in project.get("dependencies", []) or []:
|
|
1622
|
-
pkg = re.split(split_re, dep.strip())[0].strip().lower()
|
|
1623
|
-
if pkg:
|
|
1624
|
-
deps.add(pkg)
|
|
1625
|
-
for group in (project.get("optional-dependencies", {}) or {}).values():
|
|
1626
|
-
for dep in group or []:
|
|
1627
|
-
pkg = re.split(split_re, dep.strip())[0].strip().lower()
|
|
1628
|
-
if pkg:
|
|
1629
|
-
deps.add(pkg)
|
|
1630
|
-
except Exception:
|
|
1631
|
-
pass
|
|
1632
|
-
requirements = self.project_root / "requirements.txt"
|
|
1633
|
-
if requirements.exists():
|
|
1634
|
-
try:
|
|
1635
|
-
for line in requirements.read_text(encoding="utf-8").splitlines():
|
|
1636
|
-
line = line.strip()
|
|
1637
|
-
if line and not line.startswith("#"):
|
|
1638
|
-
pkg = re.split(split_re, line)[0].strip().lower()
|
|
1639
|
-
if pkg:
|
|
1640
|
-
deps.add(pkg)
|
|
1641
|
-
except Exception:
|
|
1642
|
-
pass
|
|
1643
|
-
package_json = self.project_root / "package.json"
|
|
1644
|
-
if package_json.exists():
|
|
1645
|
-
try:
|
|
1646
|
-
data = json.loads(package_json.read_text(encoding="utf-8"))
|
|
1647
|
-
for key in ("dependencies", "devDependencies", "optionalDependencies"):
|
|
1648
|
-
deps.update(k.lower() for k in (data.get(key) or {}).keys())
|
|
1649
|
-
except Exception:
|
|
1650
|
-
pass
|
|
302
|
+
deps = load_project_dependencies(self.project_root)
|
|
1651
303
|
self._project_deps_cache = deps
|
|
1652
304
|
return deps
|
|
1653
305
|
|
|
1654
|
-
# Signatures that mean the verification command itself could not run (or had
|
|
1655
|
-
# nothing to run), so its non-zero exit says nothing about whether the
|
|
1656
|
-
# implementation is correct — a tooling/plan defect, not a code defect.
|
|
1657
|
-
_MALFORMED_COMMAND_SIGNATURES = (
|
|
1658
|
-
"syntaxerror",
|
|
1659
|
-
"invalid syntax",
|
|
1660
|
-
"indentationerror",
|
|
1661
|
-
"no module named", # any tool not installed (pytest, flake8, mypy, ...)
|
|
1662
|
-
"can't open file",
|
|
1663
|
-
"no such file or directory",
|
|
1664
|
-
"file or directory not found", # pytest: target path missing
|
|
1665
|
-
"no tests ran", # pytest -k matched nothing / empty file
|
|
1666
|
-
"no tests collected",
|
|
1667
|
-
"error: not found", # pytest: test node id does not exist
|
|
1668
|
-
"is not recognized as an internal or external command",
|
|
1669
|
-
"command not found",
|
|
1670
|
-
"executable file not found",
|
|
1671
|
-
"failed to run command",
|
|
1672
|
-
"importerror", # the verification harness itself failed to import
|
|
1673
|
-
"modulenotfounderror",
|
|
1674
|
-
)
|
|
1675
|
-
# Compile-/launch-time signatures that mean the code NEVER executed — these are
|
|
1676
|
-
# always authoritative regardless of any ``File "<string>", line N`` marker (a
|
|
1677
|
-
# SyntaxError prints that marker even though nothing ran). They must not be subject
|
|
1678
|
-
# to the "signature must precede a traceback frame" rule that distinguishes a real
|
|
1679
|
-
# in-test traceback from a launcher error.
|
|
1680
|
-
_UNCONDITIONAL_UNRUNNABLE_SIGNATURES = (
|
|
1681
|
-
"syntaxerror",
|
|
1682
|
-
"invalid syntax",
|
|
1683
|
-
"indentationerror",
|
|
1684
|
-
"can't open file",
|
|
1685
|
-
"is not recognized as an internal or external command",
|
|
1686
|
-
"command not found",
|
|
1687
|
-
"executable file not found",
|
|
1688
|
-
"failed to run command",
|
|
1689
|
-
"no tests ran",
|
|
1690
|
-
"no tests collected",
|
|
1691
|
-
"error: not found",
|
|
1692
|
-
)
|
|
1693
|
-
# pytest exit codes that mean "could not run / collect", not "tests failed":
|
|
1694
|
-
# 4 = usage/collection error, 5 = no tests collected.
|
|
1695
|
-
_PYTEST_NONRUN_EXIT_CODES = {4, 5}
|
|
1696
|
-
|
|
1697
|
-
@staticmethod
|
|
1698
|
-
def _is_traceback_frame(line: str) -> bool:
|
|
1699
|
-
"""True for a Python traceback frame line: `` File "...", line N``."""
|
|
1700
|
-
stripped = line.strip()
|
|
1701
|
-
return stripped.startswith('File "') and ", line " in stripped
|
|
1702
|
-
|
|
1703
306
|
def _malformed_signature_precedes_traceback(self, text: str) -> bool:
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
A launcher/collection failure prints its error WITHOUT a Python traceback that
|
|
1707
|
-
executed the code under test (e.g. ``ModuleNotFoundError: No module named
|
|
1708
|
-
pytest`` straight from the interpreter, or pytest's collection error banner).
|
|
1709
|
-
A genuine in-test failure, by contrast, raises from inside a traceback whose
|
|
1710
|
-
frames point at the test/source files; the same signature words can appear
|
|
1711
|
-
there (``ImportError`` re-raised inside a test) but that is a real defect, not
|
|
1712
|
-
an unrunnable command.
|
|
1713
|
-
|
|
1714
|
-
So a signature only proves "unrunnable" when it appears BEFORE the first
|
|
1715
|
-
traceback frame (or there is no traceback frame at all). If a traceback frame
|
|
1716
|
-
appears at or before the signature, the code under test ran and failed — keep
|
|
1717
|
-
it a blocking test failure."""
|
|
1718
|
-
if not text:
|
|
1719
|
-
return False
|
|
1720
|
-
low_all = text.lower()
|
|
1721
|
-
# Compile-/launch-time failures: the code never executed, so a ``File ...``
|
|
1722
|
-
# marker (printed by SyntaxError) is not a real frame. Authoritative outright.
|
|
1723
|
-
if any(sig in low_all for sig in self._UNCONDITIONAL_UNRUNNABLE_SIGNATURES):
|
|
1724
|
-
return True
|
|
1725
|
-
lines = text.splitlines()
|
|
1726
|
-
lowered_lines = [ln.lower() for ln in lines]
|
|
1727
|
-
first_frame_idx: Optional[int] = None
|
|
1728
|
-
for idx, line in enumerate(lines):
|
|
1729
|
-
if self._is_traceback_frame(line):
|
|
1730
|
-
first_frame_idx = idx
|
|
1731
|
-
break
|
|
1732
|
-
for idx, low in enumerate(lowered_lines):
|
|
1733
|
-
if any(sig in low for sig in self._MALFORMED_COMMAND_SIGNATURES):
|
|
1734
|
-
# Signature found; it is only authoritative if no traceback frame
|
|
1735
|
-
# precedes it (i.e. the failure is from the launcher, not from code
|
|
1736
|
-
# that actually executed under a traceback).
|
|
1737
|
-
if first_frame_idx is None or idx < first_frame_idx:
|
|
1738
|
-
return True
|
|
1739
|
-
return False
|
|
1740
|
-
return False
|
|
307
|
+
return cmd_malf.malformed_signature_precedes_traceback(text)
|
|
1741
308
|
|
|
1742
309
|
def _launcher_text(self, result: CommandResult) -> str:
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
The traceback-precedence discriminator
|
|
1746
|
-
(:meth:`_malformed_signature_precedes_traceback`) needs to see BOTH streams:
|
|
1747
|
-
an interpreter "cannot run" error lands on stderr (with no traceback frame),
|
|
1748
|
-
while a genuine in-test failure's traceback lands on stdout (frame first, then
|
|
1749
|
-
the exception). We therefore concatenate stderr+stdout so the relative ordering
|
|
1750
|
-
of any signature vs the first traceback frame is preserved.
|
|
1751
|
-
|
|
1752
|
-
Reading the merged ``result.summary`` alone is unsafe: it hoists the salient
|
|
1753
|
-
error line to the FRONT, which would place an in-test ``ImportError`` before its
|
|
1754
|
-
own traceback frame and misclassify a real failure as unrunnable. So prefer the
|
|
1755
|
-
raw logs; only fall back to the summary when no log path is available (e.g. unit
|
|
1756
|
-
tests that stub ``_run_command``). Never raises."""
|
|
1757
|
-
parts: List[str] = []
|
|
1758
|
-
for path in (result.stderr_path, result.stdout_path):
|
|
1759
|
-
if not path:
|
|
1760
|
-
continue
|
|
1761
|
-
try:
|
|
1762
|
-
content = Path(path).read_text(encoding="utf-8", errors="replace")
|
|
1763
|
-
if content.strip():
|
|
1764
|
-
parts.append(content)
|
|
1765
|
-
except Exception:
|
|
1766
|
-
pass
|
|
1767
|
-
if parts:
|
|
1768
|
-
return "\n".join(parts)
|
|
1769
|
-
return result.summary or ""
|
|
1770
|
-
|
|
1771
|
-
# Matches a Python traceback frame: `` File "path/to/x.py", line 42, in foo``.
|
|
1772
|
-
_TRACEBACK_FRAME_RE = re.compile(r'File "(?P<file>[^"]+)", line (?P<line>\d+)')
|
|
310
|
+
return cmd_malf.launcher_text(result)
|
|
1773
311
|
|
|
1774
312
|
def _failure_location(self, result: CommandResult) -> Tuple[Optional[str], Optional[int]]:
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
The LAST frame in a Python traceback is the actual raise site, so we scan all
|
|
1778
|
-
frames and keep the last one that points at a real-looking source file (not the
|
|
1779
|
-
``<string>`` of a ``python -c`` snippet). Returns repo-relative posix paths when
|
|
1780
|
-
the frame is inside the project root. Reads the captured logs (stdout has the
|
|
1781
|
-
test traceback; stderr has interpreter errors). Never raises."""
|
|
1782
|
-
sources = []
|
|
1783
|
-
for path in (result.stdout_path, result.stderr_path):
|
|
1784
|
-
if path:
|
|
1785
|
-
try:
|
|
1786
|
-
content = Path(path).read_text(encoding="utf-8", errors="replace")
|
|
1787
|
-
if content.strip():
|
|
1788
|
-
sources.append(content)
|
|
1789
|
-
except Exception:
|
|
1790
|
-
pass
|
|
1791
|
-
sources.append(result.summary or "")
|
|
1792
|
-
best_file: Optional[str] = None
|
|
1793
|
-
best_line: Optional[int] = None
|
|
1794
|
-
for text in sources:
|
|
1795
|
-
for match in self._TRACEBACK_FRAME_RE.finditer(text):
|
|
1796
|
-
raw_file = match.group("file")
|
|
1797
|
-
if not raw_file or raw_file.startswith("<"):
|
|
1798
|
-
continue # e.g. "<string>" from python -c
|
|
1799
|
-
best_file = self._relativize(raw_file)
|
|
1800
|
-
try:
|
|
1801
|
-
best_line = int(match.group("line"))
|
|
1802
|
-
except ValueError:
|
|
1803
|
-
best_line = None
|
|
1804
|
-
if best_file is not None:
|
|
1805
|
-
return best_file, best_line
|
|
1806
|
-
return best_file, best_line
|
|
1807
|
-
|
|
1808
|
-
def _relativize(self, raw_path: str) -> str:
|
|
1809
|
-
"""Normalize a traceback file path to a repo-relative posix path when possible."""
|
|
1810
|
-
normalized = raw_path.replace("\\", "/")
|
|
1811
|
-
try:
|
|
1812
|
-
candidate = Path(raw_path)
|
|
1813
|
-
if candidate.is_absolute():
|
|
1814
|
-
rel = candidate.resolve().relative_to(self.project_root.resolve())
|
|
1815
|
-
return rel.as_posix()
|
|
1816
|
-
except Exception:
|
|
1817
|
-
pass
|
|
1818
|
-
return normalized
|
|
313
|
+
return cmd_malf.failure_location(self.project_root, result)
|
|
1819
314
|
|
|
1820
315
|
def _command_is_malformed(self, result: CommandResult) -> bool:
|
|
1821
|
-
|
|
1822
|
-
than a genuine assertion or test failure of the code under verification.
|
|
1823
|
-
|
|
1824
|
-
Authoritative signals (in priority order):
|
|
1825
|
-
1. pytest exit 4/5 -> collection/usage error -> unrunnable.
|
|
1826
|
-
2. The launcher error text: an unrunnable signature only counts when it
|
|
1827
|
-
appears BEFORE any Python traceback frame. This stops a genuinely failing
|
|
1828
|
-
test whose traceback contains ``ImportError``/``ModuleNotFoundError`` from
|
|
1829
|
-
being downgraded to a non-blocking "invalid command" (which would let
|
|
1830
|
-
verification falsely PASS)."""
|
|
1831
|
-
is_pytest = "pytest" in (result.command or "")
|
|
1832
|
-
if is_pytest and result.exit_code in self._PYTEST_NONRUN_EXIT_CODES:
|
|
1833
|
-
return True
|
|
1834
|
-
# Otherwise the exit code alone is ambiguous: pytest exit 1 is "tests ran and
|
|
1835
|
-
# FAILED" (a real defect), but a missing pytest module also exits 1 from the
|
|
1836
|
-
# interpreter (``No module named pytest``). The launcher error text is the
|
|
1837
|
-
# authoritative discriminator — a signature only means "unrunnable" when it
|
|
1838
|
-
# appears BEFORE any Python traceback frame. A genuine test failure whose
|
|
1839
|
-
# traceback merely mentions ``ImportError`` keeps a traceback frame first and so
|
|
1840
|
-
# stays a blocking test failure (preventing a false PASS).
|
|
1841
|
-
text = self._launcher_text(result)
|
|
1842
|
-
return self._malformed_signature_precedes_traceback(text)
|
|
316
|
+
return cmd_malf.command_is_malformed(result)
|
|
1843
317
|
|
|
1844
318
|
def _commands_for_task(self, task: Task) -> Dict[str, List[str]]:
|
|
1845
319
|
if task.expected_tests:
|
|
@@ -1871,6 +345,11 @@ class Verifier:
|
|
|
1871
345
|
return False, f"command targets the '{stack}' stack not present in this repo (detected: {detected})"
|
|
1872
346
|
return True, ""
|
|
1873
347
|
|
|
348
|
+
@staticmethod
|
|
349
|
+
def _is_test_path(path: str) -> bool:
|
|
350
|
+
from devcouncil.verification.checks.orphan_diff import is_test_path
|
|
351
|
+
return is_test_path(path)
|
|
352
|
+
|
|
1874
353
|
# Linters / formatters / type checkers: a non-zero exit is a style/type OPINION,
|
|
1875
354
|
# not proof of a behavioral defect. Blocking a behaviorally-correct task on these is
|
|
1876
355
|
# the false-block the benchmark surfaced (the planner even spawns dedicated
|
|
@@ -1908,26 +387,18 @@ class Verifier:
|
|
|
1908
387
|
return tool in self._QUALITY_TOOLS
|
|
1909
388
|
|
|
1910
389
|
def _command_can_prove_acceptance(self, cmd_type: str, command: str) -> bool:
|
|
390
|
+
"""Whether a run of ``command`` may count as acceptance evidence.
|
|
391
|
+
|
|
392
|
+
Declared TEST commands (planner expected_tests / config commands.test) are
|
|
393
|
+
trusted unless trivially incapable of proving behavior (``python --version``,
|
|
394
|
+
``echo ok``, ``git status``) — the deny-list keeps legitimate keyword-less
|
|
395
|
+
behavioral commands (``make check``, ``./run_smoke.sh``) evidential. Agent-
|
|
396
|
+
appended expected_tests are additionally excluded at coarse-proof time.
|
|
397
|
+
Other command types (allowed_commands) keep the strict keyword allowlist:
|
|
398
|
+
an incidental ``make build`` succeeding must not prove a behavioral AC."""
|
|
1911
399
|
if cmd_type == "test":
|
|
1912
|
-
return
|
|
1913
|
-
|
|
1914
|
-
evidence_keywords = (
|
|
1915
|
-
"test",
|
|
1916
|
-
"pytest",
|
|
1917
|
-
"vitest",
|
|
1918
|
-
"jest",
|
|
1919
|
-
"unittest",
|
|
1920
|
-
"cargo test",
|
|
1921
|
-
"go test",
|
|
1922
|
-
"mvn test",
|
|
1923
|
-
"gradle test",
|
|
1924
|
-
"ruff check",
|
|
1925
|
-
"mypy",
|
|
1926
|
-
"tsc",
|
|
1927
|
-
"typecheck",
|
|
1928
|
-
"type-check",
|
|
1929
|
-
)
|
|
1930
|
-
return any(keyword in lowered for keyword in evidence_keywords)
|
|
400
|
+
return not command_is_trivial_evidence(command)
|
|
401
|
+
return command_has_acceptance_evidence(command)
|
|
1931
402
|
|
|
1932
403
|
def _requirement_id_for_ac(self, requirements: List[Requirement], ac_id: str) -> Optional[str]:
|
|
1933
404
|
for req in requirements:
|