devcouncil 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -30
- package/package.json +6 -2
- package/packages/codeintel-grammars/hatch_build.py +43 -0
- package/packages/codeintel-grammars/pyproject.toml +16 -0
- package/packages/codeintel-grammars/src/devcouncil_codeintel_grammars/__init__.py +93 -0
- package/pyproject.toml +99 -4
- package/src/devcouncil/app/config.py +512 -20
- package/src/devcouncil/app/events.py +4 -23
- package/src/devcouncil/app/orchestrator.py +5 -0
- package/src/devcouncil/app/run_context.py +3 -3
- package/src/devcouncil/assets/__init__.py +4 -1
- package/src/devcouncil/assets/vendor/force-graph.min.js +5 -0
- package/src/devcouncil/campaign/__init__.py +71 -0
- package/src/devcouncil/campaign/bloom.py +137 -0
- package/src/devcouncil/campaign/dashboard.py +123 -0
- package/src/devcouncil/campaign/mailbox.py +305 -0
- package/src/devcouncil/campaign/notify.py +91 -0
- package/src/devcouncil/campaign/orchestrator.py +592 -0
- package/src/devcouncil/campaign/prompts/coordinator.md +29 -0
- package/src/devcouncil/campaign/prompts/director.md +21 -0
- package/src/devcouncil/campaign/prompts/protocol.md +46 -0
- package/src/devcouncil/campaign/prompts/reviewer.md +24 -0
- package/src/devcouncil/campaign/prompts/worker.md +24 -0
- package/src/devcouncil/campaign/roles.py +202 -0
- package/src/devcouncil/campaign/watcher.py +153 -0
- package/src/devcouncil/cli/commands/agents.py +24 -17
- package/src/devcouncil/cli/commands/artifacts.py +36 -27
- package/src/devcouncil/cli/commands/ast.py +12 -3
- package/src/devcouncil/cli/commands/baseline.py +21 -12
- package/src/devcouncil/cli/commands/boot.py +218 -0
- package/src/devcouncil/cli/commands/campaign.py +302 -0
- package/src/devcouncil/cli/commands/check.py +225 -12
- package/src/devcouncil/cli/commands/config.py +221 -74
- package/src/devcouncil/cli/commands/cost.py +137 -28
- package/src/devcouncil/cli/commands/dashboard.py +12 -4
- package/src/devcouncil/cli/commands/debug_cmd.py +249 -0
- package/src/devcouncil/cli/commands/design.py +27 -17
- package/src/devcouncil/cli/commands/doctor.py +790 -8
- package/src/devcouncil/cli/commands/evidence.py +41 -20
- package/src/devcouncil/cli/commands/export.py +73 -0
- package/src/devcouncil/cli/commands/gaps.py +175 -0
- package/src/devcouncil/cli/commands/gated_write.py +76 -0
- package/src/devcouncil/cli/commands/go.py +220 -68
- package/src/devcouncil/cli/commands/graph_cmd.py +1192 -0
- package/src/devcouncil/cli/commands/handoff.py +45 -34
- package/src/devcouncil/cli/commands/hook.py +630 -85
- package/src/devcouncil/cli/commands/init.py +89 -30
- package/src/devcouncil/cli/commands/integrate.py +296 -1385
- package/src/devcouncil/cli/commands/lease.py +120 -0
- package/src/devcouncil/cli/commands/logs.py +12 -5
- package/src/devcouncil/cli/commands/lsp.py +40 -5
- package/src/devcouncil/cli/commands/map.py +317 -74
- package/src/devcouncil/cli/commands/mcp_server.py +12 -2
- package/src/devcouncil/cli/commands/okf.py +44 -6
- package/src/devcouncil/cli/commands/plan.py +184 -69
- package/src/devcouncil/cli/commands/prompt.py +26 -17
- package/src/devcouncil/cli/commands/provenance.py +79 -0
- package/src/devcouncil/cli/commands/repair.py +60 -49
- package/src/devcouncil/cli/commands/report.py +148 -40
- package/src/devcouncil/cli/commands/requirements.py +104 -0
- package/src/devcouncil/cli/commands/reset_demo_state.py +13 -4
- package/src/devcouncil/cli/commands/rollback.py +46 -35
- package/src/devcouncil/cli/commands/run.py +173 -8
- package/src/devcouncil/cli/commands/runs.py +298 -68
- package/src/devcouncil/cli/commands/scaffold.py +33 -12
- package/src/devcouncil/cli/commands/semantic.py +29 -14
- package/src/devcouncil/cli/commands/setup.py +103 -93
- package/src/devcouncil/cli/commands/shell.py +51 -42
- package/src/devcouncil/cli/commands/show.py +56 -42
- package/src/devcouncil/cli/commands/skills.py +29 -20
- package/src/devcouncil/cli/commands/status.py +80 -67
- package/src/devcouncil/cli/commands/task_gate.py +295 -0
- package/src/devcouncil/cli/commands/tasks.py +248 -19
- package/src/devcouncil/cli/commands/trace.py +14 -8
- package/src/devcouncil/cli/commands/verify.py +33 -8
- package/src/devcouncil/cli/commands/version.py +14 -6
- package/src/devcouncil/cli/commands/watch.py +49 -30
- package/src/devcouncil/cli/commands/watch_fs.py +30 -19
- package/src/devcouncil/cli/commands/wiki.py +278 -0
- package/src/devcouncil/cli/main.py +58 -1
- package/src/devcouncil/codeintel/__init__.py +16 -0
- package/src/devcouncil/codeintel/build_control.py +429 -0
- package/src/devcouncil/codeintel/build_worker.py +78 -0
- package/src/devcouncil/codeintel/debug/__init__.py +17 -0
- package/src/devcouncil/codeintel/debug/broker.py +114 -0
- package/src/devcouncil/codeintel/debug/broker_client.py +61 -0
- package/src/devcouncil/codeintel/debug/consent.py +36 -0
- package/src/devcouncil/codeintel/debug/discovery.py +132 -0
- package/src/devcouncil/codeintel/debug/fingerprint.py +85 -0
- package/src/devcouncil/codeintel/debug/protocol.py +259 -0
- package/src/devcouncil/codeintel/debug/python_trace_runner.py +81 -0
- package/src/devcouncil/codeintel/debug/session.py +238 -0
- package/src/devcouncil/codeintel/debug/tracing.py +201 -0
- package/src/devcouncil/codeintel/languages/__init__.py +17 -0
- package/src/devcouncil/codeintel/languages/generic_extractor.py +236 -0
- package/src/devcouncil/codeintel/languages/registry.py +149 -0
- package/src/devcouncil/codeintel/languages/workers.py +245 -0
- package/src/devcouncil/codeintel/query/__init__.py +5 -0
- package/src/devcouncil/codeintel/query/engine.py +289 -0
- package/src/devcouncil/codeintel/resolution/__init__.py +6 -0
- package/src/devcouncil/codeintel/resolution/abstract_state.py +301 -0
- package/src/devcouncil/codeintel/resolution/frameworks/__init__.py +33 -0
- package/src/devcouncil/codeintel/resolution/frameworks/base.py +46 -0
- package/src/devcouncil/codeintel/resolution/frameworks/di.py +56 -0
- package/src/devcouncil/codeintel/resolution/frameworks/events.py +45 -0
- package/src/devcouncil/codeintel/resolution/frameworks/routes.py +88 -0
- package/src/devcouncil/codeintel/resolution/semantic.py +887 -0
- package/src/devcouncil/codeintel/service.py +104 -0
- package/src/devcouncil/codeintel/store/__init__.py +15 -0
- package/src/devcouncil/codeintel/store/sqlite.py +1565 -0
- package/src/devcouncil/codeintel/sync/__init__.py +19 -0
- package/src/devcouncil/codeintel/sync/coordinator.py +430 -0
- package/src/devcouncil/codeintel/sync/incremental.py +484 -0
- package/src/devcouncil/codeintel/sync/lease.py +96 -0
- package/src/devcouncil/codeintel/sync/scope.py +98 -0
- package/src/devcouncil/council/__init__.py +4 -0
- package/src/devcouncil/council/prompts/__init__.py +4 -0
- package/src/devcouncil/domain/checkpoint_refs.py +17 -0
- package/src/devcouncil/domain/evidence.py +1 -0
- package/src/devcouncil/domain/gap.py +10 -0
- package/src/devcouncil/domain/requirement.py +5 -1
- package/src/devcouncil/domain/task.py +43 -2
- package/src/devcouncil/execution/checkpoints.py +25 -31
- package/src/devcouncil/execution/context_builder.py +15 -44
- package/src/devcouncil/execution/fs_watcher.py +64 -0
- package/src/devcouncil/execution/gated_write.py +203 -0
- package/src/devcouncil/execution/handoff.py +2 -1
- package/src/devcouncil/execution/hook_policy.py +19 -5
- package/src/devcouncil/execution/lease_ops.py +177 -0
- package/src/devcouncil/execution/lease_validation.py +71 -0
- package/src/devcouncil/execution/patch.py +3 -0
- package/src/devcouncil/execution/permissions.py +1 -0
- package/src/devcouncil/execution/policy_engine.py +205 -10
- package/src/devcouncil/execution/prompt_builder.py +278 -33
- package/src/devcouncil/execution/run_trace.py +356 -0
- package/src/devcouncil/execution/shell_session.py +46 -5
- package/src/devcouncil/execution/stop_gate.py +746 -0
- package/src/devcouncil/execution/stop_gate_history.py +113 -0
- package/src/devcouncil/execution/stop_gate_state.py +54 -0
- package/src/devcouncil/execution/stop_gate_verify_cache.py +69 -0
- package/src/devcouncil/execution/task_gate_ops.py +590 -0
- package/src/devcouncil/execution/task_runner.py +19 -0
- package/src/devcouncil/executors/advisor_tool.py +315 -0
- package/src/devcouncil/executors/agent_registry.py +125 -17
- package/src/devcouncil/executors/claude_sdk.py +376 -0
- package/src/devcouncil/executors/coding_cli.py +724 -25
- package/src/devcouncil/executors/mini_swe.py +50 -8
- package/src/devcouncil/executors/native/agent.py +224 -19
- package/src/devcouncil/executors/openhands.py +50 -8
- package/src/devcouncil/executors/transient_retry.py +99 -0
- package/src/devcouncil/gating/checks/clean_git.py +5 -2
- package/src/devcouncil/gating/checks/planned_files_check.py +38 -11
- package/src/devcouncil/gating/checks/secret_scan_check.py +2 -2
- package/src/devcouncil/gating/policy.py +46 -2
- package/src/devcouncil/indexing/ast_matcher.py +41 -4
- package/src/devcouncil/indexing/graph/__init__.py +78 -0
- package/src/devcouncil/indexing/graph/api_routes.py +522 -0
- package/src/devcouncil/indexing/graph/build.py +862 -0
- package/src/devcouncil/indexing/graph/cache.py +329 -0
- package/src/devcouncil/indexing/graph/communities.py +28 -0
- package/src/devcouncil/indexing/graph/cypher.py +107 -0
- package/src/devcouncil/indexing/graph/embeddings.py +194 -0
- package/src/devcouncil/indexing/graph/export.py +381 -0
- package/src/devcouncil/indexing/graph/export_links.py +81 -0
- package/src/devcouncil/indexing/graph/extract_python.py +307 -0
- package/src/devcouncil/indexing/graph/extract_ts.py +1205 -0
- package/src/devcouncil/indexing/graph/intel.py +668 -0
- package/src/devcouncil/indexing/graph/liveness.py +992 -0
- package/src/devcouncil/indexing/graph/okf_export.py +65 -0
- package/src/devcouncil/indexing/graph/pdg/__init__.py +67 -0
- package/src/devcouncil/indexing/graph/pdg/build.py +11 -0
- package/src/devcouncil/indexing/graph/pdg/cdg.py +41 -0
- package/src/devcouncil/indexing/graph/pdg/cfg.py +199 -0
- package/src/devcouncil/indexing/graph/pdg/query.py +21 -0
- package/src/devcouncil/indexing/graph/pdg/reaching_def.py +126 -0
- package/src/devcouncil/indexing/graph/pdg/schema.py +253 -0
- package/src/devcouncil/indexing/graph/pdg/taint.py +154 -0
- package/src/devcouncil/indexing/graph/query.py +302 -0
- package/src/devcouncil/indexing/graph/resolve.py +1020 -0
- package/src/devcouncil/indexing/graph/schema.py +103 -0
- package/src/devcouncil/indexing/graph_index.py +20 -29
- package/src/devcouncil/indexing/lsp.py +57 -25
- package/src/devcouncil/indexing/lsp_client.py +577 -0
- package/src/devcouncil/indexing/map_artifacts.py +355 -0
- package/src/devcouncil/indexing/map_refresh.py +141 -0
- package/src/devcouncil/indexing/repo_mapper.py +1509 -138
- package/src/devcouncil/indexing/semantic_index.py +12 -6
- package/src/devcouncil/indexing/subsystem_map.py +163 -0
- package/src/devcouncil/indexing/ts_imports.py +343 -0
- package/src/devcouncil/indexing/viz.py +960 -0
- package/src/devcouncil/indexing/walk.py +52 -0
- package/src/devcouncil/indexing/wiring.py +1776 -0
- package/src/devcouncil/integrations/actions.py +27 -4
- package/src/devcouncil/integrations/check.py +211 -16
- package/src/devcouncil/integrations/claude_assets.py +209 -12
- package/src/devcouncil/integrations/clients/__init__.py +1 -0
- package/src/devcouncil/integrations/clients/aider.py +52 -0
- package/src/devcouncil/integrations/clients/antigravity.py +87 -0
- package/src/devcouncil/integrations/clients/claude.py +339 -0
- package/src/devcouncil/integrations/clients/codex.py +39 -0
- package/src/devcouncil/integrations/clients/common.py +332 -0
- package/src/devcouncil/integrations/clients/cursor.py +164 -0
- package/src/devcouncil/integrations/clients/gemini.py +49 -0
- package/src/devcouncil/integrations/clients/grok.py +105 -0
- package/src/devcouncil/integrations/clients/hooks.py +500 -0
- package/src/devcouncil/integrations/clients/opencode.py +96 -0
- package/src/devcouncil/integrations/clients/warp.py +75 -0
- package/src/devcouncil/integrations/code_review_graph.py +2 -2
- package/src/devcouncil/integrations/github.py +73 -7
- package/src/devcouncil/integrations/integration_cli.py +197 -0
- package/src/devcouncil/integrations/mcp/handlers/__init__.py +1 -0
- package/src/devcouncil/integrations/mcp/handlers/ast_lsp.py +77 -0
- package/src/devcouncil/integrations/mcp/handlers/checkout.py +50 -0
- package/src/devcouncil/integrations/mcp/handlers/cli_gate.py +43 -0
- package/src/devcouncil/integrations/mcp/handlers/codeintel.py +182 -0
- package/src/devcouncil/integrations/mcp/handlers/debug.py +236 -0
- package/src/devcouncil/integrations/mcp/handlers/evidence.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/git.py +99 -0
- package/src/devcouncil/integrations/mcp/handlers/graph.py +36 -0
- package/src/devcouncil/integrations/mcp/handlers/handoff.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/knowledge.py +30 -0
- package/src/devcouncil/integrations/mcp/handlers/lease.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/live.py +108 -0
- package/src/devcouncil/integrations/mcp/handlers/map.py +676 -0
- package/src/devcouncil/integrations/mcp/handlers/next_task.py +35 -0
- package/src/devcouncil/integrations/mcp/handlers/policy.py +80 -0
- package/src/devcouncil/integrations/mcp/handlers/prompts.py +168 -0
- package/src/devcouncil/integrations/mcp/handlers/provenance.py +101 -0
- package/src/devcouncil/integrations/mcp/handlers/read.py +74 -0
- package/src/devcouncil/integrations/mcp/handlers/router_cache.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/run.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/runs.py +84 -0
- package/src/devcouncil/integrations/mcp/handlers/scope.py +56 -0
- package/src/devcouncil/integrations/mcp/handlers/status.py +114 -0
- package/src/devcouncil/integrations/mcp/handlers/task.py +100 -0
- package/src/devcouncil/integrations/mcp/handlers/tool_specs.py +904 -0
- package/src/devcouncil/integrations/mcp/handlers/trace.py +65 -0
- package/src/devcouncil/integrations/mcp/handlers/verify.py +45 -0
- package/src/devcouncil/integrations/mcp/handlers/wiki.py +60 -0
- package/src/devcouncil/integrations/mcp/handlers/write.py +69 -0
- package/src/devcouncil/integrations/mcp/server.py +265 -2391
- package/src/devcouncil/integrations/mcp/util.py +303 -0
- package/src/devcouncil/integrations/setup.py +152 -0
- package/src/devcouncil/knowledge/fetch.py +4 -0
- package/src/devcouncil/knowledge/knowledge_select.py +38 -0
- package/src/devcouncil/knowledge/okf.py +2 -1
- package/src/devcouncil/knowledge/resource_discovery.py +40 -0
- package/src/devcouncil/knowledge/wiki.py +643 -0
- package/src/devcouncil/knowledge/wiki_read.py +87 -0
- package/src/devcouncil/live/cards.py +7 -7
- package/src/devcouncil/live/models.py +4 -1
- package/src/devcouncil/live/reviewer.py +90 -11
- package/src/devcouncil/live/signals.py +4 -2
- package/src/devcouncil/live/tasks.py +12 -3
- package/src/devcouncil/live/transcripts.py +69 -2
- package/src/devcouncil/llm/cache.py +5 -6
- package/src/devcouncil/llm/model_defaults.yaml +10 -10
- package/src/devcouncil/llm/provider.py +647 -73
- package/src/devcouncil/llm/router.py +271 -46
- package/src/devcouncil/llm/semantic_bridge.py +614 -0
- package/src/devcouncil/optimization/gepa_agent.py +6 -4
- package/src/devcouncil/optimization/skillopt.py +9 -5
- package/src/devcouncil/planning/arbiter_service.py +12 -3
- package/src/devcouncil/planning/correction_manifest.py +107 -10
- package/src/devcouncil/planning/plan_difficulty.py +69 -0
- package/src/devcouncil/planning/plan_service.py +5 -2
- package/src/devcouncil/planning/planned_files_reconcile.py +191 -0
- package/src/devcouncil/planning/prompt_enhancer_service.py +6 -5
- package/src/devcouncil/planning/question_conversion.py +56 -0
- package/src/devcouncil/planning/spec_service.py +9 -3
- package/src/devcouncil/repo/ci_scaffold.py +197 -1
- package/src/devcouncil/repo/gitignore.py +1 -2
- package/src/devcouncil/reporting/evidence_export.py +124 -0
- package/src/devcouncil/reporting/evidence_html.py +210 -0
- package/src/devcouncil/reporting/json_report.py +16 -12
- package/src/devcouncil/reporting/markdown_report.py +38 -9
- package/src/devcouncil/reporting/mcp_resources.py +142 -0
- package/src/devcouncil/reporting/report_builder.py +40 -4
- package/src/devcouncil/reporting/task_provenance.py +42 -0
- package/src/devcouncil/reporting/verdict.py +75 -0
- package/src/devcouncil/skills/library/README.md +1 -0
- package/src/devcouncil/skills/library/devcouncil-hero-loop.md +109 -0
- package/src/devcouncil/skills/library/devcouncil-verification.md +109 -0
- package/src/devcouncil/skills/library/devcouncil.md +93 -0
- package/src/devcouncil/skills/registry.py +43 -12
- package/src/devcouncil/storage/db.py +57 -11
- package/src/devcouncil/storage/models.py +6 -0
- package/src/devcouncil/storage/native.py +5 -3
- package/src/devcouncil/storage/repositories.py +50 -18
- package/src/devcouncil/telemetry/context.py +28 -0
- package/src/devcouncil/telemetry/cost.py +4 -5
- package/src/devcouncil/telemetry/logging_setup.py +78 -11
- package/src/devcouncil/telemetry/model_pricing.yaml +7 -0
- package/src/devcouncil/telemetry/stages.py +27 -2
- package/src/devcouncil/telemetry/tracker.py +50 -13
- package/src/devcouncil/ui/dashboard.py +120 -8
- package/src/devcouncil/utils/fsio.py +58 -0
- package/src/devcouncil/utils/git_snapshot.py +112 -0
- package/src/devcouncil/utils/json_persist.py +53 -0
- package/src/devcouncil/utils/proc.py +89 -0
- package/src/devcouncil/verification/acceptance_compiler.py +36 -13
- package/src/devcouncil/verification/ad_hoc_check.py +95 -3
- package/src/devcouncil/verification/checks/__init__.py +41 -0
- package/src/devcouncil/verification/checks/acceptance.py +39 -0
- package/src/devcouncil/verification/checks/acceptance_corpus.py +194 -0
- package/src/devcouncil/verification/checks/acceptance_evidence.py +239 -0
- package/src/devcouncil/verification/checks/command_evidence.py +148 -0
- package/src/devcouncil/verification/checks/compiled_acceptance.py +179 -0
- package/src/devcouncil/verification/checks/corpus_stale.py +124 -0
- package/src/devcouncil/verification/checks/corpus_verification.py +9 -0
- package/src/devcouncil/verification/checks/dead_symbols.py +360 -0
- package/src/devcouncil/verification/checks/diff_coverage_gate.py +101 -0
- package/src/devcouncil/verification/checks/doc_code_ref.py +79 -0
- package/src/devcouncil/verification/checks/liveness_ratchet.py +336 -0
- package/src/devcouncil/verification/checks/orphan_diff.py +104 -0
- package/src/devcouncil/verification/checks/planned_files.py +98 -0
- package/src/devcouncil/verification/checks/semantic_diff.py +241 -0
- package/src/devcouncil/verification/checks/stale_map.py +80 -0
- package/src/devcouncil/verification/checks/stub_scan.py +71 -0
- package/src/devcouncil/verification/checks/subsystem_boundary.py +103 -0
- package/src/devcouncil/verification/checks/wiring.py +216 -0
- package/src/devcouncil/verification/claims/__init__.py +23 -0
- package/src/devcouncil/verification/claims/checks.py +395 -0
- package/src/devcouncil/verification/claims/mapper.py +168 -0
- package/src/devcouncil/verification/claims/models.py +39 -0
- package/src/devcouncil/verification/claims/transcript.py +92 -0
- package/src/devcouncil/verification/claims/verdict.py +88 -0
- package/src/devcouncil/verification/command_evidence.py +170 -0
- package/src/devcouncil/verification/command_malformation.py +147 -0
- package/src/devcouncil/verification/command_runner.py +164 -0
- package/src/devcouncil/verification/coverage_measurement.py +292 -0
- package/src/devcouncil/verification/diff_coverage.py +151 -0
- package/src/devcouncil/verification/difficulty.py +296 -0
- package/src/devcouncil/verification/effort_heuristics.py +178 -0
- package/src/devcouncil/verification/gap_ids.py +63 -0
- package/src/devcouncil/verification/gate_cache.py +194 -0
- package/src/devcouncil/verification/gate_selector.py +344 -0
- package/src/devcouncil/verification/git_diff_fallback.py +272 -0
- package/src/devcouncil/verification/implementation_reviewer.py +13 -0
- package/src/devcouncil/verification/incremental_check.py +241 -0
- package/src/devcouncil/verification/next_actions.py +60 -1
- package/src/devcouncil/verification/rigor_analytics.py +130 -0
- package/src/devcouncil/verification/sandbox.py +38 -11
- package/src/devcouncil/verification/stub_detector.py +369 -0
- package/src/devcouncil/verification/test_resolver.py +67 -1
- package/src/devcouncil/verification/verifier.py +137 -1666
- package/src/devcouncil/verification/verify_orchestration.py +610 -0
- package/src/devcouncil/verification/verify_setup.py +176 -0
- package/src/devcouncil/verification/wiki_refresh.py +208 -0
- package/src/semantic_layer/__init__.py +58 -0
- package/src/semantic_layer/benchmark.py +75 -0
- package/src/semantic_layer/cache.py +290 -0
- package/src/semantic_layer/compressor.py +137 -0
- package/src/semantic_layer/config.py +75 -0
- package/src/semantic_layer/embeddings.py +69 -0
- package/src/semantic_layer/llm_backends.py +99 -0
- package/src/semantic_layer/pipeline.py +111 -0
- package/src/semantic_layer/router.py +128 -0
- package/src/semantic_layer/tuner.py +72 -0
- package/uv.lock +973 -9
- package/src/devcouncil/artifacts/migrations.py +0 -20
- package/src/devcouncil/artifacts/schemas.py +0 -23
- package/src/devcouncil/artifacts/serializer.py +0 -21
- package/src/devcouncil/integrations/gitnexus.py +0 -70
- package/src/devcouncil/integrations/graphify.py +0 -34
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""Codebase wiki: generate and maintain an OKF bundle documenting the repository.
|
|
2
|
+
|
|
3
|
+
This is DevCouncil's take on the "LLM wiki" pattern (OpenWiki, Karpathy's LLM-wiki
|
|
4
|
+
gist, Google's Open Knowledge Format): a directory of markdown concept documents with
|
|
5
|
+
YAML frontmatter that agents consult before doing real work, kept up to date by the
|
|
6
|
+
tool rather than by hand.
|
|
7
|
+
|
|
8
|
+
Design:
|
|
9
|
+
|
|
10
|
+
* **Deterministic skeleton** — every page is derived from ``repo_map.json``
|
|
11
|
+
(:class:`devcouncil.indexing.repo_mapper.RepoMap`), so structure, file lists, and
|
|
12
|
+
cross-links are always correct and generation works offline with no model configured.
|
|
13
|
+
* **Optional LLM enrichment** — when a :class:`devcouncil.llm.router.ModelRouter` is
|
|
14
|
+
supplied, new/stale pages get prose sections (overview, key flows, agent guidance)
|
|
15
|
+
written by the ``wiki_writer`` role. Enrichment degrades to the skeleton on any
|
|
16
|
+
model failure (the router's ``fallback`` machinery).
|
|
17
|
+
* **OKF-conformant output** — pages are :class:`devcouncil.knowledge.okf.OKFDocument`
|
|
18
|
+
bundles under ``.devcouncil/knowledge/okf/wiki/``, which
|
|
19
|
+
:func:`devcouncil.knowledge.sources.discover_knowledge_sources` already scans — so
|
|
20
|
+
wiki pages flow into planning/council/task prompts with zero extra wiring, selected
|
|
21
|
+
by their tags (subsystem path segments) like any other OKF knowledge.
|
|
22
|
+
* **Incremental updates** — a fingerprint per page (hash of the repo-map slice that
|
|
23
|
+
shapes it) is kept in a ``.wiki-state.json`` sidecar. Unchanged pages are skipped on
|
|
24
|
+
regeneration, which both keeps updates cheap and *preserves prior LLM enrichment*.
|
|
25
|
+
* **log.md** — an OKF-conventional chronological change log of what each run touched.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import hashlib
|
|
32
|
+
import json
|
|
33
|
+
import logging
|
|
34
|
+
import re
|
|
35
|
+
from datetime import datetime, timezone
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from typing import TYPE_CHECKING, Optional
|
|
38
|
+
|
|
39
|
+
from pydantic import BaseModel, Field
|
|
40
|
+
|
|
41
|
+
from devcouncil.indexing.repo_mapper import RepoMap, RepoSubsystem
|
|
42
|
+
from devcouncil.knowledge.okf import OKFBundle, OKFDocument, read_bundle, validate_bundle, write_bundle
|
|
43
|
+
from devcouncil.utils.json_persist import read_json, write_json
|
|
44
|
+
|
|
45
|
+
if TYPE_CHECKING:
|
|
46
|
+
from devcouncil.llm.router import ModelRouter
|
|
47
|
+
|
|
48
|
+
logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
# Bump when skeleton rendering changes shape, so every page is considered stale and
|
|
51
|
+
# regenerated even though its repo-map slice is unchanged.
|
|
52
|
+
GENERATOR_VERSION = 1
|
|
53
|
+
|
|
54
|
+
# Default bundle location relative to the configured knowledge directory. Living under
|
|
55
|
+
# knowledge/okf/ is what makes prompt injection automatic (sources.py scans it).
|
|
56
|
+
WIKI_SUBDIR = "okf/wiki"
|
|
57
|
+
|
|
58
|
+
_STATE_FILENAME = ".wiki-state.json"
|
|
59
|
+
_LOG_MAX_ENTRIES = 50
|
|
60
|
+
|
|
61
|
+
_ENRICH_SYSTEM = (
|
|
62
|
+
"You are a senior engineer writing agent-facing documentation for a codebase wiki. "
|
|
63
|
+
"You are given structured facts about one subsystem of a repository (its files, "
|
|
64
|
+
"entry points, roles, and neighbors). Write concise, concrete documentation that "
|
|
65
|
+
"helps a coding agent work in this subsystem. Never invent files, APIs, or "
|
|
66
|
+
"behavior not implied by the provided facts. Prefer specifics over generalities."
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class WikiProse(BaseModel):
|
|
71
|
+
"""LLM-written prose sections for one wiki page. All fields optional so the
|
|
72
|
+
enrichment call can degrade to an empty instance (skeleton-only page)."""
|
|
73
|
+
|
|
74
|
+
overview: str = Field(
|
|
75
|
+
"", description="2-4 sentences: what this subsystem does and why it exists."
|
|
76
|
+
)
|
|
77
|
+
key_flows: list[str] = Field(
|
|
78
|
+
default_factory=list,
|
|
79
|
+
description="Up to 5 short bullets tracing the important call/data flows.",
|
|
80
|
+
)
|
|
81
|
+
agent_guidance: list[str] = Field(
|
|
82
|
+
default_factory=list,
|
|
83
|
+
description="Up to 5 short bullets: conventions and pitfalls an agent editing this subsystem must respect.",
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class WikiResult(BaseModel):
|
|
88
|
+
"""Outcome of one generate/update run."""
|
|
89
|
+
|
|
90
|
+
wiki_dir: str = ""
|
|
91
|
+
created: list[str] = Field(default_factory=list)
|
|
92
|
+
updated: list[str] = Field(default_factory=list)
|
|
93
|
+
skipped: list[str] = Field(default_factory=list)
|
|
94
|
+
enriched: list[str] = Field(default_factory=list)
|
|
95
|
+
problems: list[str] = Field(default_factory=list)
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def changed(self) -> list[str]:
|
|
99
|
+
return self.created + self.updated
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def slugify(area: str) -> str:
|
|
103
|
+
"""Filesystem-safe page name for a subsystem area (``src/devcouncil/council/`` →
|
|
104
|
+
``src-devcouncil-council``)."""
|
|
105
|
+
return re.sub(r"[^a-z0-9]+", "-", area.lower()).strip("-") or "root"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _area_tags(area: str) -> list[str]:
|
|
109
|
+
"""Tags for a subsystem page: the meaningful path segments of its area. Tags double
|
|
110
|
+
as prompt-selection keywords (sources.py derives keywords from OKF tags), so a goal
|
|
111
|
+
mentioning "council" or "execution" pulls the matching wiki page into context."""
|
|
112
|
+
segments = [seg for seg in re.split(r"[/\\]+", area) if seg]
|
|
113
|
+
# Drop generic roots that would match almost any goal.
|
|
114
|
+
tags = [seg for seg in segments if seg.lower() not in {"src", "lib", "app", "pkg"}]
|
|
115
|
+
return tags or segments
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _now_iso() -> str:
|
|
119
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _fingerprint(payload: object) -> str:
|
|
123
|
+
raw = json.dumps(payload, sort_keys=True, default=str)
|
|
124
|
+
return hashlib.sha256(f"v{GENERATOR_VERSION}:{raw}".encode("utf-8")).hexdigest()[:16]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _subsystem_payload(subsystem: RepoSubsystem, repo_map: RepoMap) -> dict:
|
|
128
|
+
"""The repo-map slice that shapes one subsystem page (also the fingerprint input)."""
|
|
129
|
+
area_prefix = subsystem.area.rstrip("/")
|
|
130
|
+
files = [
|
|
131
|
+
{"path": f.path, "kind": f.kind, "summary": f.summary}
|
|
132
|
+
for f in repo_map.files
|
|
133
|
+
if f.area == subsystem.area or f.path.startswith(area_prefix)
|
|
134
|
+
][:80]
|
|
135
|
+
return {
|
|
136
|
+
"area": subsystem.area,
|
|
137
|
+
"summary": subsystem.summary,
|
|
138
|
+
"entry_points": subsystem.entry_points,
|
|
139
|
+
"critical_files": subsystem.critical_files,
|
|
140
|
+
"neighbors": subsystem.neighbors,
|
|
141
|
+
"handoff_paths": subsystem.handoff_paths,
|
|
142
|
+
"role_files": subsystem.role_files,
|
|
143
|
+
"files": files,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _overview_payload(repo_map: RepoMap) -> dict:
|
|
148
|
+
return {
|
|
149
|
+
"languages": repo_map.languages,
|
|
150
|
+
"frameworks": repo_map.frameworks,
|
|
151
|
+
"package_managers": repo_map.package_managers,
|
|
152
|
+
"test_commands": repo_map.test_commands,
|
|
153
|
+
"important_files": repo_map.important_files,
|
|
154
|
+
"subsystems": [s.area for s in repo_map.subsystems],
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# --- Skeleton rendering ----------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _bullets(items: list[str], code: bool = True) -> list[str]:
|
|
162
|
+
fmt = "- `{0}`" if code else "- {0}"
|
|
163
|
+
return [fmt.format(item) for item in items]
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _wired_to_links(project_root: Path | None, subsystem: RepoSubsystem) -> list[str]:
|
|
167
|
+
"""Graph-derived import neighbors for wiki 'Wired to' sections (OKF-style links).
|
|
168
|
+
|
|
169
|
+
Link targets use the same ``files/<path>.md`` layout as
|
|
170
|
+
``dev graph export --format okf``, via :mod:`export_links`, so wiki pages can
|
|
171
|
+
cross-link into a sibling graph OKF bundle under ``../graph/``.
|
|
172
|
+
"""
|
|
173
|
+
if project_root is None:
|
|
174
|
+
return []
|
|
175
|
+
try:
|
|
176
|
+
from devcouncil.indexing.graph.build import load_code_graph
|
|
177
|
+
from devcouncil.indexing.graph.export_links import subsystem_doc_path, wired_to_bullets
|
|
178
|
+
|
|
179
|
+
graph = load_code_graph(project_root)
|
|
180
|
+
if graph is None:
|
|
181
|
+
return []
|
|
182
|
+
# Collect files in this area from critical/entry/role lists
|
|
183
|
+
area_files = set(subsystem.entry_points + subsystem.critical_files)
|
|
184
|
+
for paths in (subsystem.role_files or {}).values():
|
|
185
|
+
area_files.update(paths)
|
|
186
|
+
targets: set[str] = set()
|
|
187
|
+
for e in graph.edges:
|
|
188
|
+
if e.kind != "imports":
|
|
189
|
+
continue
|
|
190
|
+
if "::" in e.source or "::" in e.target:
|
|
191
|
+
continue
|
|
192
|
+
if e.source in area_files and e.target not in area_files:
|
|
193
|
+
targets.add(e.target)
|
|
194
|
+
from_rel = subsystem_doc_path(subsystem.area)
|
|
195
|
+
return wired_to_bullets(targets, from_rel=from_rel, link_to_graph=True)
|
|
196
|
+
except Exception:
|
|
197
|
+
return []
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _subsystem_body(
|
|
201
|
+
subsystem: RepoSubsystem,
|
|
202
|
+
slug_by_area: dict[str, str],
|
|
203
|
+
prose: Optional[WikiProse] = None,
|
|
204
|
+
*,
|
|
205
|
+
project_root: Path | None = None,
|
|
206
|
+
) -> str:
|
|
207
|
+
lines: list[str] = [f"# {subsystem.area}", "", subsystem.summary.strip()]
|
|
208
|
+
|
|
209
|
+
if prose and prose.overview.strip():
|
|
210
|
+
lines += ["", "## Overview", "", prose.overview.strip()]
|
|
211
|
+
|
|
212
|
+
if subsystem.entry_points:
|
|
213
|
+
lines += ["", "## Entry points", ""] + _bullets(subsystem.entry_points)
|
|
214
|
+
if subsystem.critical_files:
|
|
215
|
+
lines += ["", "## Critical files", ""] + _bullets(subsystem.critical_files)
|
|
216
|
+
|
|
217
|
+
if subsystem.role_files:
|
|
218
|
+
lines += ["", "## Files by role", ""]
|
|
219
|
+
for role, paths in subsystem.role_files.items():
|
|
220
|
+
if paths:
|
|
221
|
+
lines.append(f"- **{role}**: " + ", ".join(f"`{p}`" for p in paths[:8]))
|
|
222
|
+
|
|
223
|
+
if prose and prose.key_flows:
|
|
224
|
+
lines += ["", "## Key flows", ""] + _bullets(prose.key_flows, code=False)
|
|
225
|
+
|
|
226
|
+
if subsystem.neighbors:
|
|
227
|
+
lines += ["", "## Neighbors", ""]
|
|
228
|
+
for neighbor in subsystem.neighbors:
|
|
229
|
+
slug = slug_by_area.get(neighbor)
|
|
230
|
+
if slug:
|
|
231
|
+
lines.append(f"- [{neighbor}]({slug}.md)")
|
|
232
|
+
else:
|
|
233
|
+
lines.append(f"- `{neighbor}`")
|
|
234
|
+
if subsystem.handoff_paths:
|
|
235
|
+
lines += ["", "## Handoff paths", ""] + _bullets(subsystem.handoff_paths)
|
|
236
|
+
|
|
237
|
+
wired = _wired_to_links(project_root, subsystem)
|
|
238
|
+
if wired:
|
|
239
|
+
lines += ["", "## Wired to", ""] + wired
|
|
240
|
+
|
|
241
|
+
if prose and prose.agent_guidance:
|
|
242
|
+
lines += ["", "## Guidance for agents", ""] + _bullets(prose.agent_guidance, code=False)
|
|
243
|
+
|
|
244
|
+
return "\n".join(lines).strip()
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _development_body(repo_map: RepoMap) -> str:
|
|
248
|
+
lines: list[str] = ["# Development guide", ""]
|
|
249
|
+
if repo_map.languages:
|
|
250
|
+
lines += ["## Languages", ""] + _bullets(repo_map.languages, code=False)
|
|
251
|
+
if repo_map.frameworks:
|
|
252
|
+
lines += ["", "## Frameworks", ""] + _bullets(repo_map.frameworks, code=False)
|
|
253
|
+
if repo_map.package_managers:
|
|
254
|
+
lines += ["", "## Package managers", ""] + _bullets(repo_map.package_managers, code=False)
|
|
255
|
+
if repo_map.test_commands:
|
|
256
|
+
lines += ["", "## Test and check commands", ""] + _bullets(repo_map.test_commands)
|
|
257
|
+
if repo_map.important_files:
|
|
258
|
+
lines += ["", "## Important files", ""] + _bullets(repo_map.important_files)
|
|
259
|
+
return "\n".join(lines).strip()
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _index_body(project_name: str, repo_map: RepoMap, slug_by_area: dict[str, str]) -> str:
|
|
263
|
+
lines = [
|
|
264
|
+
f"# {project_name} codebase wiki",
|
|
265
|
+
"",
|
|
266
|
+
"Agent-facing documentation for this repository, generated and maintained by "
|
|
267
|
+
"`dev wiki` from `.devcouncil/repo_map.json`. Start here, then follow the "
|
|
268
|
+
"subsystem pages. If the wiki and source disagree, trust the source and run "
|
|
269
|
+
"`dev wiki update`.",
|
|
270
|
+
"",
|
|
271
|
+
"## Subsystems",
|
|
272
|
+
"",
|
|
273
|
+
]
|
|
274
|
+
for subsystem in repo_map.subsystems:
|
|
275
|
+
slug = slug_by_area[subsystem.area]
|
|
276
|
+
summary = subsystem.summary.strip().rstrip(".")
|
|
277
|
+
lines.append(f"- [{subsystem.area}](subsystems/{slug}.md) — {summary}")
|
|
278
|
+
lines += [
|
|
279
|
+
"",
|
|
280
|
+
"## Reference",
|
|
281
|
+
"",
|
|
282
|
+
"- [Development guide](overview/development.md)",
|
|
283
|
+
"- [Change log](log.md)",
|
|
284
|
+
]
|
|
285
|
+
return "\n".join(lines).strip()
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _build_skeleton(
|
|
289
|
+
repo_map: RepoMap,
|
|
290
|
+
project_name: str,
|
|
291
|
+
timestamp: str,
|
|
292
|
+
prose_by_area: dict[str, WikiProse],
|
|
293
|
+
*,
|
|
294
|
+
project_root: Path | None = None,
|
|
295
|
+
) -> list[OKFDocument]:
|
|
296
|
+
slug_by_area = {s.area: slugify(s.area) for s in repo_map.subsystems}
|
|
297
|
+
docs: list[OKFDocument] = [
|
|
298
|
+
OKFDocument(
|
|
299
|
+
type="Codebase Wiki Index",
|
|
300
|
+
title=f"{project_name} codebase wiki",
|
|
301
|
+
description=f"Index of agent-facing wiki pages for {project_name}.",
|
|
302
|
+
tags=[],
|
|
303
|
+
timestamp=timestamp,
|
|
304
|
+
body=_index_body(project_name, repo_map, slug_by_area),
|
|
305
|
+
rel_path="index.md",
|
|
306
|
+
),
|
|
307
|
+
OKFDocument(
|
|
308
|
+
type="Development Guide",
|
|
309
|
+
title=f"{project_name} development guide",
|
|
310
|
+
description=f"Languages, tooling, and test commands for {project_name}."[:280],
|
|
311
|
+
tags=["development", "testing", "build"],
|
|
312
|
+
timestamp=timestamp,
|
|
313
|
+
body=_development_body(repo_map),
|
|
314
|
+
rel_path="overview/development.md",
|
|
315
|
+
),
|
|
316
|
+
]
|
|
317
|
+
for subsystem in repo_map.subsystems:
|
|
318
|
+
slug = slug_by_area[subsystem.area]
|
|
319
|
+
docs.append(
|
|
320
|
+
OKFDocument(
|
|
321
|
+
type="Subsystem",
|
|
322
|
+
title=subsystem.area,
|
|
323
|
+
description=subsystem.summary.strip()[:280],
|
|
324
|
+
resource=subsystem.area,
|
|
325
|
+
tags=["subsystem"] + _area_tags(subsystem.area),
|
|
326
|
+
timestamp=timestamp,
|
|
327
|
+
body=_subsystem_body(
|
|
328
|
+
subsystem,
|
|
329
|
+
slug_by_area,
|
|
330
|
+
prose_by_area.get(subsystem.area),
|
|
331
|
+
project_root=project_root,
|
|
332
|
+
),
|
|
333
|
+
rel_path=f"subsystems/{slug}.md",
|
|
334
|
+
)
|
|
335
|
+
)
|
|
336
|
+
return docs
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
# --- State + change log ------------------------------------------------------------
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _load_state(wiki_dir: Path) -> dict:
|
|
343
|
+
state_path = wiki_dir / _STATE_FILENAME
|
|
344
|
+
try:
|
|
345
|
+
data = read_json(state_path)
|
|
346
|
+
return data if isinstance(data, dict) else {}
|
|
347
|
+
except (OSError, json.JSONDecodeError):
|
|
348
|
+
return {}
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _save_state(wiki_dir: Path, state: dict) -> None:
|
|
352
|
+
wiki_dir.mkdir(parents=True, exist_ok=True)
|
|
353
|
+
write_json(wiki_dir / _STATE_FILENAME, state, sort_keys=True)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _log_document(wiki_dir: Path, timestamp: str, result: WikiResult) -> OKFDocument:
|
|
357
|
+
"""Build log.md: newest entry first, bounded to the last _LOG_MAX_ENTRIES runs."""
|
|
358
|
+
entry_lines = [f"## {timestamp}", ""]
|
|
359
|
+
for label, paths in (
|
|
360
|
+
("Created", result.created),
|
|
361
|
+
("Updated", result.updated),
|
|
362
|
+
("Enriched", result.enriched),
|
|
363
|
+
):
|
|
364
|
+
if paths:
|
|
365
|
+
entry_lines.append(f"- {label}: " + ", ".join(f"`{p}`" for p in sorted(paths)))
|
|
366
|
+
if not result.created and not result.updated:
|
|
367
|
+
entry_lines.append("- No pages changed (all fingerprints fresh).")
|
|
368
|
+
entry = "\n".join(entry_lines)
|
|
369
|
+
|
|
370
|
+
previous_entries: list[str] = []
|
|
371
|
+
log_path = wiki_dir / "log.md"
|
|
372
|
+
if log_path.is_file():
|
|
373
|
+
try:
|
|
374
|
+
existing = OKFDocument.from_markdown(
|
|
375
|
+
log_path.read_text(encoding="utf-8", errors="replace"), rel_path="log.md"
|
|
376
|
+
)
|
|
377
|
+
# Entries are "## <timestamp>" sections after the H1.
|
|
378
|
+
chunks = re.split(r"\n(?=## )", existing.body)
|
|
379
|
+
previous_entries = [c.strip() for c in chunks if c.strip().startswith("## ")]
|
|
380
|
+
except Exception:
|
|
381
|
+
# A corrupt change log is a convenience artifact — restart it rather
|
|
382
|
+
# than failing the whole wiki refresh.
|
|
383
|
+
logger.warning("wiki log.md unreadable; restarting change log", exc_info=True)
|
|
384
|
+
|
|
385
|
+
entries = [entry] + previous_entries
|
|
386
|
+
body = "# Wiki change log\n\n" + "\n\n".join(entries[:_LOG_MAX_ENTRIES])
|
|
387
|
+
return OKFDocument(
|
|
388
|
+
type="Change Log",
|
|
389
|
+
title="Wiki change log",
|
|
390
|
+
description="Chronological history of wiki generation runs.",
|
|
391
|
+
tags=[],
|
|
392
|
+
timestamp=timestamp,
|
|
393
|
+
body=body,
|
|
394
|
+
rel_path="log.md",
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
# --- Enrichment ----------------------------------------------------------------
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
async def _enrich_area(router: "ModelRouter", payload: dict) -> WikiProse:
|
|
402
|
+
messages = [
|
|
403
|
+
{"role": "system", "content": _ENRICH_SYSTEM},
|
|
404
|
+
{
|
|
405
|
+
"role": "user",
|
|
406
|
+
"content": (
|
|
407
|
+
"Write the wiki prose sections for this subsystem.\n\n"
|
|
408
|
+
f"Subsystem facts (JSON):\n{json.dumps(payload, indent=2)}"
|
|
409
|
+
),
|
|
410
|
+
},
|
|
411
|
+
]
|
|
412
|
+
return await router.complete_structured(
|
|
413
|
+
"wiki_writer", messages, WikiProse, fallback=WikiProse(overview="")
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
async def _enrich_all(router: "ModelRouter", payloads: dict[str, dict]) -> dict[str, WikiProse]:
|
|
418
|
+
areas = list(payloads)
|
|
419
|
+
results = await asyncio.gather(
|
|
420
|
+
*(_enrich_area(router, payloads[a]) for a in areas), return_exceptions=True
|
|
421
|
+
)
|
|
422
|
+
prose: dict[str, WikiProse] = {}
|
|
423
|
+
for area, res in zip(areas, results):
|
|
424
|
+
if isinstance(res, WikiProse):
|
|
425
|
+
prose[area] = res
|
|
426
|
+
else:
|
|
427
|
+
logger.warning("Wiki enrichment failed for %s: %s", area, res)
|
|
428
|
+
return prose
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# --- Public API -----------------------------------------------------------------
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def wiki_stale_pages(project_root: Path, repo_map: RepoMap, wiki_dir: Path) -> dict[str, str]:
|
|
435
|
+
"""Map of rel_path → reason for every page that would be rewritten by an update."""
|
|
436
|
+
state = _load_state(wiki_dir).get("pages", {})
|
|
437
|
+
stale: dict[str, str] = {}
|
|
438
|
+
fingerprints = _page_fingerprints(repo_map)
|
|
439
|
+
for rel_path, fp in fingerprints.items():
|
|
440
|
+
recorded = state.get(rel_path, {}).get("fingerprint")
|
|
441
|
+
if not (wiki_dir / rel_path).is_file():
|
|
442
|
+
stale[rel_path] = "missing"
|
|
443
|
+
elif recorded != fp:
|
|
444
|
+
stale[rel_path] = "outdated"
|
|
445
|
+
return stale
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _page_fingerprints(repo_map: RepoMap) -> dict[str, str]:
|
|
449
|
+
fingerprints = {
|
|
450
|
+
"index.md": _fingerprint(
|
|
451
|
+
{"kind": "index", "areas": [s.area for s in repo_map.subsystems],
|
|
452
|
+
"summaries": [s.summary for s in repo_map.subsystems]}
|
|
453
|
+
),
|
|
454
|
+
"overview/development.md": _fingerprint({"kind": "development", **_overview_payload(repo_map)}),
|
|
455
|
+
}
|
|
456
|
+
for subsystem in repo_map.subsystems:
|
|
457
|
+
rel = f"subsystems/{slugify(subsystem.area)}.md"
|
|
458
|
+
fingerprints[rel] = _fingerprint(_subsystem_payload(subsystem, repo_map))
|
|
459
|
+
return fingerprints
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def generate_wiki(
|
|
463
|
+
project_root: Path,
|
|
464
|
+
repo_map: RepoMap,
|
|
465
|
+
wiki_dir: Path,
|
|
466
|
+
*,
|
|
467
|
+
router: "Optional[ModelRouter]" = None,
|
|
468
|
+
force: bool = False,
|
|
469
|
+
project_name: str = "",
|
|
470
|
+
) -> WikiResult:
|
|
471
|
+
"""Generate or incrementally update the codebase wiki bundle in ``wiki_dir``.
|
|
472
|
+
|
|
473
|
+
Pages whose fingerprint matches the recorded state are left untouched (preserving
|
|
474
|
+
any prior LLM enrichment). New/stale pages are rewritten from the current repo map
|
|
475
|
+
and, when ``router`` is provided, enriched with LLM prose via the ``wiki_writer``
|
|
476
|
+
role (degrading to the deterministic skeleton on any model failure). ``force``
|
|
477
|
+
rewrites (and re-enriches) everything.
|
|
478
|
+
"""
|
|
479
|
+
project_name = project_name or (project_root.name or "Project")
|
|
480
|
+
timestamp = _now_iso()
|
|
481
|
+
state = _load_state(wiki_dir)
|
|
482
|
+
pages_state: dict = dict(state.get("pages", {}))
|
|
483
|
+
fingerprints = _page_fingerprints(repo_map)
|
|
484
|
+
|
|
485
|
+
result = WikiResult(wiki_dir=str(wiki_dir))
|
|
486
|
+
|
|
487
|
+
# Decide which pages need (re)writing before doing any model work.
|
|
488
|
+
to_write: set[str] = set()
|
|
489
|
+
for rel_path, fp in fingerprints.items():
|
|
490
|
+
exists = (wiki_dir / rel_path).is_file()
|
|
491
|
+
if force or not exists or pages_state.get(rel_path, {}).get("fingerprint") != fp:
|
|
492
|
+
to_write.add(rel_path)
|
|
493
|
+
(result.created if not exists else result.updated).append(rel_path)
|
|
494
|
+
else:
|
|
495
|
+
result.skipped.append(rel_path)
|
|
496
|
+
|
|
497
|
+
# Enrich only the subsystem pages being written (skeleton-only without a router).
|
|
498
|
+
prose_by_area: dict[str, WikiProse] = {}
|
|
499
|
+
if router is not None:
|
|
500
|
+
payloads = {
|
|
501
|
+
s.area: _subsystem_payload(s, repo_map)
|
|
502
|
+
for s in repo_map.subsystems
|
|
503
|
+
if f"subsystems/{slugify(s.area)}.md" in to_write
|
|
504
|
+
}
|
|
505
|
+
if payloads:
|
|
506
|
+
prose_by_area = asyncio.run(_enrich_all(router, payloads))
|
|
507
|
+
for area, prose in prose_by_area.items():
|
|
508
|
+
if prose.overview or prose.key_flows or prose.agent_guidance:
|
|
509
|
+
result.enriched.append(f"subsystems/{slugify(area)}.md")
|
|
510
|
+
|
|
511
|
+
docs = _build_skeleton(
|
|
512
|
+
repo_map, project_name, timestamp, prose_by_area, project_root=project_root
|
|
513
|
+
)
|
|
514
|
+
bundle = OKFBundle(documents=[d for d in docs if d.rel_path in to_write])
|
|
515
|
+
write_bundle(bundle, wiki_dir)
|
|
516
|
+
|
|
517
|
+
for rel_path in to_write:
|
|
518
|
+
pages_state[rel_path] = {
|
|
519
|
+
"fingerprint": fingerprints[rel_path],
|
|
520
|
+
"updated_at": timestamp,
|
|
521
|
+
"enriched": rel_path in result.enriched,
|
|
522
|
+
}
|
|
523
|
+
# Drop state for pages that no longer exist in the map (removed subsystems). Their
|
|
524
|
+
# files are left on disk deliberately — they may hold hand edits — but validation
|
|
525
|
+
# below will flag any broken links from index.md if the map shrank.
|
|
526
|
+
pages_state = {k: v for k, v in pages_state.items() if k in fingerprints}
|
|
527
|
+
|
|
528
|
+
# Change log + state sidecar.
|
|
529
|
+
write_bundle(OKFBundle(documents=[_log_document(wiki_dir, timestamp, result)]), wiki_dir)
|
|
530
|
+
_save_state(wiki_dir, {"pages": pages_state, "generated_at": timestamp,
|
|
531
|
+
"generator_version": GENERATOR_VERSION})
|
|
532
|
+
|
|
533
|
+
result.problems = validate_bundle(read_bundle(wiki_dir))
|
|
534
|
+
return result
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _knowledge_dir(root: Path) -> Path:
|
|
538
|
+
"""Resolve the configured knowledge directory (no CLI side effects)."""
|
|
539
|
+
directory = ".devcouncil/knowledge"
|
|
540
|
+
try:
|
|
541
|
+
from devcouncil.app.config import load_config
|
|
542
|
+
|
|
543
|
+
directory = load_config(root).knowledge.directory
|
|
544
|
+
except Exception as exc:
|
|
545
|
+
logger.debug("wiki refresh: knowledge directory unavailable, using default: %s", exc)
|
|
546
|
+
return root / directory
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def wiki_dir_for(root: Path) -> Path:
|
|
550
|
+
return _knowledge_dir(root) / WIKI_SUBDIR
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _project_name(root: Path) -> str:
|
|
554
|
+
name = root.name or "Project"
|
|
555
|
+
try:
|
|
556
|
+
from devcouncil.app.config import load_config
|
|
557
|
+
|
|
558
|
+
name = load_config(root).project.name or name
|
|
559
|
+
except Exception as exc:
|
|
560
|
+
logger.debug("wiki refresh: project name unavailable, using directory name: %s", exc)
|
|
561
|
+
return name
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _load_repo_map(root: Path, *, remap: bool) -> RepoMap:
|
|
565
|
+
map_path = root / ".devcouncil" / "repo_map.json"
|
|
566
|
+
if not remap and map_path.is_file():
|
|
567
|
+
try:
|
|
568
|
+
return RepoMap.model_validate_json(map_path.read_text(encoding="utf-8"))
|
|
569
|
+
except Exception:
|
|
570
|
+
# Corrupt/stale-schema map: regenerate instead of crashing — the
|
|
571
|
+
# mapper is deterministic and cheap relative to a failed wiki run.
|
|
572
|
+
logger.warning("repo_map.json unreadable; regenerating for wiki", exc_info=True)
|
|
573
|
+
from devcouncil.indexing.map_artifacts import generate_map_artifacts
|
|
574
|
+
|
|
575
|
+
return generate_map_artifacts(root, map_path)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _build_router(root: Path):
|
|
579
|
+
"""Best-effort ModelRouter for wiki enrichment; None degrades to the skeleton."""
|
|
580
|
+
try:
|
|
581
|
+
from devcouncil.app.config import get_api_key, load_config
|
|
582
|
+
from devcouncil.llm.provider import create_provider, validate_model_provider
|
|
583
|
+
from devcouncil.llm.router import ModelRouter
|
|
584
|
+
|
|
585
|
+
config = load_config(root)
|
|
586
|
+
validate_model_provider(config.models.provider)
|
|
587
|
+
api_key = get_api_key(config.models.provider, root)
|
|
588
|
+
provider = create_provider(
|
|
589
|
+
config.models.provider, api_key, project_root=root, provider_prefs=config.provider
|
|
590
|
+
)
|
|
591
|
+
role_config = {name: role.model_dump() for name, role in config.models.roles.items()}
|
|
592
|
+
if not role_config:
|
|
593
|
+
return None
|
|
594
|
+
capable = (
|
|
595
|
+
role_config.get("arbiter")
|
|
596
|
+
or role_config.get("planner_a")
|
|
597
|
+
or next(iter(role_config.values()))
|
|
598
|
+
)
|
|
599
|
+
role_config.setdefault("wiki_writer", dict(capable))
|
|
600
|
+
return ModelRouter(provider, role_config, project_root=root)
|
|
601
|
+
except Exception as exc:
|
|
602
|
+
logger.warning("Wiki enrichment unavailable (no model router): %s", exc)
|
|
603
|
+
return None
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def refresh_wiki(
|
|
607
|
+
project_root: Path,
|
|
608
|
+
*,
|
|
609
|
+
llm: bool = False,
|
|
610
|
+
force: bool = False,
|
|
611
|
+
remap: bool = False,
|
|
612
|
+
) -> WikiResult:
|
|
613
|
+
"""Refresh the codebase wiki without CLI/Rich side effects."""
|
|
614
|
+
from devcouncil.telemetry.logging_setup import set_log_dir
|
|
615
|
+
from devcouncil.telemetry.stages import log_stage, log_step
|
|
616
|
+
|
|
617
|
+
root = project_root.expanduser().resolve()
|
|
618
|
+
set_log_dir(root)
|
|
619
|
+
logger.info("wiki refresh: llm=%s force=%s remap=%s", llm, force, remap)
|
|
620
|
+
|
|
621
|
+
with log_stage("wiki", project_root=root, subcommand="update"):
|
|
622
|
+
log_step("wiki/1: loading repository map", project_root=root, trace=True)
|
|
623
|
+
repo_map = _load_repo_map(root, remap=remap)
|
|
624
|
+
|
|
625
|
+
router = _build_router(root) if llm else None
|
|
626
|
+
|
|
627
|
+
log_step("wiki/2: generating wiki pages", project_root=root, trace=True)
|
|
628
|
+
result = generate_wiki(
|
|
629
|
+
root,
|
|
630
|
+
repo_map,
|
|
631
|
+
wiki_dir_for(root),
|
|
632
|
+
router=router,
|
|
633
|
+
force=force,
|
|
634
|
+
project_name=_project_name(root),
|
|
635
|
+
)
|
|
636
|
+
log_step(
|
|
637
|
+
"wiki/complete",
|
|
638
|
+
project_root=root,
|
|
639
|
+
created=len(result.created),
|
|
640
|
+
updated=len(result.updated),
|
|
641
|
+
trace=True,
|
|
642
|
+
)
|
|
643
|
+
return result
|