devcouncil 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -33
- package/package.json +6 -2
- package/packages/codeintel-grammars/hatch_build.py +43 -0
- package/packages/codeintel-grammars/pyproject.toml +16 -0
- package/packages/codeintel-grammars/src/devcouncil_codeintel_grammars/__init__.py +93 -0
- package/pyproject.toml +99 -4
- package/src/devcouncil/app/config.py +512 -20
- package/src/devcouncil/app/events.py +4 -23
- package/src/devcouncil/app/orchestrator.py +5 -0
- package/src/devcouncil/app/run_context.py +3 -3
- package/src/devcouncil/assets/__init__.py +4 -1
- package/src/devcouncil/assets/vendor/force-graph.min.js +5 -0
- package/src/devcouncil/campaign/__init__.py +71 -0
- package/src/devcouncil/campaign/bloom.py +137 -0
- package/src/devcouncil/campaign/dashboard.py +123 -0
- package/src/devcouncil/campaign/mailbox.py +305 -0
- package/src/devcouncil/campaign/notify.py +91 -0
- package/src/devcouncil/campaign/orchestrator.py +592 -0
- package/src/devcouncil/campaign/prompts/coordinator.md +29 -0
- package/src/devcouncil/campaign/prompts/director.md +21 -0
- package/src/devcouncil/campaign/prompts/protocol.md +46 -0
- package/src/devcouncil/campaign/prompts/reviewer.md +24 -0
- package/src/devcouncil/campaign/prompts/worker.md +24 -0
- package/src/devcouncil/campaign/roles.py +202 -0
- package/src/devcouncil/campaign/watcher.py +153 -0
- package/src/devcouncil/cli/commands/agents.py +24 -17
- package/src/devcouncil/cli/commands/artifacts.py +36 -27
- package/src/devcouncil/cli/commands/ast.py +12 -3
- package/src/devcouncil/cli/commands/baseline.py +21 -12
- package/src/devcouncil/cli/commands/boot.py +218 -0
- package/src/devcouncil/cli/commands/campaign.py +302 -0
- package/src/devcouncil/cli/commands/check.py +225 -12
- package/src/devcouncil/cli/commands/config.py +221 -74
- package/src/devcouncil/cli/commands/cost.py +137 -28
- package/src/devcouncil/cli/commands/dashboard.py +12 -4
- package/src/devcouncil/cli/commands/debug_cmd.py +249 -0
- package/src/devcouncil/cli/commands/design.py +27 -17
- package/src/devcouncil/cli/commands/doctor.py +790 -8
- package/src/devcouncil/cli/commands/evidence.py +41 -20
- package/src/devcouncil/cli/commands/export.py +73 -0
- package/src/devcouncil/cli/commands/gaps.py +175 -0
- package/src/devcouncil/cli/commands/gated_write.py +76 -0
- package/src/devcouncil/cli/commands/go.py +220 -68
- package/src/devcouncil/cli/commands/graph_cmd.py +1192 -0
- package/src/devcouncil/cli/commands/handoff.py +45 -34
- package/src/devcouncil/cli/commands/hook.py +630 -85
- package/src/devcouncil/cli/commands/init.py +89 -30
- package/src/devcouncil/cli/commands/integrate.py +296 -1385
- package/src/devcouncil/cli/commands/lease.py +120 -0
- package/src/devcouncil/cli/commands/logs.py +12 -5
- package/src/devcouncil/cli/commands/lsp.py +40 -5
- package/src/devcouncil/cli/commands/map.py +317 -74
- package/src/devcouncil/cli/commands/mcp_server.py +12 -2
- package/src/devcouncil/cli/commands/okf.py +44 -6
- package/src/devcouncil/cli/commands/plan.py +184 -69
- package/src/devcouncil/cli/commands/prompt.py +26 -17
- package/src/devcouncil/cli/commands/provenance.py +79 -0
- package/src/devcouncil/cli/commands/repair.py +60 -49
- package/src/devcouncil/cli/commands/report.py +148 -40
- package/src/devcouncil/cli/commands/requirements.py +104 -0
- package/src/devcouncil/cli/commands/reset_demo_state.py +13 -4
- package/src/devcouncil/cli/commands/rollback.py +46 -35
- package/src/devcouncil/cli/commands/run.py +173 -8
- package/src/devcouncil/cli/commands/runs.py +298 -68
- package/src/devcouncil/cli/commands/scaffold.py +33 -12
- package/src/devcouncil/cli/commands/semantic.py +29 -14
- package/src/devcouncil/cli/commands/setup.py +103 -93
- package/src/devcouncil/cli/commands/shell.py +51 -42
- package/src/devcouncil/cli/commands/show.py +56 -42
- package/src/devcouncil/cli/commands/skills.py +29 -20
- package/src/devcouncil/cli/commands/status.py +80 -67
- package/src/devcouncil/cli/commands/task_gate.py +295 -0
- package/src/devcouncil/cli/commands/tasks.py +248 -19
- package/src/devcouncil/cli/commands/trace.py +14 -8
- package/src/devcouncil/cli/commands/verify.py +33 -8
- package/src/devcouncil/cli/commands/version.py +14 -6
- package/src/devcouncil/cli/commands/watch.py +56 -40
- package/src/devcouncil/cli/commands/watch_fs.py +30 -19
- package/src/devcouncil/cli/commands/wiki.py +278 -0
- package/src/devcouncil/cli/main.py +58 -1
- package/src/devcouncil/codeintel/__init__.py +16 -0
- package/src/devcouncil/codeintel/build_control.py +429 -0
- package/src/devcouncil/codeintel/build_worker.py +78 -0
- package/src/devcouncil/codeintel/debug/__init__.py +17 -0
- package/src/devcouncil/codeintel/debug/broker.py +114 -0
- package/src/devcouncil/codeintel/debug/broker_client.py +61 -0
- package/src/devcouncil/codeintel/debug/consent.py +36 -0
- package/src/devcouncil/codeintel/debug/discovery.py +132 -0
- package/src/devcouncil/codeintel/debug/fingerprint.py +85 -0
- package/src/devcouncil/codeintel/debug/protocol.py +259 -0
- package/src/devcouncil/codeintel/debug/python_trace_runner.py +81 -0
- package/src/devcouncil/codeintel/debug/session.py +238 -0
- package/src/devcouncil/codeintel/debug/tracing.py +201 -0
- package/src/devcouncil/codeintel/languages/__init__.py +17 -0
- package/src/devcouncil/codeintel/languages/generic_extractor.py +236 -0
- package/src/devcouncil/codeintel/languages/registry.py +149 -0
- package/src/devcouncil/codeintel/languages/workers.py +245 -0
- package/src/devcouncil/codeintel/query/__init__.py +5 -0
- package/src/devcouncil/codeintel/query/engine.py +289 -0
- package/src/devcouncil/codeintel/resolution/__init__.py +6 -0
- package/src/devcouncil/codeintel/resolution/abstract_state.py +301 -0
- package/src/devcouncil/codeintel/resolution/frameworks/__init__.py +33 -0
- package/src/devcouncil/codeintel/resolution/frameworks/base.py +46 -0
- package/src/devcouncil/codeintel/resolution/frameworks/di.py +56 -0
- package/src/devcouncil/codeintel/resolution/frameworks/events.py +45 -0
- package/src/devcouncil/codeintel/resolution/frameworks/routes.py +88 -0
- package/src/devcouncil/codeintel/resolution/semantic.py +887 -0
- package/src/devcouncil/codeintel/service.py +104 -0
- package/src/devcouncil/codeintel/store/__init__.py +15 -0
- package/src/devcouncil/codeintel/store/sqlite.py +1565 -0
- package/src/devcouncil/codeintel/sync/__init__.py +19 -0
- package/src/devcouncil/codeintel/sync/coordinator.py +430 -0
- package/src/devcouncil/codeintel/sync/incremental.py +484 -0
- package/src/devcouncil/codeintel/sync/lease.py +96 -0
- package/src/devcouncil/codeintel/sync/scope.py +98 -0
- package/src/devcouncil/council/__init__.py +4 -0
- package/src/devcouncil/council/prompts/__init__.py +4 -0
- package/src/devcouncil/domain/checkpoint_refs.py +17 -0
- package/src/devcouncil/domain/evidence.py +1 -0
- package/src/devcouncil/domain/gap.py +10 -0
- package/src/devcouncil/domain/requirement.py +5 -1
- package/src/devcouncil/domain/task.py +43 -2
- package/src/devcouncil/execution/checkpoints.py +25 -31
- package/src/devcouncil/execution/context_builder.py +15 -44
- package/src/devcouncil/execution/fs_watcher.py +64 -0
- package/src/devcouncil/execution/gated_write.py +203 -0
- package/src/devcouncil/execution/handoff.py +2 -1
- package/src/devcouncil/execution/hook_policy.py +19 -5
- package/src/devcouncil/execution/lease_ops.py +177 -0
- package/src/devcouncil/execution/lease_validation.py +71 -0
- package/src/devcouncil/execution/patch.py +3 -0
- package/src/devcouncil/execution/permissions.py +1 -0
- package/src/devcouncil/execution/policy_engine.py +205 -10
- package/src/devcouncil/execution/prompt_builder.py +278 -33
- package/src/devcouncil/execution/run_trace.py +356 -0
- package/src/devcouncil/execution/shell_session.py +46 -5
- package/src/devcouncil/execution/stop_gate.py +746 -0
- package/src/devcouncil/execution/stop_gate_history.py +113 -0
- package/src/devcouncil/execution/stop_gate_state.py +54 -0
- package/src/devcouncil/execution/stop_gate_verify_cache.py +69 -0
- package/src/devcouncil/execution/task_gate_ops.py +590 -0
- package/src/devcouncil/execution/task_runner.py +19 -0
- package/src/devcouncil/executors/advisor_tool.py +315 -0
- package/src/devcouncil/executors/agent_registry.py +125 -17
- package/src/devcouncil/executors/claude_sdk.py +376 -0
- package/src/devcouncil/executors/coding_cli.py +724 -25
- package/src/devcouncil/executors/mini_swe.py +50 -8
- package/src/devcouncil/executors/native/agent.py +224 -19
- package/src/devcouncil/executors/openhands.py +50 -8
- package/src/devcouncil/executors/transient_retry.py +99 -0
- package/src/devcouncil/gating/checks/clean_git.py +5 -2
- package/src/devcouncil/gating/checks/planned_files_check.py +38 -11
- package/src/devcouncil/gating/checks/secret_scan_check.py +2 -2
- package/src/devcouncil/gating/policy.py +46 -2
- package/src/devcouncil/indexing/ast_matcher.py +41 -4
- package/src/devcouncil/indexing/graph/__init__.py +78 -0
- package/src/devcouncil/indexing/graph/api_routes.py +522 -0
- package/src/devcouncil/indexing/graph/build.py +862 -0
- package/src/devcouncil/indexing/graph/cache.py +329 -0
- package/src/devcouncil/indexing/graph/communities.py +28 -0
- package/src/devcouncil/indexing/graph/cypher.py +107 -0
- package/src/devcouncil/indexing/graph/embeddings.py +194 -0
- package/src/devcouncil/indexing/graph/export.py +381 -0
- package/src/devcouncil/indexing/graph/export_links.py +81 -0
- package/src/devcouncil/indexing/graph/extract_python.py +307 -0
- package/src/devcouncil/indexing/graph/extract_ts.py +1205 -0
- package/src/devcouncil/indexing/graph/intel.py +668 -0
- package/src/devcouncil/indexing/graph/liveness.py +992 -0
- package/src/devcouncil/indexing/graph/okf_export.py +65 -0
- package/src/devcouncil/indexing/graph/pdg/__init__.py +67 -0
- package/src/devcouncil/indexing/graph/pdg/build.py +11 -0
- package/src/devcouncil/indexing/graph/pdg/cdg.py +41 -0
- package/src/devcouncil/indexing/graph/pdg/cfg.py +199 -0
- package/src/devcouncil/indexing/graph/pdg/query.py +21 -0
- package/src/devcouncil/indexing/graph/pdg/reaching_def.py +126 -0
- package/src/devcouncil/indexing/graph/pdg/schema.py +253 -0
- package/src/devcouncil/indexing/graph/pdg/taint.py +154 -0
- package/src/devcouncil/indexing/graph/query.py +302 -0
- package/src/devcouncil/indexing/graph/resolve.py +1020 -0
- package/src/devcouncil/indexing/graph/schema.py +103 -0
- package/src/devcouncil/indexing/graph_index.py +20 -29
- package/src/devcouncil/indexing/lsp.py +57 -25
- package/src/devcouncil/indexing/lsp_client.py +577 -0
- package/src/devcouncil/indexing/map_artifacts.py +355 -0
- package/src/devcouncil/indexing/map_refresh.py +141 -0
- package/src/devcouncil/indexing/repo_mapper.py +1509 -138
- package/src/devcouncil/indexing/semantic_index.py +12 -6
- package/src/devcouncil/indexing/subsystem_map.py +163 -0
- package/src/devcouncil/indexing/ts_imports.py +343 -0
- package/src/devcouncil/indexing/viz.py +964 -0
- package/src/devcouncil/indexing/walk.py +52 -0
- package/src/devcouncil/indexing/wiring.py +1776 -0
- package/src/devcouncil/integrations/actions.py +27 -4
- package/src/devcouncil/integrations/check.py +211 -16
- package/src/devcouncil/integrations/claude_assets.py +209 -12
- package/src/devcouncil/integrations/clients/__init__.py +1 -0
- package/src/devcouncil/integrations/clients/aider.py +52 -0
- package/src/devcouncil/integrations/clients/antigravity.py +87 -0
- package/src/devcouncil/integrations/clients/claude.py +339 -0
- package/src/devcouncil/integrations/clients/codex.py +39 -0
- package/src/devcouncil/integrations/clients/common.py +332 -0
- package/src/devcouncil/integrations/clients/cursor.py +164 -0
- package/src/devcouncil/integrations/clients/gemini.py +49 -0
- package/src/devcouncil/integrations/clients/grok.py +105 -0
- package/src/devcouncil/integrations/clients/hooks.py +500 -0
- package/src/devcouncil/integrations/clients/opencode.py +96 -0
- package/src/devcouncil/integrations/clients/warp.py +75 -0
- package/src/devcouncil/integrations/code_review_graph.py +2 -2
- package/src/devcouncil/integrations/github.py +73 -7
- package/src/devcouncil/integrations/integration_cli.py +197 -0
- package/src/devcouncil/integrations/mcp/handlers/__init__.py +1 -0
- package/src/devcouncil/integrations/mcp/handlers/ast_lsp.py +77 -0
- package/src/devcouncil/integrations/mcp/handlers/checkout.py +50 -0
- package/src/devcouncil/integrations/mcp/handlers/cli_gate.py +43 -0
- package/src/devcouncil/integrations/mcp/handlers/codeintel.py +182 -0
- package/src/devcouncil/integrations/mcp/handlers/debug.py +236 -0
- package/src/devcouncil/integrations/mcp/handlers/evidence.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/git.py +281 -0
- package/src/devcouncil/integrations/mcp/handlers/graph.py +34 -0
- package/src/devcouncil/integrations/mcp/handlers/handoff.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/knowledge.py +28 -0
- package/src/devcouncil/integrations/mcp/handlers/lease.py +70 -0
- package/src/devcouncil/integrations/mcp/handlers/live.py +108 -0
- package/src/devcouncil/integrations/mcp/handlers/map.py +676 -0
- package/src/devcouncil/integrations/mcp/handlers/next_task.py +35 -0
- package/src/devcouncil/integrations/mcp/handlers/policy.py +80 -0
- package/src/devcouncil/integrations/mcp/handlers/prompts.py +168 -0
- package/src/devcouncil/integrations/mcp/handlers/provenance.py +87 -0
- package/src/devcouncil/integrations/mcp/handlers/read.py +103 -0
- package/src/devcouncil/integrations/mcp/handlers/router_cache.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/run.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/runs.py +69 -0
- package/src/devcouncil/integrations/mcp/handlers/scope.py +56 -0
- package/src/devcouncil/integrations/mcp/handlers/status.py +199 -0
- package/src/devcouncil/integrations/mcp/handlers/task.py +88 -0
- package/src/devcouncil/integrations/mcp/handlers/tool_specs.py +922 -0
- package/src/devcouncil/integrations/mcp/handlers/trace.py +65 -0
- package/src/devcouncil/integrations/mcp/handlers/verify.py +45 -0
- package/src/devcouncil/integrations/mcp/handlers/wiki.py +53 -0
- package/src/devcouncil/integrations/mcp/handlers/write.py +69 -0
- package/src/devcouncil/integrations/mcp/server.py +265 -2391
- package/src/devcouncil/integrations/mcp/util.py +325 -0
- package/src/devcouncil/integrations/setup.py +152 -0
- package/src/devcouncil/knowledge/fetch.py +4 -0
- package/src/devcouncil/knowledge/knowledge_select.py +38 -0
- package/src/devcouncil/knowledge/okf.py +2 -1
- package/src/devcouncil/knowledge/resource_discovery.py +40 -0
- package/src/devcouncil/knowledge/wiki.py +643 -0
- package/src/devcouncil/knowledge/wiki_read.py +87 -0
- package/src/devcouncil/live/cards.py +7 -7
- package/src/devcouncil/live/models.py +4 -1
- package/src/devcouncil/live/reviewer.py +90 -11
- package/src/devcouncil/live/signals.py +4 -2
- package/src/devcouncil/live/summary.py +21 -3
- package/src/devcouncil/live/tasks.py +12 -3
- package/src/devcouncil/live/transcripts.py +69 -2
- package/src/devcouncil/llm/cache.py +5 -6
- package/src/devcouncil/llm/model_defaults.yaml +10 -10
- package/src/devcouncil/llm/provider.py +647 -73
- package/src/devcouncil/llm/router.py +271 -46
- package/src/devcouncil/llm/semantic_bridge.py +614 -0
- package/src/devcouncil/optimization/gepa_agent.py +6 -4
- package/src/devcouncil/optimization/skillopt.py +9 -5
- package/src/devcouncil/planning/arbiter_service.py +12 -3
- package/src/devcouncil/planning/correction_manifest.py +107 -10
- package/src/devcouncil/planning/plan_difficulty.py +69 -0
- package/src/devcouncil/planning/plan_service.py +5 -2
- package/src/devcouncil/planning/planned_files_reconcile.py +191 -0
- package/src/devcouncil/planning/prompt_enhancer_service.py +6 -5
- package/src/devcouncil/planning/question_conversion.py +56 -0
- package/src/devcouncil/planning/spec_service.py +9 -3
- package/src/devcouncil/repo/ci_scaffold.py +197 -1
- package/src/devcouncil/repo/gitignore.py +1 -2
- package/src/devcouncil/reporting/evidence_export.py +124 -0
- package/src/devcouncil/reporting/evidence_html.py +210 -0
- package/src/devcouncil/reporting/json_report.py +16 -12
- package/src/devcouncil/reporting/markdown_report.py +38 -9
- package/src/devcouncil/reporting/mcp_resources.py +142 -0
- package/src/devcouncil/reporting/report_builder.py +40 -4
- package/src/devcouncil/reporting/task_provenance.py +42 -0
- package/src/devcouncil/reporting/verdict.py +75 -0
- package/src/devcouncil/skills/library/README.md +1 -0
- package/src/devcouncil/skills/library/devcouncil-hero-loop.md +109 -0
- package/src/devcouncil/skills/library/devcouncil-verification.md +109 -0
- package/src/devcouncil/skills/library/devcouncil.md +93 -0
- package/src/devcouncil/skills/registry.py +43 -12
- package/src/devcouncil/storage/db.py +57 -11
- package/src/devcouncil/storage/models.py +6 -0
- package/src/devcouncil/storage/native.py +5 -3
- package/src/devcouncil/storage/repositories.py +50 -18
- package/src/devcouncil/telemetry/context.py +28 -0
- package/src/devcouncil/telemetry/cost.py +4 -5
- package/src/devcouncil/telemetry/logging_setup.py +78 -11
- package/src/devcouncil/telemetry/model_pricing.yaml +7 -0
- package/src/devcouncil/telemetry/stages.py +27 -2
- package/src/devcouncil/telemetry/tracker.py +50 -13
- package/src/devcouncil/ui/dashboard.py +120 -8
- package/src/devcouncil/utils/fsio.py +58 -0
- package/src/devcouncil/utils/git_snapshot.py +112 -0
- package/src/devcouncil/utils/json_persist.py +53 -0
- package/src/devcouncil/utils/proc.py +89 -0
- package/src/devcouncil/verification/acceptance_compiler.py +36 -13
- package/src/devcouncil/verification/ad_hoc_check.py +95 -3
- package/src/devcouncil/verification/checks/__init__.py +41 -0
- package/src/devcouncil/verification/checks/acceptance.py +39 -0
- package/src/devcouncil/verification/checks/acceptance_corpus.py +194 -0
- package/src/devcouncil/verification/checks/acceptance_evidence.py +239 -0
- package/src/devcouncil/verification/checks/command_evidence.py +148 -0
- package/src/devcouncil/verification/checks/compiled_acceptance.py +179 -0
- package/src/devcouncil/verification/checks/corpus_stale.py +124 -0
- package/src/devcouncil/verification/checks/corpus_verification.py +9 -0
- package/src/devcouncil/verification/checks/dead_symbols.py +360 -0
- package/src/devcouncil/verification/checks/diff_coverage_gate.py +101 -0
- package/src/devcouncil/verification/checks/doc_code_ref.py +79 -0
- package/src/devcouncil/verification/checks/liveness_ratchet.py +336 -0
- package/src/devcouncil/verification/checks/orphan_diff.py +104 -0
- package/src/devcouncil/verification/checks/planned_files.py +98 -0
- package/src/devcouncil/verification/checks/semantic_diff.py +241 -0
- package/src/devcouncil/verification/checks/stale_map.py +80 -0
- package/src/devcouncil/verification/checks/stub_scan.py +71 -0
- package/src/devcouncil/verification/checks/subsystem_boundary.py +103 -0
- package/src/devcouncil/verification/checks/wiring.py +216 -0
- package/src/devcouncil/verification/claims/__init__.py +23 -0
- package/src/devcouncil/verification/claims/checks.py +395 -0
- package/src/devcouncil/verification/claims/mapper.py +168 -0
- package/src/devcouncil/verification/claims/models.py +39 -0
- package/src/devcouncil/verification/claims/transcript.py +92 -0
- package/src/devcouncil/verification/claims/verdict.py +88 -0
- package/src/devcouncil/verification/command_evidence.py +170 -0
- package/src/devcouncil/verification/command_malformation.py +147 -0
- package/src/devcouncil/verification/command_runner.py +164 -0
- package/src/devcouncil/verification/coverage_measurement.py +292 -0
- package/src/devcouncil/verification/diff_coverage.py +151 -0
- package/src/devcouncil/verification/difficulty.py +296 -0
- package/src/devcouncil/verification/effort_heuristics.py +178 -0
- package/src/devcouncil/verification/gap_ids.py +63 -0
- package/src/devcouncil/verification/gate_cache.py +194 -0
- package/src/devcouncil/verification/gate_selector.py +344 -0
- package/src/devcouncil/verification/git_diff_fallback.py +272 -0
- package/src/devcouncil/verification/implementation_reviewer.py +13 -0
- package/src/devcouncil/verification/incremental_check.py +241 -0
- package/src/devcouncil/verification/next_actions.py +60 -1
- package/src/devcouncil/verification/rigor_analytics.py +130 -0
- package/src/devcouncil/verification/sandbox.py +38 -11
- package/src/devcouncil/verification/stub_detector.py +369 -0
- package/src/devcouncil/verification/test_resolver.py +67 -1
- package/src/devcouncil/verification/verifier.py +137 -1666
- package/src/devcouncil/verification/verify_orchestration.py +610 -0
- package/src/devcouncil/verification/verify_setup.py +176 -0
- package/src/devcouncil/verification/wiki_refresh.py +208 -0
- package/src/semantic_layer/__init__.py +58 -0
- package/src/semantic_layer/benchmark.py +75 -0
- package/src/semantic_layer/cache.py +290 -0
- package/src/semantic_layer/compressor.py +137 -0
- package/src/semantic_layer/config.py +75 -0
- package/src/semantic_layer/embeddings.py +69 -0
- package/src/semantic_layer/llm_backends.py +99 -0
- package/src/semantic_layer/pipeline.py +111 -0
- package/src/semantic_layer/router.py +128 -0
- package/src/semantic_layer/tuner.py +72 -0
- package/uv.lock +973 -9
- package/src/devcouncil/artifacts/migrations.py +0 -20
- package/src/devcouncil/artifacts/schemas.py +0 -23
- package/src/devcouncil/artifacts/serializer.py +0 -21
- package/src/devcouncil/integrations/gitnexus.py +0 -70
- package/src/devcouncil/integrations/graphify.py +0 -34
|
@@ -8,10 +8,12 @@ import subprocess
|
|
|
8
8
|
import sys
|
|
9
9
|
from collections import Counter, defaultdict
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from typing import Dict, List, Set, Tuple
|
|
11
|
+
from typing import Dict, List, Optional, Set, Tuple, cast
|
|
12
12
|
|
|
13
13
|
from pydantic import BaseModel, Field
|
|
14
14
|
|
|
15
|
+
from devcouncil.indexing.graph.cache import PARSE_CACHE_VERSION
|
|
16
|
+
from devcouncil.indexing.graph.schema import CodeGraph
|
|
15
17
|
from devcouncil.indexing.lsp import LspInspector
|
|
16
18
|
|
|
17
19
|
logger = logging.getLogger(__name__)
|
|
@@ -54,20 +56,57 @@ class RepoMap(BaseModel):
|
|
|
54
56
|
# file -> the files that import it (reverse import edges, capped per file). Lets a
|
|
55
57
|
# prompt show the blast radius of changing a file without re-parsing the repo.
|
|
56
58
|
dependents: Dict[str, List[str]] = Field(default_factory=dict)
|
|
59
|
+
# Full importer counts when ``dependents[path]`` was truncated by ``_DEPENDENTS_MAX``.
|
|
60
|
+
# Absent / empty when no path was truncated. Agents must treat listed dependents as a
|
|
61
|
+
# sample whenever ``dependents_total[path] > len(dependents[path])``.
|
|
62
|
+
dependents_total: Dict[str, int] = Field(default_factory=dict)
|
|
57
63
|
# Freshness fingerprints captured at generation: the git HEAD the map was built from
|
|
58
64
|
# and a hash of the tracked file set. Consumers compare against the current repo to
|
|
59
65
|
# detect a stale map before trusting its structure.
|
|
60
66
|
generated_head: str = ""
|
|
61
67
|
indexed_hash: str = ""
|
|
68
|
+
# sha1 over sorted (path, size, mtime_ns) so plain content edits mark the map stale.
|
|
69
|
+
# Legacy maps without this field stay non-stale (no false alarms).
|
|
70
|
+
content_fingerprint: str = ""
|
|
71
|
+
# True when the last map write used lean/degraded graph fallback. Consumers must
|
|
72
|
+
# treat this as stale until a successful graph-backed refresh clears it.
|
|
73
|
+
graph_degraded: bool = False
|
|
74
|
+
graph_degraded_reason: str = ""
|
|
62
75
|
lsp: Dict[str, object] = Field(default_factory=dict)
|
|
63
76
|
# Optional dependency-vulnerability findings. Populated only when `dev map` is
|
|
64
77
|
# run with SCA explicitly enabled (off by default so the map stays fast and
|
|
65
78
|
# offline-by-default); empty otherwise.
|
|
66
79
|
dependency_risks: List[Dict[str, str]] = Field(default_factory=list)
|
|
80
|
+
# Liveness artifact (computed by default; omit with --no-liveness).
|
|
81
|
+
entry_roots: List[str] = Field(default_factory=list)
|
|
82
|
+
unwired_candidates: List[str] = Field(default_factory=list)
|
|
83
|
+
unreachable_files: List[str] = Field(default_factory=list)
|
|
84
|
+
dead_symbol_candidates: List[str] = Field(default_factory=list)
|
|
85
|
+
# True when production entry roots were empty — unreachable BFS was skipped
|
|
86
|
+
# (fail-soft) so unreachable_files is not meaningful.
|
|
87
|
+
liveness_unreachable_unreliable: bool = False
|
|
88
|
+
liveness_meta: Dict[str, object] = Field(default_factory=dict)
|
|
89
|
+
# Top call-flow processes from the code graph (entry-root BFS), when available.
|
|
90
|
+
processes: List[Dict[str, object]] = Field(default_factory=list)
|
|
67
91
|
|
|
68
92
|
class RepoMapper:
|
|
69
|
-
def __init__(self, project_root: Path):
|
|
70
|
-
self.project_root = project_root
|
|
93
|
+
def __init__(self, project_root: Path | None = None):
|
|
94
|
+
self.project_root = project_root or Path.cwd()
|
|
95
|
+
self._DEPENDENTS_MAX = type(self)._DEPENDENTS_MAX
|
|
96
|
+
self._LIVENESS_CAP = type(self)._LIVENESS_CAP
|
|
97
|
+
self._js_alias_cache: Optional[List[Tuple[str, List[str]]]] = None
|
|
98
|
+
try:
|
|
99
|
+
from devcouncil.app.config import load_config
|
|
100
|
+
|
|
101
|
+
indexing = load_config(self.project_root).indexing
|
|
102
|
+
self._DEPENDENTS_MAX = min(
|
|
103
|
+
int(indexing.repo_map_dependents_cap), self.max_dependents_per_file
|
|
104
|
+
)
|
|
105
|
+
self._LIVENESS_CAP = min(
|
|
106
|
+
int(indexing.repo_map_liveness_cap), self.max_map_size
|
|
107
|
+
)
|
|
108
|
+
except Exception:
|
|
109
|
+
logger.debug("using default repository-map caps", exc_info=True)
|
|
71
110
|
# Common source-root prefix of the repo's primary code (e.g. "src/pkg"),
|
|
72
111
|
# computed once per map_repo run. Drives generic, non-DevCouncil subsystem
|
|
73
112
|
# inference. None until computed.
|
|
@@ -78,17 +117,29 @@ class RepoMapper:
|
|
|
78
117
|
# Import edges (importer -> imported), computed once per map_repo run and reused
|
|
79
118
|
# by subsystem inference, important-file ranking, and the dependents index.
|
|
80
119
|
self._edges: List[Tuple[str, str]] | None = None
|
|
120
|
+
self._last_code_graph: CodeGraph | None = None
|
|
81
121
|
# Cache of config-file contents (package.json, pyproject.toml, ...) so framework
|
|
82
122
|
# and test-command detection don't each re-read the same files from disk.
|
|
83
123
|
self._config_file_cache: Dict[str, str] = {}
|
|
84
124
|
|
|
85
125
|
def _read_config_file(self, name: str) -> str:
|
|
86
|
-
"""Read a repo-root config file once and cache its contents for reuse.
|
|
126
|
+
"""Read a repo-root config file once and cache its contents for reuse.
|
|
127
|
+
|
|
128
|
+
Unreadable or non-UTF-8 config files must not fail the whole map.
|
|
129
|
+
"""
|
|
87
130
|
if name not in self._config_file_cache:
|
|
88
|
-
|
|
131
|
+
try:
|
|
132
|
+
self._config_file_cache[name] = (self.project_root / name).read_text(
|
|
133
|
+
encoding="utf-8", errors="replace"
|
|
134
|
+
)
|
|
135
|
+
except OSError:
|
|
136
|
+
self._config_file_cache[name] = ""
|
|
89
137
|
return self._config_file_cache[name]
|
|
90
138
|
|
|
91
|
-
_DEPENDENTS_MAX =
|
|
139
|
+
_DEPENDENTS_MAX = 1_024 # serialized dependents per file
|
|
140
|
+
_LIVENESS_CAP = 20_000 # serialized entries per liveness debt list
|
|
141
|
+
max_dependents_per_file = 4_096
|
|
142
|
+
max_map_size = 100_000 # hard safety ceiling for each liveness debt list
|
|
92
143
|
|
|
93
144
|
_LANGUAGE_BY_EXTENSION = {
|
|
94
145
|
".py": "python",
|
|
@@ -131,6 +182,10 @@ class RepoMapper:
|
|
|
131
182
|
"src/devcouncil/ui": "Dashboard and lightweight UI helpers",
|
|
132
183
|
"src/devcouncil/utils": "Shared utilities and redaction helpers",
|
|
133
184
|
"src/devcouncil/verification": "Verification gates and implementation review",
|
|
185
|
+
"src/devcouncil/campaign": "Multi-agent campaign orchestration and roles",
|
|
186
|
+
"src/devcouncil/knowledge": "Knowledge sources, OKF, and design docs",
|
|
187
|
+
"src/devcouncil/skills": "Skill registry and matching",
|
|
188
|
+
"src/devcouncil/optimization": "Prompt and skill optimization",
|
|
134
189
|
"docs": "Repository documentation",
|
|
135
190
|
"tests": "Automated tests",
|
|
136
191
|
"scripts": "Maintenance and smoke-test scripts",
|
|
@@ -173,19 +228,19 @@ class RepoMapper:
|
|
|
173
228
|
],
|
|
174
229
|
),
|
|
175
230
|
"src/devcouncil/indexing": (
|
|
176
|
-
"Repo mapping, AST matching, semantic snapshots, and
|
|
231
|
+
"Repo mapping, AST matching, semantic snapshots, and optional live LSP client.",
|
|
177
232
|
[
|
|
178
233
|
"src/devcouncil/indexing/repo_mapper.py",
|
|
234
|
+
"src/devcouncil/indexing/wiring.py",
|
|
179
235
|
"src/devcouncil/indexing/ast_matcher.py",
|
|
180
236
|
"src/devcouncil/indexing/semantic_index.py",
|
|
181
237
|
"src/devcouncil/indexing/lsp.py",
|
|
238
|
+
"src/devcouncil/indexing/lsp_client.py",
|
|
182
239
|
],
|
|
183
240
|
),
|
|
184
241
|
"src/devcouncil/integrations": (
|
|
185
242
|
"External system integrations and MCP/Graph adapters.",
|
|
186
243
|
[
|
|
187
|
-
"src/devcouncil/integrations/gitnexus.py",
|
|
188
|
-
"src/devcouncil/integrations/graphify.py",
|
|
189
244
|
"src/devcouncil/integrations/mcp/server.py",
|
|
190
245
|
"src/devcouncil/integrations/github.py",
|
|
191
246
|
],
|
|
@@ -298,6 +353,37 @@ class RepoMapper:
|
|
|
298
353
|
"src/devcouncil/utils/redaction.py",
|
|
299
354
|
],
|
|
300
355
|
),
|
|
356
|
+
"src/devcouncil/campaign": (
|
|
357
|
+
"Multi-agent campaign orchestration, roles, mailbox, and watchers.",
|
|
358
|
+
[
|
|
359
|
+
"src/devcouncil/campaign/orchestrator.py",
|
|
360
|
+
"src/devcouncil/campaign/roles.py",
|
|
361
|
+
"src/devcouncil/campaign/mailbox.py",
|
|
362
|
+
"src/devcouncil/campaign/watcher.py",
|
|
363
|
+
],
|
|
364
|
+
),
|
|
365
|
+
"src/devcouncil/knowledge": (
|
|
366
|
+
"Knowledge sources, OKF bundles, design docs, and skill bridging.",
|
|
367
|
+
[
|
|
368
|
+
"src/devcouncil/knowledge/sources.py",
|
|
369
|
+
"src/devcouncil/knowledge/okf.py",
|
|
370
|
+
"src/devcouncil/knowledge/knowledge_select.py",
|
|
371
|
+
"src/devcouncil/knowledge/design.py",
|
|
372
|
+
],
|
|
373
|
+
),
|
|
374
|
+
"src/devcouncil/skills": (
|
|
375
|
+
"Skill registry and matching for agent capability selection.",
|
|
376
|
+
[
|
|
377
|
+
"src/devcouncil/skills/registry.py",
|
|
378
|
+
],
|
|
379
|
+
),
|
|
380
|
+
"src/devcouncil/optimization": (
|
|
381
|
+
"Prompt/skill optimization loops (GEPA, SkillOpt).",
|
|
382
|
+
[
|
|
383
|
+
"src/devcouncil/optimization/gepa_agent.py",
|
|
384
|
+
"src/devcouncil/optimization/skillopt.py",
|
|
385
|
+
],
|
|
386
|
+
),
|
|
301
387
|
}
|
|
302
388
|
|
|
303
389
|
_SUBSYSTEM_CRITICAL_MAX = 6
|
|
@@ -411,6 +497,27 @@ class RepoMapper:
|
|
|
411
497
|
"src/devcouncil/executors",
|
|
412
498
|
"src/devcouncil/llm",
|
|
413
499
|
],
|
|
500
|
+
"src/devcouncil/campaign": [
|
|
501
|
+
"src/devcouncil/cli",
|
|
502
|
+
"src/devcouncil/execution",
|
|
503
|
+
"src/devcouncil/llm",
|
|
504
|
+
"src/devcouncil/skills",
|
|
505
|
+
],
|
|
506
|
+
"src/devcouncil/knowledge": [
|
|
507
|
+
"src/devcouncil/indexing",
|
|
508
|
+
"src/devcouncil/skills",
|
|
509
|
+
"src/devcouncil/cli",
|
|
510
|
+
],
|
|
511
|
+
"src/devcouncil/skills": [
|
|
512
|
+
"src/devcouncil/knowledge",
|
|
513
|
+
"src/devcouncil/campaign",
|
|
514
|
+
"src/devcouncil/execution",
|
|
515
|
+
],
|
|
516
|
+
"src/devcouncil/optimization": [
|
|
517
|
+
"src/devcouncil/skills",
|
|
518
|
+
"src/devcouncil/llm",
|
|
519
|
+
"src/devcouncil/campaign",
|
|
520
|
+
],
|
|
414
521
|
}
|
|
415
522
|
|
|
416
523
|
_SUBSYSTEM_HANDOFFS: Dict[str, List[str]] = {
|
|
@@ -452,7 +559,6 @@ class RepoMapper:
|
|
|
452
559
|
"src/devcouncil/integrations": [
|
|
453
560
|
"integrations/mcp/server.py -> live/reviewer.py",
|
|
454
561
|
"integrations/code_review_graph.py -> live/cards.py",
|
|
455
|
-
"integrations/gitnexus.py -> reporting/report_builder.py",
|
|
456
562
|
],
|
|
457
563
|
"src/devcouncil/live": [
|
|
458
564
|
"live/summary.py -> live/cards.py",
|
|
@@ -484,6 +590,24 @@ class RepoMapper:
|
|
|
484
590
|
"src/devcouncil/ui": [
|
|
485
591
|
"ui/dashboard.py -> live/summary.py",
|
|
486
592
|
],
|
|
593
|
+
"src/devcouncil/campaign": [
|
|
594
|
+
"campaign/orchestrator.py -> campaign/mailbox.py",
|
|
595
|
+
"campaign/orchestrator.py -> execution/task_runner.py",
|
|
596
|
+
"campaign/roles.py -> skills/registry.py",
|
|
597
|
+
],
|
|
598
|
+
"src/devcouncil/knowledge": [
|
|
599
|
+
"knowledge/okf.py -> indexing/graph/export.py",
|
|
600
|
+
"knowledge/sources.py -> skills/registry.py",
|
|
601
|
+
"knowledge/knowledge_select.py -> execution/prompt_builder.py",
|
|
602
|
+
],
|
|
603
|
+
"src/devcouncil/skills": [
|
|
604
|
+
"skills/registry.py -> knowledge/sources.py",
|
|
605
|
+
"skills/registry.py -> campaign/roles.py",
|
|
606
|
+
],
|
|
607
|
+
"src/devcouncil/optimization": [
|
|
608
|
+
"optimization/skillopt.py -> skills/registry.py",
|
|
609
|
+
"optimization/gepa_agent.py -> llm/router.py",
|
|
610
|
+
],
|
|
487
611
|
}
|
|
488
612
|
|
|
489
613
|
_SUBSYSTEM_ROLE_FILES: Dict[str, List[Tuple[str, List[str]]]] = {
|
|
@@ -505,18 +629,16 @@ class RepoMapper:
|
|
|
505
629
|
("mapping", ["indexing/repo_mapper.py"]),
|
|
506
630
|
("ast", ["indexing/ast_matcher.py"]),
|
|
507
631
|
("semantic", ["indexing/semantic_index.py"]),
|
|
508
|
-
# Detection
|
|
509
|
-
("lsp", ["indexing/lsp.py"]),
|
|
510
|
-
#
|
|
511
|
-
("graph", ["indexing/graph_index.py"]),
|
|
632
|
+
# Detection + optional live client (lsp_client.py); off by default.
|
|
633
|
+
("lsp", ["indexing/lsp.py", "indexing/lsp_client.py"]),
|
|
634
|
+
# graph_index.py is a deprecated artifact-graph shim; live graph is indexing/graph/.
|
|
635
|
+
("graph", ["indexing/graph_index.py", "indexing/graph/"]),
|
|
512
636
|
],
|
|
513
637
|
"src/devcouncil/integrations": [
|
|
514
638
|
("vcs", ["integrations/github.py"]),
|
|
515
|
-
("graphify", ["integrations/graphify.py"]),
|
|
516
639
|
("code_review", ["integrations/code_review_graph.py"]),
|
|
517
640
|
("comments", ["integrations/pr_comments.py"]),
|
|
518
641
|
("mcp", ["integrations/mcp/server.py"]),
|
|
519
|
-
("third_party", ["integrations/gitnexus.py"]),
|
|
520
642
|
],
|
|
521
643
|
"src/devcouncil/cli": [
|
|
522
644
|
("entrypoints", ["cli/main.py"]),
|
|
@@ -533,8 +655,8 @@ class RepoMapper:
|
|
|
533
655
|
],
|
|
534
656
|
"src/devcouncil/artifacts": [
|
|
535
657
|
("graph", ["artifacts/graph.py"]),
|
|
536
|
-
("coverage", ["artifacts/coverage.py", "artifacts/
|
|
537
|
-
("schema", ["artifacts/
|
|
658
|
+
("coverage", ["artifacts/coverage.py", "artifacts/validators.py"]),
|
|
659
|
+
("schema", ["artifacts/graph.py", "artifacts/coverage.py"]),
|
|
538
660
|
("validation", ["artifacts/validators.py"]),
|
|
539
661
|
],
|
|
540
662
|
"src/devcouncil/executors": [
|
|
@@ -606,6 +728,25 @@ class RepoMapper:
|
|
|
606
728
|
"src/devcouncil/utils": [
|
|
607
729
|
("redaction", ["utils/redaction.py"]),
|
|
608
730
|
],
|
|
731
|
+
"src/devcouncil/campaign": [
|
|
732
|
+
("orchestration", ["campaign/orchestrator.py"]),
|
|
733
|
+
("roles", ["campaign/roles.py"]),
|
|
734
|
+
("mailbox", ["campaign/mailbox.py"]),
|
|
735
|
+
("watch", ["campaign/watcher.py", "campaign/notify.py"]),
|
|
736
|
+
],
|
|
737
|
+
"src/devcouncil/knowledge": [
|
|
738
|
+
("sources", ["knowledge/sources.py"]),
|
|
739
|
+
("okf", ["knowledge/okf.py"]),
|
|
740
|
+
("design", ["knowledge/design.py", "knowledge/design_conformance.py"]),
|
|
741
|
+
("select", ["knowledge/knowledge_select.py", "knowledge/skill_bridge.py"]),
|
|
742
|
+
],
|
|
743
|
+
"src/devcouncil/skills": [
|
|
744
|
+
("registry", ["skills/registry.py"]),
|
|
745
|
+
],
|
|
746
|
+
"src/devcouncil/optimization": [
|
|
747
|
+
("gepa", ["optimization/gepa_agent.py"]),
|
|
748
|
+
("skillopt", ["optimization/skillopt.py"]),
|
|
749
|
+
],
|
|
609
750
|
}
|
|
610
751
|
|
|
611
752
|
_COMMAND_SUMMARIES = {
|
|
@@ -646,11 +787,15 @@ class RepoMapper:
|
|
|
646
787
|
"README.md": "Project overview and usage entrypoint",
|
|
647
788
|
"architecture.md": "Top-level architecture overview",
|
|
648
789
|
"cli-reference.md": "CLI command reference",
|
|
790
|
+
"coding-cli-integration.md": "Coding CLI tiers, hooks, and stop gate",
|
|
791
|
+
"hero-loop.md": "Certified Claude Code MCP closed loop",
|
|
792
|
+
"code-graph.md": "Repo map and symbol code graph",
|
|
793
|
+
"corpus.md": "Corpus side index and verify gates",
|
|
794
|
+
"model-routing.md": "Provider and model routing setup",
|
|
649
795
|
"quickstart.md": "First-run installation and workflow",
|
|
650
796
|
"workflow.md": "Manual sidecar workflow guide",
|
|
651
797
|
"security.md": "Security and privacy model",
|
|
652
798
|
"project-status.md": "Subsystem maturity snapshot",
|
|
653
|
-
"roadmap.md": "Planned work and roadmap",
|
|
654
799
|
}
|
|
655
800
|
|
|
656
801
|
def _language_for_file(self, path: str) -> str | None:
|
|
@@ -732,10 +877,6 @@ class RepoMapper:
|
|
|
732
877
|
return "Trace logging and event persistence"
|
|
733
878
|
if normalized == "src/devcouncil/live/reviewer.py":
|
|
734
879
|
return "Live review service"
|
|
735
|
-
if normalized == "src/devcouncil/integrations/gitnexus.py":
|
|
736
|
-
return "GitNexus integration shim"
|
|
737
|
-
if normalized == "src/devcouncil/integrations/graphify.py":
|
|
738
|
-
return "Graphify integration shim"
|
|
739
880
|
if normalized.startswith("src/devcouncil/"):
|
|
740
881
|
area = "/".join(parts[:3]) if len(parts) >= 3 else "src/devcouncil"
|
|
741
882
|
return self._AREA_SUMMARIES.get(area, f"{area} subsystem")
|
|
@@ -939,44 +1080,139 @@ class RepoMapper:
|
|
|
939
1080
|
comps = comps[:-1] # `from pkg.mod import name` -> try pkg.mod, then pkg
|
|
940
1081
|
return None
|
|
941
1082
|
|
|
1083
|
+
def _ancestor_init_files(self, target: str, py_file_set: Set[str]) -> List[str]:
|
|
1084
|
+
"""Existing ancestor package ``__init__.py`` files between ``target`` and repo root.
|
|
1085
|
+
|
|
1086
|
+
Importing a submodule executes each ancestor package ``__init__``; emit those
|
|
1087
|
+
inbound edges so package inits are not falsely flagged unwired/unreachable.
|
|
1088
|
+
"""
|
|
1089
|
+
parts = target.replace("\\", "/").split("/")
|
|
1090
|
+
if not parts:
|
|
1091
|
+
return []
|
|
1092
|
+
if parts[-1] == "__init__.py":
|
|
1093
|
+
dir_parts = parts[:-2] # ancestors above this package
|
|
1094
|
+
else:
|
|
1095
|
+
dir_parts = parts[:-1]
|
|
1096
|
+
out: List[str] = []
|
|
1097
|
+
for i in range(len(dir_parts), 0, -1):
|
|
1098
|
+
init = "/".join(dir_parts[:i]) + "/__init__.py"
|
|
1099
|
+
if init in py_file_set and init != target:
|
|
1100
|
+
out.append(init)
|
|
1101
|
+
return out
|
|
1102
|
+
|
|
1103
|
+
# Single owner: graph.cache.PARSE_CACHE_VERSION (v4 = + import_details).
|
|
1104
|
+
_PARSE_CACHE_VERSION = PARSE_CACHE_VERSION
|
|
1105
|
+
_JS_SUFFIXES = frozenset({".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"})
|
|
1106
|
+
|
|
1107
|
+
def _parse_cache_path(self) -> Path:
|
|
1108
|
+
from devcouncil.indexing.graph.cache import cache_path
|
|
1109
|
+
|
|
1110
|
+
return cache_path(self.project_root)
|
|
1111
|
+
|
|
1112
|
+
def _load_parse_cache(self) -> Dict[str, Dict[str, object]]:
|
|
1113
|
+
"""Delegate to graph.cache (single version + merge policy)."""
|
|
1114
|
+
from devcouncil.indexing.graph.cache import load_parse_cache
|
|
1115
|
+
|
|
1116
|
+
return cast(Dict[str, Dict[str, object]], load_parse_cache(self.project_root))
|
|
1117
|
+
|
|
1118
|
+
def _save_parse_cache(self, files: Dict[str, Dict[str, object]]) -> None:
|
|
1119
|
+
"""Delegate to graph.cache."""
|
|
1120
|
+
from devcouncil.indexing.graph.cache import save_parse_cache
|
|
1121
|
+
|
|
1122
|
+
save_parse_cache(self.project_root, cast(Dict[str, Dict[str, object]], files))
|
|
1123
|
+
|
|
1124
|
+
@staticmethod
|
|
1125
|
+
def _is_js_source_path(path: str) -> bool:
|
|
1126
|
+
return Path(path).suffix.lower() in RepoMapper._JS_SUFFIXES
|
|
1127
|
+
|
|
1128
|
+
def _merge_parse_cache(
|
|
1129
|
+
self,
|
|
1130
|
+
updates: Dict[str, Dict[str, object]],
|
|
1131
|
+
managed: Set[str],
|
|
1132
|
+
) -> None:
|
|
1133
|
+
"""Delegate to graph.cache merge (preserves sibling language/field keys)."""
|
|
1134
|
+
from devcouncil.indexing.graph.cache import merge_parse_cache
|
|
1135
|
+
|
|
1136
|
+
merge_parse_cache(
|
|
1137
|
+
self.project_root,
|
|
1138
|
+
cast(Dict[str, Dict[str, object]], updates),
|
|
1139
|
+
managed,
|
|
1140
|
+
)
|
|
1141
|
+
|
|
1142
|
+
def _extract_python_import_modules(self, rel: str, source: str) -> List[str]:
|
|
1143
|
+
"""Raw module strings referenced by ``rel``'s import statements, with relative
|
|
1144
|
+
imports expanded against the file's package path. A pure function of the file's
|
|
1145
|
+
path + content — which is what makes the result safe to cache by sha256."""
|
|
1146
|
+
tree = ast.parse(source)
|
|
1147
|
+
pkg_parts = rel[:-3].replace("/", ".").split(".") # importer's dotted path
|
|
1148
|
+
modules: List[str] = []
|
|
1149
|
+
for node in ast.walk(tree):
|
|
1150
|
+
if isinstance(node, ast.Import):
|
|
1151
|
+
modules.extend(alias.name for alias in node.names)
|
|
1152
|
+
elif isinstance(node, ast.ImportFrom):
|
|
1153
|
+
if node.level:
|
|
1154
|
+
base = pkg_parts[: -node.level] if node.level <= len(pkg_parts) else []
|
|
1155
|
+
base_mod = ".".join(base + ([node.module] if node.module else []))
|
|
1156
|
+
else:
|
|
1157
|
+
base_mod = node.module or ""
|
|
1158
|
+
if base_mod:
|
|
1159
|
+
modules.append(base_mod)
|
|
1160
|
+
# `from pkg import sub` / `from . import sub` may import a SUBMODULE
|
|
1161
|
+
# file, not just a symbol — resolve each name as a candidate module so
|
|
1162
|
+
# those edges aren't silently dropped.
|
|
1163
|
+
for alias in node.names:
|
|
1164
|
+
if alias.name and alias.name != "*":
|
|
1165
|
+
modules.append(f"{base_mod}.{alias.name}" if base_mod else alias.name)
|
|
1166
|
+
return modules
|
|
1167
|
+
|
|
942
1168
|
def _python_import_edges(self, files: List[str]) -> List[Tuple[str, str]]:
|
|
943
|
-
"""Resolve Python import statements into (importer, imported) file edges.
|
|
1169
|
+
"""Resolve Python import statements into (importer, imported) file edges.
|
|
1170
|
+
|
|
1171
|
+
Extraction (ast.parse + walk) is cached in .devcouncil/cache/repo_map_parse.json
|
|
1172
|
+
keyed by each file's sha256, so unchanged files skip re-parsing across runs.
|
|
1173
|
+
Resolution against the module index always runs fresh — it depends on the
|
|
1174
|
+
CURRENT file set, not on any single file's content."""
|
|
944
1175
|
py_files = [f for f in self._code_files(files) if f.endswith(".py")]
|
|
945
1176
|
if not py_files:
|
|
946
1177
|
return []
|
|
1178
|
+
py_file_set = set(py_files)
|
|
947
1179
|
index = self._module_suffix_index(py_files)
|
|
1180
|
+
cache = self._load_parse_cache()
|
|
1181
|
+
fresh: Dict[str, Dict[str, object]] = {}
|
|
948
1182
|
edges: List[Tuple[str, str]] = []
|
|
949
1183
|
seen: Set[Tuple[str, str]] = set() # dedupe so in-degree isn't inflated by repeats
|
|
950
1184
|
for rel in py_files:
|
|
951
1185
|
try:
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
except (OSError, SyntaxError, ValueError):
|
|
1186
|
+
raw = (self.project_root / rel).read_bytes()
|
|
1187
|
+
except OSError:
|
|
955
1188
|
continue
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
if
|
|
978
|
-
seen.add((rel,
|
|
979
|
-
edges.append((rel,
|
|
1189
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
1190
|
+
entry = cache.get(rel)
|
|
1191
|
+
cached_modules = entry.get("modules") if isinstance(entry, dict) and entry.get("sha256") == digest else None
|
|
1192
|
+
if isinstance(cached_modules, list):
|
|
1193
|
+
modules = [m for m in cached_modules if isinstance(m, str)]
|
|
1194
|
+
else:
|
|
1195
|
+
try:
|
|
1196
|
+
modules = self._extract_python_import_modules(
|
|
1197
|
+
rel, raw.decode("utf-8", errors="replace")
|
|
1198
|
+
)
|
|
1199
|
+
except (SyntaxError, ValueError):
|
|
1200
|
+
# Unparseable content contributes no edges; cache the empty result
|
|
1201
|
+
# so the same broken content isn't re-parsed every run.
|
|
1202
|
+
modules = []
|
|
1203
|
+
fresh[rel] = {"sha256": digest, "modules": modules}
|
|
1204
|
+
for module in modules:
|
|
1205
|
+
target = self._resolve_module(module, index)
|
|
1206
|
+
if not target:
|
|
1207
|
+
continue
|
|
1208
|
+
targets = [target, *self._ancestor_init_files(target, py_file_set)]
|
|
1209
|
+
for dest in targets:
|
|
1210
|
+
if dest != rel and (rel, dest) not in seen:
|
|
1211
|
+
seen.add((rel, dest))
|
|
1212
|
+
edges.append((rel, dest))
|
|
1213
|
+
# Preserve JS/TS cache entries; prune deleted Python files from this pass.
|
|
1214
|
+
managed = {k for k in cache if k.endswith(".py")} | set(py_files)
|
|
1215
|
+
self._merge_parse_cache(fresh, managed)
|
|
980
1216
|
return edges
|
|
981
1217
|
|
|
982
1218
|
# Module specifiers in import/require statements: import ... from "x"; require("x");
|
|
@@ -985,68 +1221,475 @@ class RepoMapper:
|
|
|
985
1221
|
r"""(?:import|export)\s[^'"]*?from\s*['"](?P<spec>[^'"]+)['"]"""
|
|
986
1222
|
r"""|(?:require|import)\s*\(\s*['"](?P<spec2>[^'"]+)['"]\s*\)"""
|
|
987
1223
|
)
|
|
1224
|
+
# Vite/webpack worker + import.meta.url asset loads (e.g. mupdf-worker.ts).
|
|
1225
|
+
_JS_WORKER_URL_RE = re.compile(
|
|
1226
|
+
r"""(?:new\s+(?:Worker|SharedWorker)\s*\(\s*)?"""
|
|
1227
|
+
r"""new\s+URL\s*\(\s*['"](?P<spec>[^'"]+)['"]\s*,\s*import\.meta\.url"""
|
|
1228
|
+
)
|
|
988
1229
|
_JS_BARE_IMPORT_RE = re.compile(r"""^\s*import\s*['"](?P<spec>[^'"]+)['"]""")
|
|
989
1230
|
_JS_RESOLVE_EXTS = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs")
|
|
1231
|
+
_TSCONFIG_WALK_CAP = 200
|
|
990
1232
|
_GO_IMPORT_BLOCK_RE = re.compile(r"import\s*\((?P<body>[^)]*)\)", re.DOTALL)
|
|
991
1233
|
_GO_IMPORT_SINGLE_RE = re.compile(r"""^\s*import\s+(?:[A-Za-z_.]\w*\s+)?['"](?P<spec>[^'"]+)['"]""")
|
|
992
1234
|
_GO_IMPORT_SPEC_RE = re.compile(r"""['"](?P<spec>[^'"]+)['"]""")
|
|
993
1235
|
_GO_MODULE_RE = re.compile(r"^\s*module\s+(?P<mod>\S+)", re.MULTILINE)
|
|
994
1236
|
|
|
995
1237
|
def _resolve_js_spec(self, importer: str, spec: str, file_set: Set[str]) -> str | None:
|
|
996
|
-
"""Resolve a
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1238
|
+
"""Resolve a TS/JS import specifier to a repo file.
|
|
1239
|
+
|
|
1240
|
+
Handles relative specs (``./x`` / ``../y``) and tsconfig/jsconfig path
|
|
1241
|
+
aliases (``@/x``, ``~/x``, custom). Bare node_modules packages return None.
|
|
1242
|
+
"""
|
|
1243
|
+
if spec.startswith("."):
|
|
1244
|
+
return self._resolve_js_relative(importer, spec, file_set)
|
|
1245
|
+
return self._resolve_js_alias(spec, file_set)
|
|
1246
|
+
|
|
1247
|
+
def _normalize_js_path(self, target: str) -> str:
|
|
1248
|
+
parts: List[str] = []
|
|
1249
|
+
for comp in target.replace("\\", "/").split("/"):
|
|
1250
|
+
if comp in ("", "."):
|
|
1251
|
+
continue
|
|
1252
|
+
if comp == "..":
|
|
1253
|
+
if parts:
|
|
1254
|
+
parts.pop()
|
|
1255
|
+
continue
|
|
1256
|
+
parts.append(comp)
|
|
1257
|
+
return "/".join(parts)
|
|
1258
|
+
|
|
1259
|
+
def _normalize_js_alias_target(self, target: str) -> str:
|
|
1260
|
+
"""Like ``_normalize_js_path`` but keeps leading ``../`` segments intact."""
|
|
1006
1261
|
parts: List[str] = []
|
|
1007
|
-
|
|
1262
|
+
leading_dots = 0
|
|
1263
|
+
for comp in target.replace("\\", "/").split("/"):
|
|
1008
1264
|
if comp in ("", "."):
|
|
1009
1265
|
continue
|
|
1010
1266
|
if comp == "..":
|
|
1011
1267
|
if parts:
|
|
1012
1268
|
parts.pop()
|
|
1269
|
+
else:
|
|
1270
|
+
leading_dots += 1
|
|
1013
1271
|
continue
|
|
1014
1272
|
parts.append(comp)
|
|
1015
|
-
|
|
1273
|
+
prefix = "/".join([".."] * leading_dots)
|
|
1274
|
+
rest = "/".join(parts)
|
|
1275
|
+
if prefix and rest:
|
|
1276
|
+
return f"{prefix}/{rest}"
|
|
1277
|
+
return prefix or rest
|
|
1278
|
+
|
|
1279
|
+
def _probe_js_candidates(self, norm: str, file_set: Set[str]) -> str | None:
|
|
1280
|
+
"""Resolve a normalized path against ``file_set``.
|
|
1281
|
+
|
|
1282
|
+
Handles extensionless specs (``./auth`` → ``auth.ts``) and TypeScript
|
|
1283
|
+
ESM suffix rewriting (``./auth.js`` → ``auth.ts`` / ``auth.tsx``).
|
|
1284
|
+
"""
|
|
1016
1285
|
if not norm:
|
|
1017
1286
|
return None
|
|
1018
1287
|
candidates = [norm]
|
|
1019
1288
|
candidates += [f"{norm}{ext}" for ext in self._JS_RESOLVE_EXTS]
|
|
1020
1289
|
candidates += [f"{norm}/index{ext}" for ext in self._JS_RESOLVE_EXTS]
|
|
1290
|
+
# TS/Node ESM: import './foo.js' resolves to foo.ts / foo.tsx on disk.
|
|
1291
|
+
suffix = Path(norm).suffix.lower()
|
|
1292
|
+
if suffix in self._JS_SUFFIXES:
|
|
1293
|
+
stem = norm[: -len(suffix)]
|
|
1294
|
+
candidates += [f"{stem}{ext}" for ext in self._JS_RESOLVE_EXTS]
|
|
1295
|
+
candidates += [f"{stem}/index{ext}" for ext in self._JS_RESOLVE_EXTS]
|
|
1021
1296
|
for cand in candidates:
|
|
1022
1297
|
if cand in file_set:
|
|
1023
1298
|
return cand
|
|
1024
1299
|
return None
|
|
1025
1300
|
|
|
1301
|
+
def _resolve_js_relative(self, importer: str, spec: str, file_set: Set[str]) -> str | None:
|
|
1302
|
+
base = Path(importer).parent
|
|
1303
|
+
try:
|
|
1304
|
+
target = (base / spec).as_posix()
|
|
1305
|
+
except Exception:
|
|
1306
|
+
return None
|
|
1307
|
+
return self._probe_js_candidates(self._normalize_js_path(target), file_set)
|
|
1308
|
+
|
|
1309
|
+
def _load_js_path_aliases(self) -> List[Tuple[str, List[str]]]:
|
|
1310
|
+
"""Parse tsconfig/jsconfig path aliases across extends + project references.
|
|
1311
|
+
|
|
1312
|
+
Merges ``compilerOptions.paths`` (and ``baseUrl``) from:
|
|
1313
|
+
- Root ``tsconfig.json`` / ``jsconfig.json`` / ``tsconfig.base.json``
|
|
1314
|
+
- Each config's ``extends`` chain (child overrides parent)
|
|
1315
|
+
- ``references[].path`` project configs (and their extends)
|
|
1316
|
+
|
|
1317
|
+
Returns ``(pattern_prefix, [target_prefixes])`` sorted longest-key first.
|
|
1318
|
+
Cached per mapper instance. Never raises.
|
|
1319
|
+
"""
|
|
1320
|
+
cached = getattr(self, "_js_alias_cache", None)
|
|
1321
|
+
if cached is not None:
|
|
1322
|
+
return cast(List[Tuple[str, List[str]]], cached)
|
|
1323
|
+
rules: List[Tuple[str, List[str]]] = []
|
|
1324
|
+
seen_configs: Set[str] = set()
|
|
1325
|
+
|
|
1326
|
+
def _strip_json_comments(raw: str) -> str:
|
|
1327
|
+
raw = re.sub(r"/\*.*?\*/", "", raw, flags=re.DOTALL)
|
|
1328
|
+
return re.sub(r"(?m)^\s*//.*?$", "", raw)
|
|
1329
|
+
|
|
1330
|
+
def _read_config(path: Path) -> dict | None:
|
|
1331
|
+
try:
|
|
1332
|
+
key = path.resolve().as_posix()
|
|
1333
|
+
except OSError:
|
|
1334
|
+
key = path.as_posix()
|
|
1335
|
+
if key in seen_configs:
|
|
1336
|
+
return None
|
|
1337
|
+
seen_configs.add(key)
|
|
1338
|
+
if not path.is_file():
|
|
1339
|
+
return None
|
|
1340
|
+
try:
|
|
1341
|
+
raw = _strip_json_comments(path.read_text(encoding="utf-8", errors="replace"))
|
|
1342
|
+
data = json.loads(raw)
|
|
1343
|
+
except Exception:
|
|
1344
|
+
return None
|
|
1345
|
+
return data if isinstance(data, dict) else None
|
|
1346
|
+
|
|
1347
|
+
def _merge_extends(path: Path, data: dict) -> dict:
|
|
1348
|
+
"""Return compilerOptions with parent ``extends`` merged under child."""
|
|
1349
|
+
extends = data.get("extends")
|
|
1350
|
+
parent_co: dict = {}
|
|
1351
|
+
if isinstance(extends, str) and extends:
|
|
1352
|
+
# Node-style: bare @scoped packages ignored; relative/absolute only.
|
|
1353
|
+
if extends.startswith("."):
|
|
1354
|
+
parent_path = (path.parent / extends).resolve()
|
|
1355
|
+
if not parent_path.suffix:
|
|
1356
|
+
# extends may omit .json
|
|
1357
|
+
for suffix in (".json", ""):
|
|
1358
|
+
candidate = Path(str(parent_path) + suffix) if suffix else parent_path
|
|
1359
|
+
if candidate.is_file():
|
|
1360
|
+
parent_path = candidate
|
|
1361
|
+
break
|
|
1362
|
+
parent_data = _read_config(parent_path)
|
|
1363
|
+
if parent_data:
|
|
1364
|
+
parent_co = _merge_extends(parent_path, parent_data)
|
|
1365
|
+
child_co = data.get("compilerOptions") or {}
|
|
1366
|
+
if not isinstance(child_co, dict):
|
|
1367
|
+
child_co = {}
|
|
1368
|
+
merged = dict(parent_co)
|
|
1369
|
+
for k, v in child_co.items():
|
|
1370
|
+
if k == "paths" and isinstance(v, dict) and isinstance(merged.get("paths"), dict):
|
|
1371
|
+
paths = dict(merged["paths"])
|
|
1372
|
+
paths.update(v)
|
|
1373
|
+
merged["paths"] = paths
|
|
1374
|
+
else:
|
|
1375
|
+
merged[k] = v
|
|
1376
|
+
return merged
|
|
1377
|
+
|
|
1378
|
+
def _rules_from_co(co: dict, config_dir: Path) -> None:
|
|
1379
|
+
if not isinstance(co, dict):
|
|
1380
|
+
return
|
|
1381
|
+
base_url = str(co.get("baseUrl") or ".").replace("\\", "/").rstrip("/")
|
|
1382
|
+
if base_url in (".", ""):
|
|
1383
|
+
base_url = ""
|
|
1384
|
+
# Paths are relative to the config file's directory (TS spec), then baseUrl.
|
|
1385
|
+
try:
|
|
1386
|
+
config_rel = config_dir.relative_to(self.project_root).as_posix()
|
|
1387
|
+
except ValueError:
|
|
1388
|
+
config_rel = ""
|
|
1389
|
+
if config_rel in (".", ""):
|
|
1390
|
+
config_prefix = ""
|
|
1391
|
+
else:
|
|
1392
|
+
config_prefix = config_rel
|
|
1393
|
+
|
|
1394
|
+
paths = co.get("paths") or {}
|
|
1395
|
+
if not isinstance(paths, dict):
|
|
1396
|
+
return
|
|
1397
|
+
for key, targets in paths.items():
|
|
1398
|
+
if not isinstance(key, str):
|
|
1399
|
+
continue
|
|
1400
|
+
target_list = targets if isinstance(targets, list) else [targets]
|
|
1401
|
+
resolved_targets: List[str] = []
|
|
1402
|
+
for t in target_list:
|
|
1403
|
+
if not isinstance(t, str):
|
|
1404
|
+
continue
|
|
1405
|
+
t_norm = t.replace("\\", "/")
|
|
1406
|
+
if t_norm.endswith("/*"):
|
|
1407
|
+
t_norm = t_norm[:-2]
|
|
1408
|
+
elif t_norm.endswith("*"):
|
|
1409
|
+
t_norm = t_norm[:-1]
|
|
1410
|
+
# Resolve relative to config dir + baseUrl. Preserve leading
|
|
1411
|
+
# ``../`` (Phase 0: never lstrip/collapse away parent refs).
|
|
1412
|
+
pieces: List[str] = []
|
|
1413
|
+
if config_prefix:
|
|
1414
|
+
pieces.append(config_prefix)
|
|
1415
|
+
if base_url:
|
|
1416
|
+
pieces.append(base_url)
|
|
1417
|
+
if t_norm:
|
|
1418
|
+
pieces.append(t_norm)
|
|
1419
|
+
joined = "/".join(p for p in pieces if p)
|
|
1420
|
+
joined = self._normalize_js_alias_target(joined)
|
|
1421
|
+
while joined.startswith("./"):
|
|
1422
|
+
joined = joined[2:]
|
|
1423
|
+
resolved_targets.append(joined.rstrip("/"))
|
|
1424
|
+
if not resolved_targets:
|
|
1425
|
+
continue
|
|
1426
|
+
pattern = key
|
|
1427
|
+
if pattern.endswith("/*"):
|
|
1428
|
+
pattern = pattern[:-2]
|
|
1429
|
+
elif pattern.endswith("*"):
|
|
1430
|
+
pattern = pattern[:-1]
|
|
1431
|
+
rules.append((pattern, resolved_targets))
|
|
1432
|
+
|
|
1433
|
+
def _ingest(path: Path) -> None:
|
|
1434
|
+
data = _read_config(path)
|
|
1435
|
+
if not data:
|
|
1436
|
+
return
|
|
1437
|
+
co = _merge_extends(path, data)
|
|
1438
|
+
_rules_from_co(co, path.parent)
|
|
1439
|
+
refs = data.get("references") or []
|
|
1440
|
+
if isinstance(refs, list):
|
|
1441
|
+
for ref in refs:
|
|
1442
|
+
if not isinstance(ref, dict):
|
|
1443
|
+
continue
|
|
1444
|
+
ref_path = ref.get("path")
|
|
1445
|
+
if not isinstance(ref_path, str) or not ref_path:
|
|
1446
|
+
continue
|
|
1447
|
+
target = (path.parent / ref_path).resolve()
|
|
1448
|
+
if target.is_dir():
|
|
1449
|
+
for name in ("tsconfig.json", "jsconfig.json"):
|
|
1450
|
+
candidate = target / name
|
|
1451
|
+
if candidate.is_file():
|
|
1452
|
+
_ingest(candidate)
|
|
1453
|
+
break
|
|
1454
|
+
elif target.is_file():
|
|
1455
|
+
_ingest(target)
|
|
1456
|
+
else:
|
|
1457
|
+
# path may omit .json
|
|
1458
|
+
for suffix in (".json", "/tsconfig.json"):
|
|
1459
|
+
candidate = Path(str(target) + suffix) if suffix.startswith(".") else Path(str(target) + suffix)
|
|
1460
|
+
if candidate.is_file():
|
|
1461
|
+
_ingest(candidate)
|
|
1462
|
+
break
|
|
1463
|
+
|
|
1464
|
+
try:
|
|
1465
|
+
for name in ("tsconfig.json", "jsconfig.json", "tsconfig.base.json"):
|
|
1466
|
+
root_cfg = self.project_root / name
|
|
1467
|
+
if root_cfg.is_file():
|
|
1468
|
+
_ingest(root_cfg)
|
|
1469
|
+
# Nested package tsconfigs (monorepos without root project references).
|
|
1470
|
+
# Caps keep large trees bounded; node_modules / dist are skipped.
|
|
1471
|
+
_TSCONFIG_NAMES = {
|
|
1472
|
+
"tsconfig.json",
|
|
1473
|
+
"jsconfig.json",
|
|
1474
|
+
"tsconfig.base.json",
|
|
1475
|
+
"tsconfig.app.json",
|
|
1476
|
+
"tsconfig.node.json",
|
|
1477
|
+
}
|
|
1478
|
+
file_set = getattr(self, "_last_file_set", None) or set()
|
|
1479
|
+
nested_manifests: List[str] = []
|
|
1480
|
+
if file_set:
|
|
1481
|
+
nested_manifests = sorted(
|
|
1482
|
+
p
|
|
1483
|
+
for p in file_set
|
|
1484
|
+
if p.rsplit("/", 1)[-1] in _TSCONFIG_NAMES
|
|
1485
|
+
)[:200]
|
|
1486
|
+
# Always also walk the tree: git file lists often omit tsconfig.json,
|
|
1487
|
+
# which previously left monorepo ``apps/*/tsconfig.json`` undiscovered.
|
|
1488
|
+
# Prune vendored trees before capping: node_modules commonly holds
|
|
1489
|
+
# hundreds of tsconfig.json, and capping the unfiltered glob silently
|
|
1490
|
+
# dropped real ``packages/*/tsconfig.json`` alias rules.
|
|
1491
|
+
try:
|
|
1492
|
+
skip_dirs = {"node_modules", "dist", "build", ".git", "target"}
|
|
1493
|
+
walked: List[str] = []
|
|
1494
|
+
for dirpath, dirnames, filenames in os.walk(self.project_root):
|
|
1495
|
+
dirnames[:] = sorted(d for d in dirnames if d not in skip_dirs)
|
|
1496
|
+
for fname in filenames:
|
|
1497
|
+
if fname != "jsconfig.json" and not (
|
|
1498
|
+
fname.startswith("tsconfig") and fname.endswith(".json")
|
|
1499
|
+
):
|
|
1500
|
+
continue
|
|
1501
|
+
try:
|
|
1502
|
+
rel = (
|
|
1503
|
+
(Path(dirpath) / fname)
|
|
1504
|
+
.relative_to(self.project_root)
|
|
1505
|
+
.as_posix()
|
|
1506
|
+
)
|
|
1507
|
+
except ValueError:
|
|
1508
|
+
continue
|
|
1509
|
+
walked.append(rel)
|
|
1510
|
+
if len(walked) >= self._TSCONFIG_WALK_CAP:
|
|
1511
|
+
break
|
|
1512
|
+
for rel in sorted(walked)[: self._TSCONFIG_WALK_CAP]:
|
|
1513
|
+
if rel not in nested_manifests:
|
|
1514
|
+
nested_manifests.append(rel)
|
|
1515
|
+
except Exception:
|
|
1516
|
+
logger.debug("nested tsconfig walk failed", exc_info=True)
|
|
1517
|
+
nested_manifests = nested_manifests[:200]
|
|
1518
|
+
for rel in nested_manifests:
|
|
1519
|
+
_ingest(self.project_root / rel)
|
|
1520
|
+
# package.json name → source dir (best-effort monorepo/workspace).
|
|
1521
|
+
try:
|
|
1522
|
+
pkg_text = (self.project_root / "package.json").read_text(
|
|
1523
|
+
encoding="utf-8", errors="replace"
|
|
1524
|
+
)
|
|
1525
|
+
pkg = json.loads(pkg_text)
|
|
1526
|
+
pkg_name = pkg.get("name")
|
|
1527
|
+
if isinstance(pkg_name, str) and pkg_name:
|
|
1528
|
+
for candidate in ("src", "lib", "app", "."):
|
|
1529
|
+
probe = "src" if candidate == "src" else candidate
|
|
1530
|
+
if any(
|
|
1531
|
+
f == probe or f.startswith(probe + "/")
|
|
1532
|
+
for f in getattr(self, "_last_file_set", set())
|
|
1533
|
+
) or (self.project_root / probe).is_dir():
|
|
1534
|
+
rules.append((pkg_name, [probe if probe != "." else ""]))
|
|
1535
|
+
break
|
|
1536
|
+
except Exception:
|
|
1537
|
+
pass
|
|
1538
|
+
# Deduplicate identical (pattern, targets) preserving longest-first sort.
|
|
1539
|
+
dedup: Dict[str, List[str]] = {}
|
|
1540
|
+
for pattern, targets in rules:
|
|
1541
|
+
existing = dedup.get(pattern)
|
|
1542
|
+
if existing is None:
|
|
1543
|
+
dedup[pattern] = list(targets)
|
|
1544
|
+
else:
|
|
1545
|
+
for t in targets:
|
|
1546
|
+
if t not in existing:
|
|
1547
|
+
existing.append(t)
|
|
1548
|
+
rules = [(p, ts) for p, ts in dedup.items()]
|
|
1549
|
+
rules.sort(key=lambda r: len(r[0]), reverse=True)
|
|
1550
|
+
except Exception:
|
|
1551
|
+
logger.debug("JS path-alias load failed", exc_info=True)
|
|
1552
|
+
rules = []
|
|
1553
|
+
self._js_alias_cache = rules
|
|
1554
|
+
return rules
|
|
1555
|
+
|
|
1556
|
+
def _resolve_js_alias(self, spec: str, file_set: Set[str]) -> str | None:
|
|
1557
|
+
rules = self._load_js_path_aliases()
|
|
1558
|
+
for pattern, targets in rules:
|
|
1559
|
+
rest = ""
|
|
1560
|
+
if spec == pattern:
|
|
1561
|
+
rest = ""
|
|
1562
|
+
elif pattern and spec.startswith(pattern + "/"):
|
|
1563
|
+
rest = spec[len(pattern) + 1 :]
|
|
1564
|
+
else:
|
|
1565
|
+
continue
|
|
1566
|
+
for target_base in targets:
|
|
1567
|
+
if rest:
|
|
1568
|
+
candidate = f"{target_base}/{rest}" if target_base else rest
|
|
1569
|
+
else:
|
|
1570
|
+
candidate = target_base
|
|
1571
|
+
hit = self._probe_js_candidates(self._normalize_js_path(candidate), file_set)
|
|
1572
|
+
if hit:
|
|
1573
|
+
return hit
|
|
1574
|
+
return None
|
|
1575
|
+
|
|
1576
|
+
def _extract_js_import_specs(self, source: str) -> List[str]:
|
|
1577
|
+
"""Raw import/require/export-from/worker-URL specifiers in ``source``.
|
|
1578
|
+
|
|
1579
|
+
Pure function of content — safe to cache by sha256. Resolution against
|
|
1580
|
+
the file set runs fresh.
|
|
1581
|
+
"""
|
|
1582
|
+
specs: List[str] = []
|
|
1583
|
+
for m in self._JS_IMPORT_RE.finditer(source):
|
|
1584
|
+
spec = m.group("spec") or m.group("spec2")
|
|
1585
|
+
if spec:
|
|
1586
|
+
specs.append(spec)
|
|
1587
|
+
for m in self._JS_WORKER_URL_RE.finditer(source):
|
|
1588
|
+
spec = m.group("spec")
|
|
1589
|
+
if spec:
|
|
1590
|
+
specs.append(spec)
|
|
1591
|
+
for line in source.splitlines():
|
|
1592
|
+
bare = self._JS_BARE_IMPORT_RE.match(line)
|
|
1593
|
+
if bare:
|
|
1594
|
+
specs.append(bare.group("spec"))
|
|
1595
|
+
return specs
|
|
1596
|
+
|
|
1597
|
+
_JS_REEXPORT_RE = re.compile(
|
|
1598
|
+
r"""export\s+(?:\*|\{[^}]*\})\s+from\s*['"](?P<spec>[^'"]+)['"]"""
|
|
1599
|
+
)
|
|
1600
|
+
|
|
1601
|
+
def _extract_js_reexport_specs(self, source: str) -> List[str]:
|
|
1602
|
+
"""``export * from`` / ``export {…} from`` targets (barrel re-exports)."""
|
|
1603
|
+
return [m.group("spec") for m in self._JS_REEXPORT_RE.finditer(source)]
|
|
1604
|
+
|
|
1605
|
+
def _follow_js_reexports(
|
|
1606
|
+
self,
|
|
1607
|
+
importer: str,
|
|
1608
|
+
target: str,
|
|
1609
|
+
file_set: Set[str],
|
|
1610
|
+
*,
|
|
1611
|
+
depth: int = 0,
|
|
1612
|
+
seen: Set[str] | None = None,
|
|
1613
|
+
) -> List[str]:
|
|
1614
|
+
"""Follow barrel re-exports from ``target`` up to a small hop limit.
|
|
1615
|
+
|
|
1616
|
+
Returns additional files reachable via ``export … from`` so an importer of
|
|
1617
|
+
a barrel also depends on the re-exported modules.
|
|
1618
|
+
"""
|
|
1619
|
+
if depth >= 4:
|
|
1620
|
+
return []
|
|
1621
|
+
visited = seen if seen is not None else set()
|
|
1622
|
+
if target in visited:
|
|
1623
|
+
return []
|
|
1624
|
+
visited.add(target)
|
|
1625
|
+
try:
|
|
1626
|
+
source = (self.project_root / target).read_text(encoding="utf-8", errors="replace")
|
|
1627
|
+
except OSError:
|
|
1628
|
+
return []
|
|
1629
|
+
extra: List[str] = []
|
|
1630
|
+
for spec in self._extract_js_reexport_specs(source):
|
|
1631
|
+
resolved = self._resolve_js_spec(target, spec, file_set)
|
|
1632
|
+
if not resolved or resolved == importer or resolved == target:
|
|
1633
|
+
continue
|
|
1634
|
+
extra.append(resolved)
|
|
1635
|
+
extra.extend(
|
|
1636
|
+
self._follow_js_reexports(
|
|
1637
|
+
importer, resolved, file_set, depth=depth + 1, seen=visited
|
|
1638
|
+
)
|
|
1639
|
+
)
|
|
1640
|
+
return extra
|
|
1641
|
+
|
|
1026
1642
|
def _js_import_edges(self, files: List[str], file_set: Set[str]) -> List[Tuple[str, str]]:
|
|
1027
|
-
"""Resolve TS/JS relative import/require/export-from edges to repo files.
|
|
1028
|
-
|
|
1643
|
+
"""Resolve TS/JS relative + alias import/require/export-from edges to repo files.
|
|
1644
|
+
|
|
1645
|
+
Specifier extraction is cached in ``.devcouncil/cache/repo_map_parse.json``
|
|
1646
|
+
keyed by each file's sha256 (same store as Python ``modules``). Resolution
|
|
1647
|
+
against the current file set always runs fresh. Barrel ``export * from`` /
|
|
1648
|
+
``export {x} from`` targets are followed so importers of an index also
|
|
1649
|
+
depend on the re-exported modules.
|
|
1650
|
+
"""
|
|
1651
|
+
self._last_file_set = file_set
|
|
1652
|
+
# Alias rules may have been cached before the file set was known (root-only
|
|
1653
|
+
# tsconfigs). Force a reload so nested apps/*/tsconfig.json are included.
|
|
1654
|
+
self._js_alias_cache = None
|
|
1655
|
+
js_files = [f for f in files if self._is_js_source_path(f)]
|
|
1656
|
+
if not js_files:
|
|
1657
|
+
return []
|
|
1658
|
+
cache = self._load_parse_cache()
|
|
1659
|
+
fresh: Dict[str, Dict[str, object]] = {}
|
|
1029
1660
|
edges: List[Tuple[str, str]] = []
|
|
1030
1661
|
seen: Set[Tuple[str, str]] = set()
|
|
1031
1662
|
for rel in js_files:
|
|
1032
1663
|
try:
|
|
1033
|
-
|
|
1664
|
+
raw = (self.project_root / rel).read_bytes()
|
|
1034
1665
|
except OSError:
|
|
1035
1666
|
continue
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1667
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
1668
|
+
entry = cache.get(rel)
|
|
1669
|
+
cached_specs = (
|
|
1670
|
+
entry.get("specs")
|
|
1671
|
+
if isinstance(entry, dict) and entry.get("sha256") == digest
|
|
1672
|
+
else None
|
|
1673
|
+
)
|
|
1674
|
+
if isinstance(cached_specs, list):
|
|
1675
|
+
specs = [s for s in cached_specs if isinstance(s, str)]
|
|
1676
|
+
else:
|
|
1677
|
+
specs = self._extract_js_import_specs(
|
|
1678
|
+
raw.decode("utf-8", errors="replace")
|
|
1679
|
+
)
|
|
1680
|
+
fresh[rel] = {"sha256": digest, "specs": specs}
|
|
1045
1681
|
for spec in specs:
|
|
1046
1682
|
target = self._resolve_js_spec(rel, spec, file_set)
|
|
1047
|
-
if target
|
|
1048
|
-
|
|
1049
|
-
|
|
1683
|
+
if not target or target == rel:
|
|
1684
|
+
continue
|
|
1685
|
+
to_add = [target]
|
|
1686
|
+
to_add.extend(self._follow_js_reexports(rel, target, file_set))
|
|
1687
|
+
for dest in to_add:
|
|
1688
|
+
if dest != rel and (rel, dest) not in seen:
|
|
1689
|
+
seen.add((rel, dest))
|
|
1690
|
+
edges.append((rel, dest))
|
|
1691
|
+
managed = {k for k in cache if self._is_js_source_path(k)} | set(js_files)
|
|
1692
|
+
self._merge_parse_cache(fresh, managed)
|
|
1050
1693
|
return edges
|
|
1051
1694
|
|
|
1052
1695
|
def _go_module_prefix(self, file_set: Set[str]) -> str | None:
|
|
@@ -1061,42 +1704,166 @@ class RepoMapper:
|
|
|
1061
1704
|
return m.group("mod").strip()
|
|
1062
1705
|
return None
|
|
1063
1706
|
|
|
1707
|
+
def _extract_go_import_specs_fallback(self, source: str) -> List[str]:
|
|
1708
|
+
specs: List[str] = []
|
|
1709
|
+
for block in self._GO_IMPORT_BLOCK_RE.finditer(source):
|
|
1710
|
+
for sm in self._GO_IMPORT_SPEC_RE.finditer(block.group("body")):
|
|
1711
|
+
specs.append(sm.group("spec"))
|
|
1712
|
+
for line in source.splitlines():
|
|
1713
|
+
single = self._GO_IMPORT_SINGLE_RE.match(line)
|
|
1714
|
+
if single:
|
|
1715
|
+
specs.append(single.group("spec"))
|
|
1716
|
+
return specs
|
|
1717
|
+
|
|
1064
1718
|
def _go_import_edges(self, files: List[str], file_set: Set[str]) -> List[Tuple[str, str]]:
|
|
1065
|
-
"""Resolve Go import paths
|
|
1066
|
-
|
|
1719
|
+
"""Resolve Go import paths under the module prefix to *all* package .go files.
|
|
1720
|
+
|
|
1721
|
+
File-level (not representative-file) so liveness/dependents treat each
|
|
1722
|
+
member of an imported package as wired. Prefer tree-sitter extraction
|
|
1723
|
+
when available; fall back to regex.
|
|
1724
|
+
"""
|
|
1067
1725
|
module = self._go_module_prefix(file_set)
|
|
1068
1726
|
if not module:
|
|
1069
1727
|
return []
|
|
1070
1728
|
go_files = [f for f in files if f.endswith(".go")]
|
|
1071
1729
|
if not go_files:
|
|
1072
1730
|
return []
|
|
1073
|
-
# package dir -> .go files in it (excluding _test.go, which aren't imported).
|
|
1074
1731
|
pkg_files: Dict[str, List[str]] = defaultdict(list)
|
|
1075
1732
|
for f in go_files:
|
|
1076
1733
|
if f.endswith("_test.go"):
|
|
1077
1734
|
continue
|
|
1078
1735
|
pkg_files[Path(f).parent.as_posix()].append(f)
|
|
1736
|
+
|
|
1737
|
+
from devcouncil.indexing.ts_imports import extract_go_import_specs
|
|
1738
|
+
|
|
1079
1739
|
edges: List[Tuple[str, str]] = []
|
|
1080
1740
|
seen: Set[Tuple[str, str]] = set()
|
|
1741
|
+
|
|
1742
|
+
# Same-package co-membership: Go compiles all files in a directory as one
|
|
1743
|
+
# package, so members wire each other even without import statements.
|
|
1744
|
+
for members in pkg_files.values():
|
|
1745
|
+
if len(members) < 2:
|
|
1746
|
+
continue
|
|
1747
|
+
for a in members:
|
|
1748
|
+
for b in members:
|
|
1749
|
+
if a != b and (a, b) not in seen:
|
|
1750
|
+
seen.add((a, b))
|
|
1751
|
+
edges.append((a, b))
|
|
1752
|
+
|
|
1081
1753
|
for rel in go_files:
|
|
1082
1754
|
try:
|
|
1083
1755
|
source = (self.project_root / rel).read_text(encoding="utf-8", errors="replace")
|
|
1084
1756
|
except OSError:
|
|
1085
1757
|
continue
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
for sm in self._GO_IMPORT_SPEC_RE.finditer(block.group("body")):
|
|
1089
|
-
specs.append(sm.group("spec"))
|
|
1090
|
-
for line in source.splitlines():
|
|
1091
|
-
single = self._GO_IMPORT_SINGLE_RE.match(line)
|
|
1092
|
-
if single:
|
|
1093
|
-
specs.append(single.group("spec"))
|
|
1758
|
+
ts_specs = extract_go_import_specs(source)
|
|
1759
|
+
specs = ts_specs if ts_specs is not None else self._extract_go_import_specs_fallback(source)
|
|
1094
1760
|
for spec in specs:
|
|
1095
1761
|
if spec != module and not spec.startswith(module + "/"):
|
|
1096
1762
|
continue # external/stdlib package
|
|
1097
1763
|
rel_pkg = spec[len(module):].lstrip("/")
|
|
1098
1764
|
target_dir = rel_pkg if rel_pkg else "."
|
|
1099
|
-
for target in pkg_files.get(target_dir,
|
|
1765
|
+
for target in sorted(pkg_files.get(target_dir, ())):
|
|
1766
|
+
if target != rel and (rel, target) not in seen:
|
|
1767
|
+
seen.add((rel, target))
|
|
1768
|
+
edges.append((rel, target))
|
|
1769
|
+
return edges
|
|
1770
|
+
|
|
1771
|
+
def _rust_crate_root(self, importer: str, file_set: Set[str]) -> str:
|
|
1772
|
+
"""Directory containing lib.rs/main.rs for ``importer``, or its parent."""
|
|
1773
|
+
parts = Path(importer).parts
|
|
1774
|
+
for i in range(len(parts) - 1, -1, -1):
|
|
1775
|
+
prefix = "/".join(parts[:i]) if i else ""
|
|
1776
|
+
for name in ("lib.rs", "main.rs"):
|
|
1777
|
+
candidate = f"{prefix}/{name}" if prefix else name
|
|
1778
|
+
if candidate in file_set:
|
|
1779
|
+
return prefix if prefix else "."
|
|
1780
|
+
parent = Path(importer).parent.as_posix()
|
|
1781
|
+
return parent if parent != "." else "."
|
|
1782
|
+
|
|
1783
|
+
def _probe_rust_module(self, base_dir: str, segments: List[str], file_set: Set[str]) -> List[str]:
|
|
1784
|
+
"""Resolve module path segments under ``base_dir`` to existing .rs files."""
|
|
1785
|
+
if not segments:
|
|
1786
|
+
return []
|
|
1787
|
+
hits: List[str] = []
|
|
1788
|
+
cur = base_dir if base_dir not in (".", "") else ""
|
|
1789
|
+
for i, seg in enumerate(segments):
|
|
1790
|
+
if seg in {"crate", "super", "self", "Self"}:
|
|
1791
|
+
continue
|
|
1792
|
+
# Try path-so-far as a module file at each segment (last may be an item).
|
|
1793
|
+
prefix = f"{cur}/{seg}" if cur else seg
|
|
1794
|
+
candidates = [
|
|
1795
|
+
f"{prefix}.rs",
|
|
1796
|
+
f"{prefix}/mod.rs",
|
|
1797
|
+
]
|
|
1798
|
+
found = None
|
|
1799
|
+
for cand in candidates:
|
|
1800
|
+
norm = self._normalize_js_path(cand)
|
|
1801
|
+
if norm in file_set:
|
|
1802
|
+
found = norm
|
|
1803
|
+
break
|
|
1804
|
+
if found:
|
|
1805
|
+
hits.append(found)
|
|
1806
|
+
cur = prefix
|
|
1807
|
+
elif i < len(segments) - 1:
|
|
1808
|
+
# Intermediate segment must exist as a directory module.
|
|
1809
|
+
cur = prefix
|
|
1810
|
+
# else: final segment may be a type/fn name — keep prior hits only
|
|
1811
|
+
return hits
|
|
1812
|
+
|
|
1813
|
+
def _rust_import_edges(self, files: List[str], file_set: Set[str]) -> List[Tuple[str, str]]:
|
|
1814
|
+
"""Rust ``mod`` / ``use`` → module files via optional tree-sitter layer.
|
|
1815
|
+
|
|
1816
|
+
No edges when tree-sitter is unavailable (degrades to pre-Phase-3 output).
|
|
1817
|
+
"""
|
|
1818
|
+
from devcouncil.indexing.ts_imports import extract_rust_import_refs, tree_sitter_available
|
|
1819
|
+
|
|
1820
|
+
if not tree_sitter_available():
|
|
1821
|
+
return []
|
|
1822
|
+
rust_files = [f for f in files if f.endswith(".rs")]
|
|
1823
|
+
if not rust_files:
|
|
1824
|
+
return []
|
|
1825
|
+
edges: List[Tuple[str, str]] = []
|
|
1826
|
+
seen: Set[Tuple[str, str]] = set()
|
|
1827
|
+
for rel in rust_files:
|
|
1828
|
+
try:
|
|
1829
|
+
source = (self.project_root / rel).read_text(encoding="utf-8", errors="replace")
|
|
1830
|
+
except OSError:
|
|
1831
|
+
continue
|
|
1832
|
+
refs = extract_rust_import_refs(source)
|
|
1833
|
+
if not refs:
|
|
1834
|
+
continue
|
|
1835
|
+
importer_dir = Path(rel).parent.as_posix()
|
|
1836
|
+
if importer_dir == ".":
|
|
1837
|
+
importer_dir = ""
|
|
1838
|
+
crate_root = self._rust_crate_root(rel, file_set)
|
|
1839
|
+
crate_dir = "" if crate_root in (".", "") else crate_root
|
|
1840
|
+
for ref in refs:
|
|
1841
|
+
targets: List[str] = []
|
|
1842
|
+
if ref.get("kind") == "mod":
|
|
1843
|
+
name = ref.get("name")
|
|
1844
|
+
if isinstance(name, str) and name:
|
|
1845
|
+
targets = self._probe_rust_module(importer_dir, [name], file_set)
|
|
1846
|
+
elif ref.get("kind") == "use":
|
|
1847
|
+
segments = ref.get("segments") or []
|
|
1848
|
+
if not isinstance(segments, list) or not segments:
|
|
1849
|
+
continue
|
|
1850
|
+
segs = [s for s in segments if isinstance(s, str)]
|
|
1851
|
+
if not segs:
|
|
1852
|
+
continue
|
|
1853
|
+
if segs[0] == "crate":
|
|
1854
|
+
targets = self._probe_rust_module(crate_dir, segs[1:], file_set)
|
|
1855
|
+
elif segs[0] == "super":
|
|
1856
|
+
parent = Path(rel).parent.parent.as_posix()
|
|
1857
|
+
parent_dir = "" if parent == "." else parent
|
|
1858
|
+
targets = self._probe_rust_module(parent_dir, segs[1:], file_set)
|
|
1859
|
+
elif segs[0] == "self":
|
|
1860
|
+
targets = self._probe_rust_module(importer_dir, segs[1:], file_set)
|
|
1861
|
+
else:
|
|
1862
|
+
# Relative / bare path — try from importer dir then crate root.
|
|
1863
|
+
targets = self._probe_rust_module(importer_dir, segs, file_set)
|
|
1864
|
+
if not targets:
|
|
1865
|
+
targets = self._probe_rust_module(crate_dir, segs, file_set)
|
|
1866
|
+
for target in targets:
|
|
1100
1867
|
if target != rel and (rel, target) not in seen:
|
|
1101
1868
|
seen.add((rel, target))
|
|
1102
1869
|
edges.append((rel, target))
|
|
@@ -1115,6 +1882,10 @@ class RepoMapper:
|
|
|
1115
1882
|
edges.extend(self._go_import_edges(files, file_set))
|
|
1116
1883
|
except Exception:
|
|
1117
1884
|
logger.debug("Go import-edge resolution failed", exc_info=True)
|
|
1885
|
+
try:
|
|
1886
|
+
edges.extend(self._rust_import_edges(files, file_set))
|
|
1887
|
+
except Exception:
|
|
1888
|
+
logger.debug("Rust import-edge resolution failed", exc_info=True)
|
|
1118
1889
|
return edges
|
|
1119
1890
|
|
|
1120
1891
|
def _rank_area_files(self, area_files: List[str], in_degree: Counter) -> List[str]:
|
|
@@ -1150,6 +1921,13 @@ class RepoMapper:
|
|
|
1150
1921
|
if len(area_handoffs[a]) < 3:
|
|
1151
1922
|
area_handoffs[a].append(f"{importer} -> {imported}")
|
|
1152
1923
|
|
|
1924
|
+
graph_roots: List[str] = []
|
|
1925
|
+
cg = getattr(self, "_last_code_graph", None)
|
|
1926
|
+
if cg is not None:
|
|
1927
|
+
roots_attr = getattr(cg, "entry_roots", None)
|
|
1928
|
+
if roots_attr is not None:
|
|
1929
|
+
graph_roots = [str(r) for r in roots_attr]
|
|
1930
|
+
|
|
1153
1931
|
subsystems: List[RepoSubsystem] = []
|
|
1154
1932
|
for area in sorted(by_area):
|
|
1155
1933
|
area_files = by_area[area]
|
|
@@ -1157,16 +1935,68 @@ class RepoMapper:
|
|
|
1157
1935
|
# but keep every real source subsystem.
|
|
1158
1936
|
if len(area_files) < 2 and area.split("/")[0] in _AUX_AREA_ROOTS:
|
|
1159
1937
|
continue
|
|
1938
|
+
area_file_set = set(area_files)
|
|
1939
|
+
area_roots = [r for r in graph_roots if r in area_file_set]
|
|
1160
1940
|
ranked = self._rank_area_files(area_files, in_degree)
|
|
1161
|
-
|
|
1162
|
-
|
|
1941
|
+
|
|
1942
|
+
def _entry_named(path: str) -> bool:
|
|
1943
|
+
name = Path(path).stem.lower()
|
|
1944
|
+
return any(name == hint for hint in _ENTRY_NAME_HINTS)
|
|
1945
|
+
|
|
1946
|
+
# Prefer real production entry roots + imported hubs over zero-degree
|
|
1947
|
+
# orphans when filling critical_files (avoids listing dead.py as critical).
|
|
1948
|
+
critical_files: List[str] = []
|
|
1949
|
+
for p in area_roots + ranked:
|
|
1950
|
+
if p in critical_files:
|
|
1951
|
+
continue
|
|
1952
|
+
if p in area_roots or in_degree.get(p, 0) > 0 or _entry_named(p):
|
|
1953
|
+
critical_files.append(p)
|
|
1954
|
+
if len(critical_files) >= self._SUBSYSTEM_CRITICAL_MAX:
|
|
1955
|
+
break
|
|
1956
|
+
# Only pad with remaining files when the area is tiny / has no hubs yet.
|
|
1957
|
+
if len(critical_files) < 2:
|
|
1958
|
+
for p in ranked:
|
|
1959
|
+
if p not in critical_files:
|
|
1960
|
+
critical_files.append(p)
|
|
1961
|
+
if len(critical_files) >= min(2, self._SUBSYSTEM_CRITICAL_MAX):
|
|
1962
|
+
break
|
|
1963
|
+
for p in ranked:
|
|
1964
|
+
if len(critical_files) >= self._SUBSYSTEM_CRITICAL_MAX:
|
|
1965
|
+
break
|
|
1966
|
+
if p not in critical_files and (
|
|
1967
|
+
in_degree.get(p, 0) > 0 or _entry_named(p) or p in area_roots
|
|
1968
|
+
):
|
|
1969
|
+
critical_files.append(p)
|
|
1970
|
+
|
|
1971
|
+
entry_points = list(area_roots)
|
|
1972
|
+
for p in critical_files:
|
|
1973
|
+
if p in entry_points:
|
|
1974
|
+
continue
|
|
1975
|
+
if in_degree.get(p, 0) > 0 or _entry_named(p):
|
|
1976
|
+
entry_points.append(p)
|
|
1977
|
+
if len(entry_points) >= 3:
|
|
1978
|
+
break
|
|
1979
|
+
if not entry_points:
|
|
1980
|
+
entry_points = critical_files[:1]
|
|
1981
|
+
|
|
1163
1982
|
stems = ", ".join(Path(p).stem for p in critical_files[:3])
|
|
1164
1983
|
summary = f"{Path(area).name or area}: {stems}" if stems else f"{area} ({len(area_files)} files)"
|
|
1984
|
+
# Optional community label from code graph (generic repos).
|
|
1985
|
+
community = ""
|
|
1986
|
+
if cg is not None:
|
|
1987
|
+
try:
|
|
1988
|
+
from devcouncil.indexing.graph.communities import community_label_for_area
|
|
1989
|
+
|
|
1990
|
+
community = community_label_for_area(cg, area)
|
|
1991
|
+
if community and community not in summary:
|
|
1992
|
+
summary = f"{summary} [{community}]"
|
|
1993
|
+
except Exception:
|
|
1994
|
+
community = ""
|
|
1165
1995
|
subsystems.append(
|
|
1166
1996
|
RepoSubsystem(
|
|
1167
1997
|
area=area,
|
|
1168
1998
|
summary=summary,
|
|
1169
|
-
entry_points=entry_points,
|
|
1999
|
+
entry_points=entry_points[:3],
|
|
1170
2000
|
critical_files=critical_files,
|
|
1171
2001
|
neighbors=sorted(area_neighbors.get(area, set()))[:6],
|
|
1172
2002
|
handoff_paths=area_handoffs.get(area, []),
|
|
@@ -1185,17 +2015,287 @@ class RepoMapper:
|
|
|
1185
2015
|
ranked = [path for path, _ in in_degree.most_common()]
|
|
1186
2016
|
return ranked[:8]
|
|
1187
2017
|
|
|
1188
|
-
def build_dependents(
|
|
2018
|
+
def build_dependents(
|
|
2019
|
+
self, edges: List[Tuple[str, str]]
|
|
2020
|
+
) -> Tuple[Dict[str, List[str]], Dict[str, int]]:
|
|
1189
2021
|
"""Reverse the import edges into a file -> dependents map (who imports each file),
|
|
1190
|
-
capped per file.
|
|
2022
|
+
capped per file. Returns ``(capped_dependents, totals_when_truncated)``.
|
|
2023
|
+
|
|
2024
|
+
``totals_when_truncated[path]`` is the full importer count when the listed
|
|
2025
|
+
sample was truncated by ``_DEPENDENTS_MAX`` — so agents know the blast radius
|
|
2026
|
+
is incomplete rather than silently missing importers.
|
|
2027
|
+
"""
|
|
1191
2028
|
reverse: Dict[str, Set[str]] = defaultdict(set)
|
|
1192
2029
|
for importer, imported in edges:
|
|
1193
2030
|
reverse[imported].add(importer)
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
if importers
|
|
1198
|
-
|
|
2031
|
+
capped: Dict[str, List[str]] = {}
|
|
2032
|
+
totals: Dict[str, int] = {}
|
|
2033
|
+
for path, importers in sorted(reverse.items()):
|
|
2034
|
+
if not importers:
|
|
2035
|
+
continue
|
|
2036
|
+
full = sorted(importers)
|
|
2037
|
+
capped[path] = full[: self._DEPENDENTS_MAX]
|
|
2038
|
+
if len(full) > self._DEPENDENTS_MAX:
|
|
2039
|
+
totals[path] = len(full)
|
|
2040
|
+
return capped, totals
|
|
2041
|
+
|
|
2042
|
+
def import_edges_for(self, files: List[str] | None = None) -> List[Tuple[str, str]]:
|
|
2043
|
+
"""Public uncapped forward import edges for the given files (or whole repo).
|
|
2044
|
+
|
|
2045
|
+
Verify gates need live edges — the persisted map won't include brand-new
|
|
2046
|
+
files until the next ``dev map``. Never raises.
|
|
2047
|
+
"""
|
|
2048
|
+
try:
|
|
2049
|
+
if files is None:
|
|
2050
|
+
files = self.get_git_files()
|
|
2051
|
+
return self._all_import_edges(files)
|
|
2052
|
+
except Exception:
|
|
2053
|
+
logger.debug("import_edges_for failed", exc_info=True)
|
|
2054
|
+
return []
|
|
2055
|
+
|
|
2056
|
+
def dependents_for(self, files: List[str] | None = None) -> Dict[str, Set[str]]:
|
|
2057
|
+
"""Uncapped reverse index (imported -> set of importers) for presence checks.
|
|
2058
|
+
|
|
2059
|
+
Unlike :meth:`build_dependents`, this includes zero-importer keys only when
|
|
2060
|
+
asked about specific files via the returned dict's ``.get``, and never caps
|
|
2061
|
+
the importer lists. Never raises.
|
|
2062
|
+
"""
|
|
2063
|
+
try:
|
|
2064
|
+
edges = self.import_edges_for(files)
|
|
2065
|
+
reverse: Dict[str, Set[str]] = defaultdict(set)
|
|
2066
|
+
for importer, imported in edges:
|
|
2067
|
+
reverse[imported].add(importer)
|
|
2068
|
+
return dict(reverse)
|
|
2069
|
+
except Exception:
|
|
2070
|
+
logger.debug("dependents_for failed", exc_info=True)
|
|
2071
|
+
return {}
|
|
2072
|
+
|
|
2073
|
+
def _compute_liveness(
|
|
2074
|
+
self,
|
|
2075
|
+
files: List[str],
|
|
2076
|
+
edges: List[Tuple[str, str]],
|
|
2077
|
+
*,
|
|
2078
|
+
cap: Optional[int] = None,
|
|
2079
|
+
lsp_refs: bool = False,
|
|
2080
|
+
) -> Tuple[List[str], List[str], List[str], List[str], List[str], bool]:
|
|
2081
|
+
"""Compute entry_roots, unwired_candidates, unreachable_files, dead_symbol_candidates.
|
|
2082
|
+
|
|
2083
|
+
Also returns ``symbol_index`` (``path::name`` keys) as the fifth element and
|
|
2084
|
+
``liveness_unreachable_unreliable`` as the sixth.
|
|
2085
|
+
|
|
2086
|
+
File lists delegate to ``graph.liveness.file_liveness`` so map vs ratchet
|
|
2087
|
+
cannot drift on empty-root guards / caps.
|
|
2088
|
+
|
|
2089
|
+
``cap`` defaults to ``_LIVENESS_CAP`` for the stored map debt lists. Pass ``0``
|
|
2090
|
+
(or any non-positive) for uncapped lists used by the liveness ratchet baseline.
|
|
2091
|
+
|
|
2092
|
+
Stored ``entry_roots`` are production-only and never capped (only debt lists
|
|
2093
|
+
are). Structural exemptions are a skip-list, not BFS seeds. Never raises;
|
|
2094
|
+
returns empty lists on failure.
|
|
2095
|
+
|
|
2096
|
+
When ``lsp_refs`` is True, dead-symbol candidates are confirmed via the
|
|
2097
|
+
optional live LSP client (external references clear false positives).
|
|
2098
|
+
"""
|
|
2099
|
+
try:
|
|
2100
|
+
from devcouncil.indexing.graph.liveness import file_liveness
|
|
2101
|
+
|
|
2102
|
+
if cap is None:
|
|
2103
|
+
file_cap: Optional[int] = self._LIVENESS_CAP
|
|
2104
|
+
elif cap <= 0:
|
|
2105
|
+
file_cap = 0
|
|
2106
|
+
else:
|
|
2107
|
+
file_cap = cap
|
|
2108
|
+
|
|
2109
|
+
prod_roots, unwired, unreachable, unreliable = file_liveness(
|
|
2110
|
+
self.project_root, files, edges, cap=file_cap,
|
|
2111
|
+
)
|
|
2112
|
+
|
|
2113
|
+
dead_cap = 0 if file_cap is None or file_cap <= 0 else file_cap
|
|
2114
|
+
dead_symbols, symbol_index = self._dead_symbol_candidates(
|
|
2115
|
+
files, cap=dead_cap, with_index=True, lsp_refs=lsp_refs,
|
|
2116
|
+
)
|
|
2117
|
+
return (
|
|
2118
|
+
prod_roots,
|
|
2119
|
+
unwired,
|
|
2120
|
+
unreachable,
|
|
2121
|
+
dead_symbols,
|
|
2122
|
+
symbol_index,
|
|
2123
|
+
unreliable,
|
|
2124
|
+
)
|
|
2125
|
+
except Exception:
|
|
2126
|
+
logger.debug("liveness computation failed", exc_info=True)
|
|
2127
|
+
return [], [], [], [], [], True
|
|
2128
|
+
|
|
2129
|
+
def liveness_snapshot(self) -> Dict[str, object]:
|
|
2130
|
+
"""Uncapped liveness lists for ratchet baseline / verify-side current.
|
|
2131
|
+
|
|
2132
|
+
Always recomputes import edges (never reuses a stale ``self._edges`` cache).
|
|
2133
|
+
Includes ``symbol_index`` (``path::name`` keys) so the ratchet can require
|
|
2134
|
+
a symbol to have existed at baseline before calling it stranded.
|
|
2135
|
+
Never raises; returns empty lists on failure.
|
|
2136
|
+
"""
|
|
2137
|
+
try:
|
|
2138
|
+
files = self.get_git_files()
|
|
2139
|
+
# Invalidate any prior edge cache — snapshot must reflect current tree.
|
|
2140
|
+
self._edges = None
|
|
2141
|
+
edges = self._all_import_edges(files)
|
|
2142
|
+
use_lsp = False
|
|
2143
|
+
try:
|
|
2144
|
+
from devcouncil.indexing.lsp_client import lsp_refs_enabled
|
|
2145
|
+
|
|
2146
|
+
use_lsp = lsp_refs_enabled(self.project_root)
|
|
2147
|
+
except Exception:
|
|
2148
|
+
use_lsp = False
|
|
2149
|
+
roots, unwired, unreachable, dead, symbol_index, unreliable = (
|
|
2150
|
+
self._compute_liveness(files, edges, cap=0, lsp_refs=use_lsp)
|
|
2151
|
+
)
|
|
2152
|
+
return {
|
|
2153
|
+
"entry_roots": roots,
|
|
2154
|
+
"unwired_candidates": unwired,
|
|
2155
|
+
"unreachable_files": unreachable,
|
|
2156
|
+
"dead_symbol_candidates": dead,
|
|
2157
|
+
"symbol_index": symbol_index,
|
|
2158
|
+
"liveness_unreachable_unreliable": unreliable,
|
|
2159
|
+
}
|
|
2160
|
+
except Exception:
|
|
2161
|
+
logger.debug("liveness_snapshot failed", exc_info=True)
|
|
2162
|
+
return {
|
|
2163
|
+
"entry_roots": [],
|
|
2164
|
+
"unwired_candidates": [],
|
|
2165
|
+
"unreachable_files": [],
|
|
2166
|
+
"dead_symbol_candidates": [],
|
|
2167
|
+
"symbol_index": [],
|
|
2168
|
+
"liveness_unreachable_unreliable": True,
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
def _dead_symbol_candidates(
|
|
2172
|
+
self,
|
|
2173
|
+
files: List[str],
|
|
2174
|
+
*,
|
|
2175
|
+
cap: int = 200,
|
|
2176
|
+
with_index: bool = False,
|
|
2177
|
+
lsp_refs: bool = False,
|
|
2178
|
+
):
|
|
2179
|
+
"""Public top-level symbols never referenced outside their own definition span.
|
|
2180
|
+
|
|
2181
|
+
Format: ``path:line name``. Python + JS/TS only. Best-effort; never raises.
|
|
2182
|
+
Same-file uses outside the symbol's span clear it; recursive self-references
|
|
2183
|
+
inside the span do not. Test-file references clear a symbol (parity with the
|
|
2184
|
+
verify gate); test files themselves are not scanned for definitions.
|
|
2185
|
+
``cap <= 0`` means uncapped.
|
|
2186
|
+
|
|
2187
|
+
When ``with_index`` is True, returns ``(dead_list, symbol_index)`` where
|
|
2188
|
+
``symbol_index`` is sorted ``path::name`` keys for all scanned definitions.
|
|
2189
|
+
Otherwise returns just the dead list (legacy callers).
|
|
2190
|
+
|
|
2191
|
+
When ``lsp_refs`` is True, candidates are confirmed via the optional live
|
|
2192
|
+
LSP client before inclusion (external references clear false positives).
|
|
2193
|
+
"""
|
|
2194
|
+
empty: List[str] = []
|
|
2195
|
+
try:
|
|
2196
|
+
from devcouncil.indexing.wiring import (
|
|
2197
|
+
decorator_names,
|
|
2198
|
+
is_liveness_code_file,
|
|
2199
|
+
is_private_symbol,
|
|
2200
|
+
is_test_path,
|
|
2201
|
+
is_vendored_path,
|
|
2202
|
+
is_wiring_decorated,
|
|
2203
|
+
iter_js_export_symbols,
|
|
2204
|
+
parse_python_all_exports,
|
|
2205
|
+
strip_js_comments,
|
|
2206
|
+
strip_py_comments,
|
|
2207
|
+
strip_string_literals,
|
|
2208
|
+
)
|
|
2209
|
+
|
|
2210
|
+
# path, start, end, name
|
|
2211
|
+
definitions: List[Tuple[str, int, int, str]] = []
|
|
2212
|
+
token_files: Dict[str, Set[str]] = defaultdict(set)
|
|
2213
|
+
token_lines: Dict[str, Dict[str, Set[int]]] = defaultdict(lambda: defaultdict(set))
|
|
2214
|
+
ident_re = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)\b")
|
|
2215
|
+
|
|
2216
|
+
for rel in sorted(files):
|
|
2217
|
+
if not is_liveness_code_file(rel):
|
|
2218
|
+
continue
|
|
2219
|
+
if is_vendored_path(rel):
|
|
2220
|
+
continue
|
|
2221
|
+
try:
|
|
2222
|
+
source = (self.project_root / rel).read_text(encoding="utf-8", errors="replace")
|
|
2223
|
+
except OSError:
|
|
2224
|
+
continue
|
|
2225
|
+
suffix = Path(rel).suffix.lower()
|
|
2226
|
+
if suffix == ".py":
|
|
2227
|
+
cleaned = strip_string_literals(strip_py_comments(source))
|
|
2228
|
+
else:
|
|
2229
|
+
cleaned = strip_string_literals(strip_js_comments(source))
|
|
2230
|
+
|
|
2231
|
+
# Index tokens from all code files including tests so a test
|
|
2232
|
+
# reference clears a production symbol (verify-gate parity).
|
|
2233
|
+
for lineno, line in enumerate(cleaned.splitlines(), 1):
|
|
2234
|
+
for tok in ident_re.findall(line):
|
|
2235
|
+
if len(tok) < 2:
|
|
2236
|
+
continue
|
|
2237
|
+
token_files[tok].add(rel)
|
|
2238
|
+
token_lines[tok][rel].add(lineno)
|
|
2239
|
+
|
|
2240
|
+
if is_test_path(rel):
|
|
2241
|
+
continue
|
|
2242
|
+
|
|
2243
|
+
if suffix == ".py":
|
|
2244
|
+
try:
|
|
2245
|
+
tree = ast.parse(source)
|
|
2246
|
+
protected = parse_python_all_exports(source)
|
|
2247
|
+
for node in tree.body:
|
|
2248
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
2249
|
+
name = node.name
|
|
2250
|
+
if is_private_symbol(name):
|
|
2251
|
+
continue
|
|
2252
|
+
if name in protected:
|
|
2253
|
+
continue
|
|
2254
|
+
if is_wiring_decorated(decorator_names(node)):
|
|
2255
|
+
continue
|
|
2256
|
+
start = getattr(node, "lineno", 1) or 1
|
|
2257
|
+
end = getattr(node, "end_lineno", start) or start
|
|
2258
|
+
definitions.append((rel, start, end, name))
|
|
2259
|
+
except SyntaxError:
|
|
2260
|
+
pass
|
|
2261
|
+
else:
|
|
2262
|
+
for line_no, name in iter_js_export_symbols(source):
|
|
2263
|
+
prev_lines = source.splitlines()[: max(0, line_no - 1)]
|
|
2264
|
+
if prev_lines:
|
|
2265
|
+
prev = prev_lines[-1].strip()
|
|
2266
|
+
if prev.startswith("@"):
|
|
2267
|
+
continue
|
|
2268
|
+
definitions.append((rel, line_no, line_no, name))
|
|
2269
|
+
|
|
2270
|
+
symbol_index = sorted({f"{path}::{name}" for path, _s, _e, name in definitions})
|
|
2271
|
+
out: List[str] = []
|
|
2272
|
+
for path, start, end, name in definitions:
|
|
2273
|
+
refs = token_files.get(name, set())
|
|
2274
|
+
if refs - {path}:
|
|
2275
|
+
continue
|
|
2276
|
+
same_lines = token_lines.get(name, {}).get(path, set())
|
|
2277
|
+
if any(ln < start or ln > end for ln in same_lines):
|
|
2278
|
+
continue
|
|
2279
|
+
out.append(f"{path}:{start} {name}")
|
|
2280
|
+
if cap > 0 and len(out) >= cap:
|
|
2281
|
+
break
|
|
2282
|
+
if lsp_refs and out:
|
|
2283
|
+
try:
|
|
2284
|
+
from devcouncil.indexing.lsp_client import filter_dead_symbols_with_lsp
|
|
2285
|
+
|
|
2286
|
+
out = filter_dead_symbols_with_lsp(self.project_root, out)
|
|
2287
|
+
if cap > 0:
|
|
2288
|
+
out = out[:cap]
|
|
2289
|
+
except Exception:
|
|
2290
|
+
logger.debug("LSP dead-symbol confirmation failed", exc_info=True)
|
|
2291
|
+
if with_index:
|
|
2292
|
+
return out, symbol_index
|
|
2293
|
+
return out
|
|
2294
|
+
except Exception:
|
|
2295
|
+
logger.debug("dead_symbol_candidates failed", exc_info=True)
|
|
2296
|
+
if with_index:
|
|
2297
|
+
return empty, empty
|
|
2298
|
+
return empty
|
|
1199
2299
|
|
|
1200
2300
|
def describe_file(self, path: str) -> RepoFileEntry:
|
|
1201
2301
|
return RepoFileEntry(
|
|
@@ -1210,55 +2310,101 @@ class RepoMapper:
|
|
|
1210
2310
|
normalized = path.replace("\\", "/")
|
|
1211
2311
|
parts = set(normalized.split("/"))
|
|
1212
2312
|
name = Path(normalized).name
|
|
2313
|
+
lower_name = name.lower()
|
|
1213
2314
|
if "__pycache__" in parts or normalized.endswith(".pyc"):
|
|
1214
2315
|
return True
|
|
1215
2316
|
if parts.intersection({".git", ".devcouncil", ".pytest_cache", ".ruff_cache", ".mypy_cache", ".venv"}):
|
|
1216
2317
|
return True
|
|
1217
2318
|
if normalized.startswith("dist/") or normalized.startswith("build/"):
|
|
1218
2319
|
return True
|
|
1219
|
-
if
|
|
2320
|
+
if lower_name.endswith((".tgz", ".whl", ".tar.gz")):
|
|
2321
|
+
return True
|
|
2322
|
+
if name.startswith(("tmp", "temp", ".tmp")) or name.endswith("~"):
|
|
1220
2323
|
return True
|
|
1221
2324
|
return False
|
|
1222
2325
|
|
|
1223
2326
|
def _git_head(self) -> str:
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
).decode("utf-8", errors="replace").strip()
|
|
1228
|
-
except Exception:
|
|
1229
|
-
return ""
|
|
2327
|
+
from devcouncil.utils.proc import git_output
|
|
2328
|
+
|
|
2329
|
+
return git_output(["rev-parse", "HEAD"], cwd=self.project_root, default="").strip()
|
|
1230
2330
|
|
|
1231
2331
|
def _files_fingerprint(self, files: List[str]) -> str:
|
|
1232
2332
|
return hashlib.sha1("\n".join(sorted(files)).encode("utf-8")).hexdigest()
|
|
1233
2333
|
|
|
2334
|
+
def _content_fingerprint(self, files: List[str]) -> str:
|
|
2335
|
+
from devcouncil.indexing.graph.build import content_fingerprint
|
|
2336
|
+
|
|
2337
|
+
return content_fingerprint(self.project_root, files)
|
|
2338
|
+
|
|
1234
2339
|
def map_is_stale(self, repo_map: Dict[str, object]) -> bool:
|
|
1235
|
-
"""True when the stored map no longer matches the repo's current git HEAD
|
|
1236
|
-
tracked file set — i.e. commits
|
|
1237
|
-
|
|
2340
|
+
"""True when the stored map no longer matches the repo's current git HEAD,
|
|
2341
|
+
tracked file set, or content fingerprint — i.e. commits, file add/removes, or
|
|
2342
|
+
plain edits happened since ``dev map`` last ran.
|
|
2343
|
+
|
|
2344
|
+
Fail-closed: exceptions from ``get_git_files`` / content fingerprinting
|
|
2345
|
+
return True (stale) so ``--if-stale`` / verify rebuild rather than trusting
|
|
2346
|
+
an unverifiable map.
|
|
2347
|
+
|
|
2348
|
+
Returns False for maps written before fingerprinting (no false alarms).
|
|
2349
|
+
Legacy maps without ``content_fingerprint`` skip the content check.
|
|
2350
|
+
"""
|
|
1238
2351
|
stored_head = str(repo_map.get("generated_head") or "")
|
|
1239
2352
|
stored_hash = str(repo_map.get("indexed_hash") or "")
|
|
1240
2353
|
if not stored_head and not stored_hash:
|
|
1241
2354
|
return False
|
|
2355
|
+
# Lean/degraded maps re-stamp fingerprints but lack a trustworthy graph —
|
|
2356
|
+
# fail closed so --if-stale / watch / verify keep retrying until healthy.
|
|
2357
|
+
if bool(repo_map.get("graph_degraded")):
|
|
2358
|
+
return True
|
|
1242
2359
|
try:
|
|
1243
2360
|
files = self.get_git_files()
|
|
1244
2361
|
except Exception:
|
|
2362
|
+
# Fail closed: cannot prove freshness → treat as stale so --if-stale /
|
|
2363
|
+
# verify rebuild rather than trusting a possibly outdated map.
|
|
2364
|
+
return True
|
|
2365
|
+
if self._git_head() != stored_head or self._files_fingerprint(files) != stored_hash:
|
|
2366
|
+
return True
|
|
2367
|
+
stored_content = str(repo_map.get("content_fingerprint") or "")
|
|
2368
|
+
if not stored_content:
|
|
2369
|
+
# Legacy maps without content_fingerprint skip the content check.
|
|
1245
2370
|
return False
|
|
1246
|
-
|
|
2371
|
+
try:
|
|
2372
|
+
return self._content_fingerprint(files) != stored_content
|
|
2373
|
+
except Exception:
|
|
2374
|
+
return True
|
|
1247
2375
|
|
|
1248
2376
|
def get_git_files(self) -> List[str]:
|
|
1249
2377
|
try:
|
|
1250
|
-
|
|
1251
|
-
|
|
2378
|
+
from devcouncil.utils.proc import git_output
|
|
2379
|
+
|
|
2380
|
+
# ``-z`` avoids C-quoting of non-ASCII paths (otherwise ``café.py`` becomes
|
|
2381
|
+
# ``"src/caf\303\251.py"`` and fails the ``is_file()`` existence filter).
|
|
2382
|
+
output = git_output(
|
|
2383
|
+
["ls-files", "-z", "--cached", "--others", "--exclude-standard"],
|
|
1252
2384
|
cwd=self.project_root,
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
2385
|
+
)
|
|
2386
|
+
paths = [p.replace("\\", "/") for p in output.split("\0") if p]
|
|
2387
|
+
# Skip index entries whose working-tree file was deleted but not staged.
|
|
2388
|
+
return [
|
|
2389
|
+
path
|
|
2390
|
+
for path in paths
|
|
2391
|
+
if not self._is_runtime_or_generated_file(path)
|
|
2392
|
+
and (self.project_root / path).is_file()
|
|
2393
|
+
]
|
|
1256
2394
|
except Exception:
|
|
1257
2395
|
# Fallback to os.walk if not a git repo or git missing
|
|
2396
|
+
from devcouncil.indexing.walk import IGNORED_DIR_NAMES
|
|
2397
|
+
|
|
1258
2398
|
files = []
|
|
1259
|
-
for root,
|
|
2399
|
+
for root, dirnames, filenames in os.walk(self.project_root):
|
|
2400
|
+
dirnames[:] = [name for name in dirnames if name not in IGNORED_DIR_NAMES]
|
|
1260
2401
|
for f in filenames:
|
|
1261
|
-
|
|
2402
|
+
# Normalize to forward slashes like the git branch above —
|
|
2403
|
+
# backslash paths make every stored (posix) path look
|
|
2404
|
+
# non-live downstream and defeat incremental sync.
|
|
2405
|
+
rel_path = os.path.relpath(
|
|
2406
|
+
os.path.join(root, f), self.project_root
|
|
2407
|
+
).replace(os.sep, "/")
|
|
1262
2408
|
if not rel_path.startswith(".") and not self._is_runtime_or_generated_file(rel_path):
|
|
1263
2409
|
files.append(rel_path)
|
|
1264
2410
|
return files
|
|
@@ -1373,23 +2519,108 @@ class RepoMapper:
|
|
|
1373
2519
|
return commands
|
|
1374
2520
|
|
|
1375
2521
|
def _ripgrep_search(self, goal: str, files: List[str]) -> List[Dict[str, str]]:
|
|
1376
|
-
"""
|
|
1377
|
-
|
|
2522
|
+
"""Rank candidate files for ``goal`` via ripgrep content search, else path tokens.
|
|
2523
|
+
|
|
2524
|
+
Multi-word goals are OR'd as separate ``-e`` patterns (a single quoted phrase
|
|
2525
|
+
almost never appears in source). Search is scoped to roots present in ``files``
|
|
2526
|
+
so unscoped ``rg`` cannot hang on huge trees or miss the tree entirely.
|
|
2527
|
+
Results are intersected with ``files`` so untracked noise cannot appear.
|
|
2528
|
+
"""
|
|
2529
|
+
file_set = {f.replace("\\", "/") for f in files}
|
|
2530
|
+
# Split on whitespace and ``|``; strip call/index punctuation so goals like
|
|
2531
|
+
# ``file_liveness(`` or ``(unwired|unreachable)`` still match identifiers.
|
|
2532
|
+
raw_parts: List[str] = []
|
|
2533
|
+
for part in goal.replace("|", " ").split():
|
|
2534
|
+
raw_parts.append(part)
|
|
2535
|
+
tokens: List[str] = []
|
|
2536
|
+
for raw in raw_parts:
|
|
2537
|
+
t = raw.strip().strip("()[]{}<>,:;\"'`")
|
|
2538
|
+
while t and t[-1] in "+*?^$\\.":
|
|
2539
|
+
t = t[:-1]
|
|
2540
|
+
while t and t[0] in "+*?^$\\.":
|
|
2541
|
+
t = t[1:]
|
|
2542
|
+
if len(t) >= 2:
|
|
2543
|
+
tokens.append(t)
|
|
2544
|
+
if not tokens and goal.strip():
|
|
2545
|
+
tokens = [goal.strip()]
|
|
2546
|
+
if not tokens:
|
|
2547
|
+
return []
|
|
2548
|
+
|
|
1378
2549
|
try:
|
|
1379
|
-
#
|
|
2550
|
+
# Prefer real source/test roots so alphabetical tops like `.github` /
|
|
2551
|
+
# README.md cannot crowd `src`/`tests` out of a small roots budget.
|
|
2552
|
+
preferred = (
|
|
2553
|
+
"src", "tests", "test", "benchmarks", "scripts", "docs", "bin",
|
|
2554
|
+
"packages", "lib", "app", "apps", "services", "cmd", "internal",
|
|
2555
|
+
)
|
|
2556
|
+
tops_in_files: set[str] = set()
|
|
2557
|
+
for f in file_set:
|
|
2558
|
+
top = f.split("/", 1)[0]
|
|
2559
|
+
if top:
|
|
2560
|
+
tops_in_files.add(top)
|
|
2561
|
+
|
|
2562
|
+
roots: List[str] = []
|
|
2563
|
+
for top in preferred:
|
|
2564
|
+
if top in tops_in_files and (self.project_root / top).is_dir():
|
|
2565
|
+
roots.append(top)
|
|
2566
|
+
for top in sorted(tops_in_files):
|
|
2567
|
+
if top in roots:
|
|
2568
|
+
continue
|
|
2569
|
+
if top.startswith("."):
|
|
2570
|
+
continue
|
|
2571
|
+
if not (self.project_root / top).is_dir():
|
|
2572
|
+
continue
|
|
2573
|
+
roots.append(top)
|
|
2574
|
+
if len(roots) >= 16:
|
|
2575
|
+
break
|
|
2576
|
+
if not roots:
|
|
2577
|
+
roots = ["."]
|
|
2578
|
+
|
|
2579
|
+
cmd = [
|
|
2580
|
+
"rg",
|
|
2581
|
+
"--files-with-matches",
|
|
2582
|
+
"--ignore-case",
|
|
2583
|
+
# Goal tokens are keywords/identifiers, not regex. Without -F,
|
|
2584
|
+
# trailing meta like ``file_liveness(`` or ``foo[`` silently miss.
|
|
2585
|
+
"-F",
|
|
2586
|
+
"--glob", "!**/.git/**",
|
|
2587
|
+
"--glob", "!**/.devcouncil/**",
|
|
2588
|
+
"--glob", "!**/.venv/**",
|
|
2589
|
+
"--glob", "!**/node_modules/**",
|
|
2590
|
+
]
|
|
2591
|
+
for token in tokens:
|
|
2592
|
+
cmd.extend(["-e", token])
|
|
2593
|
+
cmd.extend(roots)
|
|
2594
|
+
|
|
1380
2595
|
result = subprocess.run(
|
|
1381
|
-
|
|
1382
|
-
capture_output=True,
|
|
2596
|
+
cmd,
|
|
2597
|
+
capture_output=True,
|
|
2598
|
+
text=True,
|
|
2599
|
+
cwd=self.project_root,
|
|
2600
|
+
timeout=10,
|
|
1383
2601
|
)
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
2602
|
+
# rg exits 1 when no matches — still a successful run.
|
|
2603
|
+
if result.returncode in (0, 1):
|
|
2604
|
+
hits: List[Dict[str, str]] = []
|
|
2605
|
+
for line in sorted(result.stdout.strip().splitlines()):
|
|
2606
|
+
path = line.strip().replace("\\", "/")
|
|
2607
|
+
if path.startswith("./"):
|
|
2608
|
+
path = path[2:]
|
|
2609
|
+
if not path or path not in file_set:
|
|
2610
|
+
continue
|
|
2611
|
+
hits.append({
|
|
2612
|
+
"path": path,
|
|
2613
|
+
"reason": f"ripgrep match for '{goal}'",
|
|
2614
|
+
})
|
|
2615
|
+
if len(hits) >= 10:
|
|
2616
|
+
break
|
|
2617
|
+
if hits:
|
|
2618
|
+
return hits
|
|
1388
2619
|
except Exception:
|
|
1389
|
-
|
|
2620
|
+
logger.debug("ripgrep search failed; falling back to naive matching", exc_info=True)
|
|
1390
2621
|
|
|
1391
|
-
# Naive keyword matching fallback
|
|
1392
|
-
goal_words = set(
|
|
2622
|
+
# Naive keyword matching fallback (path tokens only).
|
|
2623
|
+
goal_words = set(t.lower() for t in tokens)
|
|
1393
2624
|
scored_candidates: list[tuple[int, str]] = []
|
|
1394
2625
|
for f in files:
|
|
1395
2626
|
f_lower = f.lower()
|
|
@@ -1412,23 +2643,84 @@ class RepoMapper:
|
|
|
1412
2643
|
logger.debug("Dependency-risk scan failed", exc_info=True)
|
|
1413
2644
|
return []
|
|
1414
2645
|
|
|
1415
|
-
def map_repo(
|
|
2646
|
+
def map_repo(
|
|
2647
|
+
self,
|
|
2648
|
+
goal: str = "",
|
|
2649
|
+
*,
|
|
2650
|
+
scan_dependencies: bool = False,
|
|
2651
|
+
liveness: bool = True,
|
|
2652
|
+
lsp_refs: bool = False,
|
|
2653
|
+
) -> RepoMap:
|
|
1416
2654
|
"""Build the repo map.
|
|
1417
2655
|
|
|
1418
2656
|
``scan_dependencies`` is opt-in (default off) so the common ``dev map`` path
|
|
1419
2657
|
stays fast and never shells out to a vulnerability auditor. When enabled, a
|
|
1420
2658
|
best-effort SCA scan runs locally (only if an auditor is installed) and its
|
|
1421
2659
|
findings are attached as ``dependency_risks``.
|
|
2660
|
+
|
|
2661
|
+
``liveness`` (default on) computes entry_roots / unwired / unreachable /
|
|
2662
|
+
dead_symbol candidate lists. Pass False (``--no-liveness``) to skip.
|
|
2663
|
+
|
|
2664
|
+
``lsp_refs`` (default off) confirms dead-symbol candidates via the optional
|
|
2665
|
+
live LSP client when a language server is available.
|
|
2666
|
+
|
|
2667
|
+
Builds the symbol-level code knowledge graph first, writes
|
|
2668
|
+
``.devcouncil/graph/code_graph.json``, then derives the summary ``RepoMap``.
|
|
1422
2669
|
"""
|
|
1423
2670
|
files = self.get_git_files()
|
|
1424
2671
|
# Decide DevCouncil-vs-generic and the source root BEFORE describing files, so
|
|
1425
2672
|
# area bucketing and subsystem inference agree within a single run.
|
|
1426
2673
|
self._use_generic = not any(path.startswith("src/devcouncil/") for path in files)
|
|
1427
2674
|
self._source_root = self.detect_source_root(files)
|
|
1428
|
-
|
|
1429
|
-
#
|
|
1430
|
-
#
|
|
1431
|
-
|
|
2675
|
+
|
|
2676
|
+
# Single-pass: extract + resolve + liveness/token-scan once; derive map lists
|
|
2677
|
+
# and graph dead_code from that pass (no second _token_scan_dead).
|
|
2678
|
+
changed = getattr(self, "_graph_changed_paths", None)
|
|
2679
|
+
code_graph: CodeGraph | None = getattr(self, "_prebuilt_code_graph", None)
|
|
2680
|
+
prebuilt_graph = code_graph is not None
|
|
2681
|
+
skip_graph_build = bool(getattr(self, "_skip_code_graph_build", False))
|
|
2682
|
+
if code_graph is not None:
|
|
2683
|
+
self._last_code_graph = code_graph
|
|
2684
|
+
self._edges = [
|
|
2685
|
+
(e.source, e.target)
|
|
2686
|
+
for e in code_graph.edges
|
|
2687
|
+
if e.kind == "imports" and "::" not in e.source and "::" not in e.target
|
|
2688
|
+
]
|
|
2689
|
+
elif skip_graph_build:
|
|
2690
|
+
self._last_code_graph = None
|
|
2691
|
+
self._edges = self._all_import_edges(files)
|
|
2692
|
+
else:
|
|
2693
|
+
try:
|
|
2694
|
+
from devcouncil.indexing.graph.build import build_code_graph, write_code_graph
|
|
2695
|
+
|
|
2696
|
+
code_graph = build_code_graph(
|
|
2697
|
+
self.project_root,
|
|
2698
|
+
files,
|
|
2699
|
+
changed_paths=changed,
|
|
2700
|
+
liveness=liveness,
|
|
2701
|
+
lsp_refs=lsp_refs,
|
|
2702
|
+
mapper=self,
|
|
2703
|
+
)
|
|
2704
|
+
self._last_code_graph = code_graph
|
|
2705
|
+
# File→file import edges only (named-import edges target symbols).
|
|
2706
|
+
self._edges = [
|
|
2707
|
+
(e.source, e.target)
|
|
2708
|
+
for e in code_graph.edges
|
|
2709
|
+
if e.kind == "imports" and "::" not in e.source and "::" not in e.target
|
|
2710
|
+
]
|
|
2711
|
+
except Exception:
|
|
2712
|
+
logger.warning(
|
|
2713
|
+
"code graph build failed; falling back to import edges only "
|
|
2714
|
+
"(dead_symbol_candidates will be omitted — refusing token-only flood)",
|
|
2715
|
+
exc_info=True,
|
|
2716
|
+
)
|
|
2717
|
+
code_graph = None
|
|
2718
|
+
self._last_code_graph = None
|
|
2719
|
+
self._edges = self._all_import_edges(files)
|
|
2720
|
+
|
|
2721
|
+
if self._edges is None:
|
|
2722
|
+
self._edges = self._all_import_edges(files)
|
|
2723
|
+
|
|
1432
2724
|
file_entries = [self.describe_file(path) for path in sorted(files)]
|
|
1433
2725
|
file_set = set(files)
|
|
1434
2726
|
|
|
@@ -1460,6 +2752,74 @@ class RepoMapper:
|
|
|
1460
2752
|
if goal:
|
|
1461
2753
|
candidates = self._ripgrep_search(goal, files)
|
|
1462
2754
|
|
|
2755
|
+
entry_roots_list: List[str] = []
|
|
2756
|
+
unwired: List[str] = []
|
|
2757
|
+
unreachable: List[str] = []
|
|
2758
|
+
dead_syms: List[str] = []
|
|
2759
|
+
unreachable_unreliable = False
|
|
2760
|
+
liveness_meta: Dict[str, object] = {}
|
|
2761
|
+
if liveness and code_graph is not None:
|
|
2762
|
+
entry_roots_list = list(code_graph.entry_roots)
|
|
2763
|
+
# Caps apply only when serializing repo_map.json (graph stays uncapped).
|
|
2764
|
+
from devcouncil.indexing.graph.liveness import apply_liveness_cap
|
|
2765
|
+
|
|
2766
|
+
cap = self._LIVENESS_CAP
|
|
2767
|
+
unwired, um = apply_liveness_cap(list(code_graph.unwired_candidates), cap)
|
|
2768
|
+
unreachable, rm = apply_liveness_cap(list(code_graph.unreachable_files), cap)
|
|
2769
|
+
dead_syms, dm = apply_liveness_cap(list(code_graph.meta.get("legacy_dead_symbol_candidates") or []), cap)
|
|
2770
|
+
liveness_meta = {"unwired": um, "unreachable": rm, "dead_symbol": dm}
|
|
2771
|
+
unreachable_unreliable = bool(
|
|
2772
|
+
code_graph.meta.get("liveness_unreachable_unreliable")
|
|
2773
|
+
)
|
|
2774
|
+
elif liveness:
|
|
2775
|
+
# Graph assemble failed: keep file-level liveness from import edges,
|
|
2776
|
+
# but do NOT run the token-only dead-symbol scan (misleading flood).
|
|
2777
|
+
logger.warning(
|
|
2778
|
+
"code graph unavailable; leaving dead_symbol_candidates empty"
|
|
2779
|
+
)
|
|
2780
|
+
try:
|
|
2781
|
+
from devcouncil.indexing.graph.liveness import file_liveness
|
|
2782
|
+
|
|
2783
|
+
cap = self._LIVENESS_CAP
|
|
2784
|
+
entry_roots_list, unwired, unreachable, unreachable_unreliable = (
|
|
2785
|
+
file_liveness(
|
|
2786
|
+
self.project_root,
|
|
2787
|
+
files,
|
|
2788
|
+
self._edges or [],
|
|
2789
|
+
cap=cap,
|
|
2790
|
+
)
|
|
2791
|
+
)
|
|
2792
|
+
except Exception:
|
|
2793
|
+
logger.debug("file_liveness fallback after graph failure failed", exc_info=True)
|
|
2794
|
+
entry_roots_list, unwired, unreachable = [], [], []
|
|
2795
|
+
unreachable_unreliable = True
|
|
2796
|
+
dead_syms = []
|
|
2797
|
+
|
|
2798
|
+
if code_graph is not None and not prebuilt_graph:
|
|
2799
|
+
try:
|
|
2800
|
+
from devcouncil.indexing.graph.build import write_code_graph
|
|
2801
|
+
|
|
2802
|
+
code_graph.generated_head = self._git_head()
|
|
2803
|
+
code_graph.indexed_hash = self._files_fingerprint(files)
|
|
2804
|
+
code_graph.content_fingerprint = self._content_fingerprint(files)
|
|
2805
|
+
write_code_graph(self.project_root, code_graph)
|
|
2806
|
+
except Exception:
|
|
2807
|
+
# A missing/stale code_graph.json silently degrades every graph
|
|
2808
|
+
# consumer — this must be visible, not a DEBUG-only whisper.
|
|
2809
|
+
logger.warning(
|
|
2810
|
+
"failed to write code graph export (.devcouncil/graph/code_graph.json)",
|
|
2811
|
+
exc_info=True,
|
|
2812
|
+
)
|
|
2813
|
+
|
|
2814
|
+
processes: List[Dict[str, object]] = []
|
|
2815
|
+
if code_graph is not None:
|
|
2816
|
+
raw_procs = code_graph.meta.get("processes") or []
|
|
2817
|
+
if isinstance(raw_procs, list):
|
|
2818
|
+
processes = [p for p in raw_procs[:12] if isinstance(p, dict)]
|
|
2819
|
+
|
|
2820
|
+
dependents, dependents_total = self.build_dependents(self._edges or [])
|
|
2821
|
+
subsystems = self._build_subsystem_index(files)
|
|
2822
|
+
|
|
1463
2823
|
return RepoMap(
|
|
1464
2824
|
languages=self.detect_languages(files),
|
|
1465
2825
|
frameworks=self.detect_frameworks(files),
|
|
@@ -1468,10 +2828,21 @@ class RepoMapper:
|
|
|
1468
2828
|
important_files=important_files,
|
|
1469
2829
|
candidate_files=candidates,
|
|
1470
2830
|
files=file_entries,
|
|
1471
|
-
subsystems=
|
|
1472
|
-
dependents=
|
|
2831
|
+
subsystems=subsystems,
|
|
2832
|
+
dependents=dependents,
|
|
2833
|
+
dependents_total=dependents_total,
|
|
1473
2834
|
generated_head=self._git_head(),
|
|
1474
2835
|
indexed_hash=self._files_fingerprint(files),
|
|
1475
|
-
|
|
2836
|
+
content_fingerprint=self._content_fingerprint(files),
|
|
2837
|
+
lsp=LspInspector(self.project_root).summary(files, client_enabled=lsp_refs),
|
|
1476
2838
|
dependency_risks=self._scan_dependency_risks() if scan_dependencies else [],
|
|
2839
|
+
entry_roots=entry_roots_list,
|
|
2840
|
+
unwired_candidates=unwired,
|
|
2841
|
+
unreachable_files=unreachable,
|
|
2842
|
+
dead_symbol_candidates=dead_syms,
|
|
2843
|
+
liveness_unreachable_unreliable=unreachable_unreliable,
|
|
2844
|
+
liveness_meta=liveness_meta,
|
|
2845
|
+
processes=processes,
|
|
1477
2846
|
)
|
|
2847
|
+
|
|
2848
|
+
RepositoryMapper = RepoMapper
|