woods 1.6.1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2035 -0
  3. data/CONTRIBUTING.md +253 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +655 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +455 -0
  19. data/docs/INTERNALS.md +418 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +213 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +171 -2
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2783 -238
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +147 -26
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +175 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +564 -87
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +236 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
  218. data/lib/woods/ruby_analyzer.rb +21 -5
  219. data/lib/woods/session_tracer/file_store.rb +138 -19
  220. data/lib/woods/session_tracer/middleware.rb +1 -2
  221. data/lib/woods/session_tracer/redis_store.rb +122 -12
  222. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  223. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  224. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  225. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  226. data/lib/woods/session_tracer/store.rb +14 -1
  227. data/lib/woods/storage/metadata_store.rb +230 -26
  228. data/lib/woods/storage/pgvector.rb +180 -22
  229. data/lib/woods/storage/qdrant.rb +367 -41
  230. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  231. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  232. data/lib/woods/storage/snapshotter.rb +23 -5
  233. data/lib/woods/storage/vector_store.rb +49 -8
  234. data/lib/woods/storage_identity.rb +28 -0
  235. data/lib/woods/tasks.rb +53 -2
  236. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  237. data/lib/woods/temporal/snapshot_store.rb +139 -42
  238. data/lib/woods/unblocked/client.rb +119 -17
  239. data/lib/woods/unblocked/document_builder.rb +34 -2
  240. data/lib/woods/unblocked/exporter.rb +63 -27
  241. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  242. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  243. data/lib/woods/update_check.rb +24 -1
  244. data/lib/woods/util/uuid5.rb +124 -0
  245. data/lib/woods/version.rb +1 -1
  246. data/lib/woods/watch/daemon.rb +1345 -0
  247. data/lib/woods/watch/listen_watcher.rb +81 -0
  248. data/lib/woods/watch/polling_watcher.rb +137 -0
  249. data/lib/woods/watch/status.rb +169 -0
  250. data/lib/woods/watch/tree_scan.rb +163 -0
  251. data/lib/woods/watch/watcher.rb +100 -0
  252. data/lib/woods.rb +138 -9
  253. data/plugin/.claude-plugin/plugin.json +18 -0
  254. data/plugin/hooks/hooks.json +29 -0
  255. data/plugin/hooks/woods-post-edit.sh +226 -0
  256. data/plugin/hooks/woods-session-start.sh +77 -0
  257. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  258. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  259. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  260. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  261. data/plugin/skills/woods-setup/SKILL.md +99 -0
  262. metadata +134 -23
  263. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  264. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  265. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  266. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  267. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  268. data/lib/woods/console/bridge.rb +0 -210
  269. data/lib/woods/formatting/claude_adapter.rb +0 -98
  270. data/lib/woods/formatting/generic_adapter.rb +0 -56
  271. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  272. data/lib/woods/notion/mapper.rb +0 -40
  273. data/lib/woods/observability/health_check.rb +0 -79
  274. data/lib/woods/observability/instrumentation.rb +0 -34
@@ -31,9 +31,6 @@ module Woods
31
31
  app/policies
32
32
  ].freeze
33
33
 
34
- # Method name patterns that indicate decision/eligibility logic
35
- DECISION_METHOD_PATTERN = /\b(allowed|eligible|valid|permitted|can_\w+|should_\w+|qualifies|meets_\w+|satisfies)\?/
36
-
37
34
  def initialize
38
35
  @directories = POLICY_DIRECTORIES.map { |d| Rails.root.join(d) }
39
36
  .select(&:directory?)
@@ -144,8 +141,12 @@ module Woods
144
141
 
145
142
  # From initialize params
146
143
  if source =~ /def\s+initialize\s*\(([^)]*)\)/
147
- params = ::Regexp.last_match(1)
148
- params.scan(/(\w+)/).flatten.each do |param|
144
+ # One parameter per comma. Keyword arguments (`strict: false`),
145
+ # default values (`user = nil`), and splats name no model, so only
146
+ # a bare positional identifier survives.
147
+ ::Regexp.last_match(1).split(',').each do |raw|
148
+ param = raw.strip[/\A(?:\*\*?|&)?([a-z_]\w*)\s*(?:=|\z)/, 1]
149
+ next if param.nil? || raw.strip.start_with?('*', '&')
149
150
  # Skip generic param names
150
151
  next if %w[args options params attributes context].include?(param)
151
152
 
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative 'shared_utility_methods'
4
4
  require_relative 'shared_dependency_scanner'
5
+ require_relative 'source_nesting'
5
6
 
6
7
  module Woods
7
8
  module Extractors
@@ -25,6 +26,7 @@ module Woods
25
26
  class PoroExtractor
26
27
  include SharedUtilityMethods
27
28
  include SharedDependencyScanner
29
+ include SourceNesting
28
30
 
29
31
  # Glob pattern for all Ruby files in app/models/ (recursive).
30
32
  MODELS_GLOB = 'app/models/**/*.rb'
@@ -123,25 +125,22 @@ module Woods
123
125
 
124
126
  # Infer the primary class name from source or fall back to file path.
125
127
  #
126
- # For regular class definitions we parse the first `class Foo` line,
127
- # joining outer module namespaces when present. For Struct.new / Data.define
128
- # patterns we read the constant assignment name. Falls back to the
129
- # Rails camelize convention on the relative path.
128
+ # For regular class definitions the position-aware nesting scan
129
+ # (SourceNesting) qualifies the first `class` declaration with the
130
+ # modules actually open at that position so a helper module nested
131
+ # inside the class, or a sibling module that closed before the class
132
+ # opened, no longer pollutes the identifier (#174). For Struct.new /
133
+ # Data.define patterns we read the constant assignment name. Falls back
134
+ # to the Rails camelize convention on the relative path.
130
135
  #
131
136
  # @param file_path [String] Absolute path to the file
132
137
  # @param source [String] Ruby source code
133
138
  # @return [String, nil] The inferred class name
134
139
  def infer_class_name(file_path, source)
135
- # Explicit class keyword — combine outer module namespaces + class name
136
- class_match = source.match(/^\s*class\s+([\w:]+)/)
137
- if class_match
138
- base = class_match[1]
139
- # If already fully qualified (e.g., Order::Update), use as-is
140
- return base if base.include?('::')
141
-
142
- namespaces = source.scan(/^\s*module\s+([\w:]+)/).flatten
143
- return namespaces.any? ? "#{namespaces.join('::')}::#{base}" : base
144
- end
140
+ # Explicit class keyword — Zeitwerk-governed naming first (G-1), then
141
+ # enclosing modules joined by position (#174)
142
+ qualified = governed_class_name(file_path, source) || qualified_first_class_name(source)
143
+ return qualified if qualified
145
144
 
146
145
  # Struct.new / Data.define: ConstantName = Struct.new(...)
147
146
  struct_match = source.match(/^(\w[\w:]*)\s*=\s*(?:Struct\.new|Data\.define)/)
@@ -83,9 +83,9 @@ module Woods
83
83
  # @param source [String]
84
84
  # @return [String, nil]
85
85
  def extract_class_name(file_path, source)
86
- return ::Regexp.last_match(1) if source =~ /^\s*class\s+([\w:]+)/
87
-
88
- file_path
86
+ # Zeitwerk-governed naming (G-1), then position-aware (#174), then
87
+ # convention.
88
+ governed_class_name(file_path, source) || qualified_first_class_name(source) || file_path
89
89
  .sub("#{Rails.root}/", '')
90
90
  .sub(%r{^app/policies/}, '')
91
91
  .sub('.rb', '')
@@ -13,6 +13,25 @@ module Woods
13
13
  #
14
14
  # Only high-value, frequently-referenced code is indexed to avoid bloat.
15
15
  #
16
+ # Emits TWO unit types — `:rails_source` (framework files) and
17
+ # `:gem_source` (configured gems) — both written under the `rails_source/`
18
+ # output directory. `Extractor::TYPE_TO_EXTRACTOR_KEY` maps both back to
19
+ # this extractor; keep that in sync if a type is ever added here.
20
+ #
21
+ # Unit `file_path`s are absolute installed-gem paths and stay
22
+ # environment-specific by design (#169): they are read from
23
+ # `Gem::Specification` in the extracting environment, there is no
24
+ # meaningful Rails.root-relative form for them, and rewriting them to some
25
+ # portable fiction would break the one thing they are for — opening the
26
+ # exact source the running app loads. An index read in a different
27
+ # environment simply cannot resolve them; `woods:validate` warns when
28
+ # that is the case rather than treating it as corruption.
29
+ #
30
+ # Kept current by the `Gemfile.lock` whole-app trigger (incremental runs),
31
+ # by full extraction when `include_framework_sources` is enabled (the
32
+ # default), and on demand via `woods:refresh[rails_source]` /
33
+ # `woods:extract_framework`.
34
+ #
16
35
  # @example
17
36
  # extractor = RailsSourceExtractor.new
18
37
  # units = extractor.extract_all
@@ -36,12 +55,11 @@ module Woods
36
55
  'lib/active_record/nested_attributes.rb'
37
56
  ],
38
57
  'actionpack' => [
58
+ # callbacks.rb, rendering.rb, redirecting.rb, and params_wrapper.rb
59
+ # all live under metal/, not directly under action_controller/ —
60
+ # the metal/ directory entry below already covers them.
39
61
  'lib/action_controller/metal',
40
- 'lib/action_controller/callbacks.rb',
41
- 'lib/abstract_controller/callbacks.rb',
42
- 'lib/action_controller/rendering.rb',
43
- 'lib/action_controller/redirecting.rb',
44
- 'lib/action_controller/params_wrapper.rb'
62
+ 'lib/abstract_controller/callbacks.rb'
45
63
  ],
46
64
  'activesupport' => [
47
65
  'lib/active_support/callbacks.rb',
@@ -58,8 +76,7 @@ module Woods
58
76
  ],
59
77
  'actionmailer' => [
60
78
  'lib/action_mailer/base.rb',
61
- 'lib/action_mailer/delivery_methods.rb',
62
- 'lib/action_mailer/callbacks.rb'
79
+ 'lib/action_mailer/delivery_methods.rb'
63
80
  ]
64
81
  }.freeze
65
82
 
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'line_neutralizer'
3
4
  require_relative 'shared_utility_methods'
4
5
  require_relative 'shared_dependency_scanner'
5
6
 
@@ -26,6 +27,20 @@ module Woods
26
27
  # Namespaces to exclude from extraction (this gem's own tasks)
27
28
  EXCLUDED_NAMESPACES = %w[woods].freeze
28
29
 
30
+ # Matches a line-leading `end` that closes a block, whether it stands
31
+ # alone or is followed by a method chain (`end.freeze`) or a closing
32
+ # delimiter (`end)`, `end]`, `end,`). An exact `== 'end'` check misses
33
+ # those, leaving the namespace/task depth counters out of sync with
34
+ # blocks that never get popped.
35
+ END_LINE = /\Aend\b/
36
+
37
+ # One task-dependency token: a quoted name (`'assets:precompile'`, whose
38
+ # namespace segments must survive — EXTB-9) or a plain symbol
39
+ # (`:environment`). The quoted branch comes first so the `:` inside a
40
+ # quoted name is consumed with it instead of being read as symbol
41
+ # syntax and truncated to its last segment.
42
+ DEPENDENCY_TOKEN = /(?:(['"])([\w:]+)\1|:(\w+))/
43
+
29
44
  def initialize
30
45
  @directories = RAKE_DIRECTORIES.map { |d| Rails.root.join(d) }.select(&:directory?)
31
46
  end
@@ -34,7 +49,7 @@ module Woods
34
49
  #
35
50
  # @return [Array<ExtractedUnit>] List of rake task units
36
51
  def extract_all
37
- find_files_in_directories(@directories, '**/*.rake').flat_map { |file| extract_rake_file(file) }
52
+ rake_files.flat_map { |file| extract_rake_file(file) }.uniq(&:identifier)
38
53
  end
39
54
 
40
55
  # Extract rake tasks from a single .rake file.
@@ -46,13 +61,16 @@ module Woods
46
61
  def extract_rake_file(file_path)
47
62
  return [] unless file_path.to_s.end_with?('.rake')
48
63
 
49
- source = File.read(file_path)
50
- tasks = parse_tasks(source)
64
+ data = rake_file_data(file_path.to_s)
65
+ return [] unless data
66
+
67
+ tasks = data[:tasks]
68
+ Rails.logger.debug("Woods: no rake tasks parsed from #{file_path}") if tasks.empty?
51
69
 
52
70
  tasks.filter_map do |task_data|
53
71
  next if excluded_namespace?(task_data[:full_name])
54
72
 
55
- build_unit(task_data, file_path, source)
73
+ build_unit(task_data, file_path, data[:source], sibling_definitions(task_data[:full_name], file_path))
56
74
  end
57
75
  rescue StandardError => e
58
76
  Rails.logger.error("Failed to extract rake tasks from #{file_path}: #{e.message}")
@@ -61,10 +79,70 @@ module Woods
61
79
 
62
80
  private
63
81
 
82
+ def rake_files
83
+ find_files_in_directories(@directories, '**/*.rake').map(&:to_s).sort
84
+ end
85
+
86
+ # One read and one parse per .rake file per extractor instance (audit
87
+ # P9a). {#extract_rake_file} and {#all_definitions} both need every
88
+ # file's source and tasks; the sibling index used to re-read and
89
+ # re-parse all of them on its first access. The bytes cannot change
90
+ # mid-run, so the first read/parse answers both paths.
91
+ #
92
+ # A file that fails to read or parse is memoized as nil: it produced no
93
+ # definitions and no units before either.
94
+ #
95
+ # @param file [String] Absolute path of the .rake file
96
+ # @return [Hash{Symbol => String, Array<Hash>}, nil] `{ source:, tasks: }`
97
+ def rake_file_data(file)
98
+ (@rake_file_data ||= {})[file] ||= begin
99
+ source = File.read(file)
100
+ { source: source, tasks: parse_tasks(source) }
101
+ end
102
+ rescue StandardError => e
103
+ Rails.logger.error("Failed to scan rake tasks in #{file}: #{e.message}")
104
+ nil
105
+ end
106
+
107
+ # Every definition of every task across the rake directories, keyed by
108
+ # full name (B-126). Rake merges a task reopened in two files into one
109
+ # task, so the index does the same: one unit whose source carries every
110
+ # definition. Built lazily once per extractor instance, which the
111
+ # orchestrator rebuilds per run.
112
+ #
113
+ # @return [Hash{String => Array<Hash>}] `{ full_name => [{ file:, task:, source: }] }`
114
+ def all_definitions
115
+ @all_definitions ||= rake_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, index|
116
+ data = rake_file_data(file)
117
+ next unless data
118
+
119
+ data[:tasks].each do |task_data|
120
+ index[task_data[:full_name]] << { file: file, task: task_data, source: data[:source] }
121
+ end
122
+ end
123
+ end
124
+
125
+ # Definitions of +full_name+ in files other than +file_path+.
126
+ #
127
+ # @return [Array<Hash>]
128
+ def sibling_definitions(full_name, file_path)
129
+ all_definitions[full_name].reject { |definition| definition[:file] == file_path.to_s }
130
+ end
131
+
64
132
  # Parse task definitions from rake source using a line-by-line state machine.
65
133
  #
66
134
  # Tracks namespace nesting, desc buffers, and task blocks.
67
135
  #
136
+ # Every structural decision (is this a declaration, does this line open
137
+ # or close a block) is made against the *neutralized* line — comments,
138
+ # string bodies, and heredoc bodies blanked by {LineNeutralizer} —
139
+ # while names and descriptions are still read from the raw line, which
140
+ # is where their string literals live. Scanning raw source let prose
141
+ # steer the parser: `# do not touch production` inflated depth with no
142
+ # matching `end`, so later namespaces inherited a stale prefix, and a
143
+ # heredoc line reading `end of the road` popped a task's own frame and
144
+ # truncated its body (EXTB-1).
145
+ #
68
146
  # @param source [String] Rake file source code
69
147
  # @return [Array<Hash>] Parsed task data
70
148
  def parse_tasks(source)
@@ -76,44 +154,51 @@ module Woods
76
154
  pending_desc = nil
77
155
  depth = 0
78
156
  lines = source.lines
157
+ scannable = LineNeutralizer.neutralize_lines(source)
79
158
 
80
159
  lines.each_with_index do |line, index|
81
160
  stripped = line.strip
161
+ code = scannable[index].to_s.strip
162
+ next if code.empty?
82
163
 
83
164
  # Track namespace blocks
84
- if stripped.match?(/\Anamespace\s+/)
165
+ if code.match?(/\Anamespace\s+/)
85
166
  name = extract_namespace_name(stripped)
86
167
  if name
87
168
  namespace_stack.push(name)
88
169
  namespace_depths.push(depth)
89
170
  depth += 1
171
+ elsif code.include?(' do')
172
+ # Unparseable namespace name (e.g. dynamic/interpolated) — still
173
+ # count its block so depth stays in sync with the closing `end`.
174
+ depth += 1
90
175
  end
91
176
  next
92
177
  end
93
178
 
94
179
  # Buffer desc for the next task
95
- if stripped.match?(/\Adesc\s+/)
180
+ if code.match?(/\Adesc\s+/)
96
181
  pending_desc = extract_desc(stripped)
97
182
  next
98
183
  end
99
184
 
100
185
  # Detect task definitions
101
- if stripped.match?(/\Atask\s+/)
186
+ if code.match?(/\Atask\s+/)
102
187
  task_data = parse_task_line(stripped, namespace_stack, pending_desc, index + 1)
103
188
  if task_data
104
- task_data[:block_source] = extract_task_block(lines, index)
189
+ task_data[:block_source] = extract_task_block(lines, scannable, index)
105
190
  tasks << task_data
106
191
  end
107
192
  pending_desc = nil
108
- depth += 1 if stripped.include?(' do')
193
+ depth += 1 if code.include?(' do')
109
194
  next
110
195
  end
111
196
 
112
197
  # Track block openers (non-namespace, non-task)
113
- depth += 1 if block_opener?(stripped)
198
+ depth += 1 if block_opener?(code)
114
199
 
115
200
  # Track end keywords
116
- next unless stripped == 'end'
201
+ next unless code.match?(END_LINE)
117
202
 
118
203
  depth -= 1
119
204
  # Pop namespace if we've returned to the depth where it was opened
@@ -128,11 +213,17 @@ module Woods
128
213
 
129
214
  # Extract the namespace name from a namespace declaration line.
130
215
  #
216
+ # Handles both the symbol form (`namespace :foo do`) and the string
217
+ # form (`namespace 'foo' do` / `namespace "foo" do`).
218
+ #
131
219
  # @param line [String] e.g. "namespace :foo do"
132
220
  # @return [String, nil] The namespace name
133
221
  def extract_namespace_name(line)
134
222
  match = line.match(/\Anamespace\s+:(\w+)/)
135
- match ? match[1] : nil
223
+ return match[1] if match
224
+
225
+ match = line.match(/\Anamespace\s+(['"])(\w+)\1/)
226
+ match ? match[2] : nil
136
227
  end
137
228
 
138
229
  # Extract the description string from a desc line.
@@ -176,6 +267,8 @@ module Woods
176
267
  # task :name => :dep
177
268
  # task :name => [:dep1, :dep2]
178
269
  # task :name, [:arg1, :arg2] => :dep
270
+ # task name: :dep
271
+ # task name: [:dep1, :dep2]
179
272
  #
180
273
  # @param line [String] The task line
181
274
  # @return [Array(String, Array<String>, Array<String>)] [name, deps, args]
@@ -202,6 +295,15 @@ module Woods
202
295
  return [name, deps, []]
203
296
  end
204
297
 
298
+ # Ruby 1.9 label form: task name: :dep / task name: [:dep1, :dep2]
299
+ # Ordered after the args branch so `task :name, [:args] => :dep`
300
+ # is never misread (the leading colon keeps it out of this regex).
301
+ if line.match(/\Atask\s+(\w+):\s*(.+?)(?:\s+do|\s*$)/)
302
+ name = ::Regexp.last_match(1)
303
+ deps = parse_dependency_list(::Regexp.last_match(2))
304
+ return [name, deps, []]
305
+ end
306
+
205
307
  # Simple task: task :name
206
308
  return [::Regexp.last_match(1), [], []] if line.match(/\Atask\s+:(\w+)/)
207
309
 
@@ -213,46 +315,62 @@ module Woods
213
315
  # @param dep_str [String] e.g. ":environment" or "[:dep1, :dep2]"
214
316
  # @return [Array<String>]
215
317
  def parse_dependency_list(dep_str)
216
- dep_str.scan(/:(\w+)/).flatten
318
+ dep_str.scan(DEPENDENCY_TOKEN).map { |_quote, quoted, symbol| quoted || symbol }
217
319
  end
218
320
 
219
321
  # Extract the task block body (lines between task...do and matching end).
220
322
  #
323
+ # Depth is counted against the neutralized lines so a keyword inside a
324
+ # comment, string, or heredoc body cannot open or close a block
325
+ # (EXTB-1); the body itself is collected from the raw lines.
326
+ #
221
327
  # @param lines [Array<String>] All source lines
328
+ # @param scannable [Array<String>] Neutralized counterparts of +lines+
222
329
  # @param task_line_index [Integer] 0-based index of the task line
223
330
  # @return [String] The block body source
224
- def extract_task_block(lines, task_line_index)
225
- task_line = lines[task_line_index]
226
- return '' unless task_line&.include?('do')
331
+ def extract_task_block(lines, scannable, task_line_index)
332
+ task_line = scannable[task_line_index]
333
+ # `block_opener?`, not a bare `include?('do')` substring check — a
334
+ # task named `docs` (`task docs: :environment`) contains the
335
+ # substring "do" in its own name, which made a blockless task look
336
+ # like it opened a `do` block and swallowed the following task's
337
+ # lines as its own body.
338
+ return '' unless task_line && block_opener?(task_line.strip)
227
339
 
228
340
  depth = 1
229
341
  body_lines = []
230
342
 
231
343
  ((task_line_index + 1)...lines.size).each do |i|
232
- line = lines[i]
233
- stripped = line.strip
344
+ code = scannable[i].to_s.strip
234
345
 
235
- depth += 1 if block_opener?(stripped)
236
- depth -= 1 if stripped == 'end'
346
+ depth += 1 if block_opener?(code)
347
+ depth -= 1 if code.match?(END_LINE)
237
348
 
238
349
  break if depth.zero?
239
350
 
240
- body_lines << line
351
+ body_lines << lines[i]
241
352
  end
242
353
 
243
354
  body_lines.join
244
355
  end
245
356
 
246
357
  # Check if a line opens a new block (do...end or def...end).
247
- # Note: if/unless only count as block openers when they start the line
248
- # (standalone form), not as trailing modifiers (e.g., `return if x`).
358
+ #
359
+ # +if+/+unless+ count as block openers in statement position: leading
360
+ # the line, or directly after an assignment operator (`value = if x`,
361
+ # `@memo ||= unless y` — the {Woods::Chunking::SemanticChunker}
362
+ # precedent). Trailing modifiers (`return if x`, `value = 1 if x`)
363
+ # must not count, and a line whose opener closes on the same line
364
+ # (`x = if a then b end`) is self-balancing: its trailing `end` never
365
+ # matches the line-leading {END_LINE}, so counting it would push a
366
+ # frame nothing pops — the same `end` guard the keyword branch uses.
249
367
  #
250
368
  # @param stripped [String] Stripped line content
251
369
  # @return [Boolean]
252
370
  def block_opener?(stripped)
253
371
  return true if stripped.match?(/\b(do|def|case|begin|class|module|while|until|for)\b.*(?<!\bend)\s*$/)
254
372
 
255
- stripped.match?(/\A(if|unless)\b/)
373
+ stripped.match?(/(?:\A|=\s*)(?:if|unless)\b.*(?<!\bend)\s*$/)
256
374
  end
257
375
 
258
376
  # Check if a task name falls under an excluded namespace.
@@ -269,21 +387,30 @@ module Woods
269
387
  # @param file_path [String] Path to the .rake file
270
388
  # @param file_source [String] Full file source
271
389
  # @return [ExtractedUnit]
272
- def build_unit(task_data, file_path, file_source)
390
+ def build_unit(task_data, file_path, file_source, siblings = [])
391
+ definitions = ([{ file: file_path.to_s, task: task_data, source: file_source }] + siblings)
392
+ .sort_by { |definition| definition[:file] }
393
+ primary = definitions.first
394
+
273
395
  unit = ExtractedUnit.new(
274
396
  type: :rake_task,
275
397
  identifier: task_data[:full_name],
276
- file_path: file_path
398
+ file_path: primary[:file]
277
399
  )
278
400
 
279
401
  unit.namespace = task_data[:task_namespace]
280
- unit.source_code = build_source_annotation(task_data, file_source)
281
- unit.metadata = build_metadata(task_data)
282
- unit.dependencies = extract_dependencies(task_data, file_source)
402
+ unit.source_code = definitions.map { |d| build_source_annotation(d[:task], d[:source]) }.join("\n\n")
403
+ unit.metadata = build_metadata(primary[:task])
404
+ unit.metadata[:defined_in] = definitions.map { |d| relative_path(d[:file]) } if definitions.size > 1
405
+ unit.dependencies = definitions.flat_map { |d| extract_dependencies(d[:task], d[:source]) }.uniq
283
406
 
284
407
  unit
285
408
  end
286
409
 
410
+ def relative_path(file)
411
+ file.to_s.sub("#{Rails.root}/", '')
412
+ end
413
+
287
414
  # Build annotated source code for the unit.
288
415
  #
289
416
  # @param task_data [Hash] Parsed task data
@@ -308,7 +435,8 @@ module Woods
308
435
  task_dependencies: task_data[:task_dependencies],
309
436
  arguments: task_data[:arguments],
310
437
  has_environment_dependency: task_data[:task_dependencies].include?('environment'),
311
- source_lines: (task_data[:block_source] || '').lines.size
438
+ source_lines: (task_data[:block_source] || '').lines.size,
439
+ line_number: task_data[:line_number]
312
440
  }
313
441
  end
314
442
 
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Woods
4
+ module Extractors
5
+ # Regexes for "this source references that class", shared by every site
6
+ # that scans Ruby text for service, job/worker, and mailer references.
7
+ #
8
+ # Three implementations of "detect an enqueue" used to disagree pairwise —
9
+ # the scanner missed `*Worker` and `.set(…)`, JobExtractor missed
10
+ # `*Worker`, CallbackAnalyzer missed `.set(…)` — so which units recorded
11
+ # an edge to a job depended on which extractor happened to look (EXTA-4).
12
+ # They all use {JOB_ENQUEUE} now.
13
+ #
14
+ # Every pattern is namespace-capable (EXTA-2). Since G-1 a namespaced
15
+ # unit's identifier is fully qualified (`Billing::ChargeService`), so a
16
+ # `\w+`-only capture recorded `ChargeService` — an edge target matching no
17
+ # node, invisible to `dependents`, PageRank, and the incremental blast
18
+ # radius. The leading `(?:\w+::)*` is greedy but backtracks to the last
19
+ # segment carrying the suffix, so `Billing::ChargeService::VERSION` still
20
+ # targets +Billing::ChargeService+.
21
+ module ReferencePatterns
22
+ # Async dispatch methods that enqueue a job. `set` is ActiveJob's
23
+ # delayed-enqueue entry point (`SyncJob.set(wait: 5).perform_later`).
24
+ ENQUEUE_METHODS = %w[perform_later perform_async perform_in perform_at].freeze
25
+
26
+ # `FooService.call` / `FooService::new`, namespace included.
27
+ SERVICE_REFERENCE = /((?:\w+::)*\w+Service)(?:\.|::)/
28
+
29
+ # `FooMailer.welcome`, namespace included.
30
+ MAILER_REFERENCE = /((?:\w+::)*\w+Mailer)\./
31
+
32
+ # `FooJob.perform_later` / `HardWorker.perform_async` /
33
+ # `SyncJob.set(wait: …).perform_later`, namespace included.
34
+ JOB_ENQUEUE =
35
+ /((?:\w+::)*\w+(?:Job|Worker))\.(?:#{ENQUEUE_METHODS.map { |m| Regexp.escape(m) }.join('|')}|set\b)/
36
+ end
37
+ end
38
+ end
@@ -31,7 +31,7 @@ module Woods
31
31
  return [] unless rails_routes_available?
32
32
 
33
33
  routes = Rails.application.routes.routes
34
- routes.filter_map { |route| extract_route(route) }
34
+ number_colliding_identifiers(routes.filter_map { |route| extract_route(route) })
35
35
  end
36
36
 
37
37
  private
@@ -59,7 +59,7 @@ module Woods
59
59
 
60
60
  verb = route_verb(route)
61
61
  path = route_path(route)
62
- identifier = "#{verb} #{path}"
62
+ identifier = route_identifier(verb, path, route)
63
63
 
64
64
  controller_class = "#{controller.camelize}Controller"
65
65
 
@@ -80,6 +80,62 @@ module Woods
80
80
  nil
81
81
  end
82
82
 
83
+ # Identifier for a route: `VERB /path`, qualified by its request
84
+ # constraints when it has any (B-127). Two routes that share a verb and
85
+ # path but differ by subdomain, header, or format would otherwise
86
+ # collapse into one unit, and the second silently vanished from the
87
+ # index. Path-segment requirements (`id: /\d+/`) do not qualify: they
88
+ # do not distinguish routes with the same path spec, and folding them
89
+ # in would rename every `resources` route with an `id` constraint.
90
+ #
91
+ # @return [String] e.g. "GET /users" or "GET /users [subdomain=api]"
92
+ def route_identifier(verb, path, route)
93
+ base = "#{verb} #{path}"
94
+ parts = identifier_constraints(path, route)
95
+ parts.empty? ? base : "#{base} [#{parts.join(', ')}]"
96
+ end
97
+
98
+ # @return [Array<String>] sorted `key=value` pairs for the constraints
99
+ # that qualify the identifier, plus `constraint=proc` when the route
100
+ # is wrapped by a callable constraint
101
+ def identifier_constraints(path, route)
102
+ segment_keys = path.scan(/:(\w+)/).flatten.map(&:to_sym)
103
+ pairs = route_constraints(route).reject { |key, _| segment_keys.include?(key.to_sym) }
104
+ requirements = route.respond_to?(:requirements) && route.requirements.is_a?(Hash) ? route.requirements : {}
105
+ pairs[:format] = requirements[:format] if requirements.key?(:format) && !pairs.key?(:format)
106
+
107
+ parts = pairs.sort_by { |key, _| key.to_s }.map { |key, value| "#{key}=#{constraint_value(value)}" }
108
+ parts << 'constraint=proc' if callable_constraint?(route)
109
+ parts
110
+ end
111
+
112
+ def constraint_value(value)
113
+ value.is_a?(Regexp) ? value.source : value.to_s
114
+ end
115
+
116
+ # A `constraints -> (req) { ... }` block wraps the endpoint in
117
+ # ActionDispatch::Routing::Mapper::Constraints, which exposes the
118
+ # callables as +constraints+.
119
+ def callable_constraint?(route)
120
+ app = route.respond_to?(:app) ? route.app : nil
121
+ app.respond_to?(:constraints) && app.constraints.is_a?(Array) && app.constraints.any?
122
+ end
123
+
124
+ # Routes still sharing an identifier after constraints are applied
125
+ # (two callable constraints, say) are numbered in route order:
126
+ # `GET /users`, `GET /users #2`. Deterministic for a given routes file,
127
+ # and the second route is indexed instead of dropped.
128
+ def number_colliding_identifiers(units)
129
+ seen = Hash.new(0)
130
+ units.each do |unit|
131
+ seen[unit.identifier] += 1
132
+ next if seen[unit.identifier] == 1
133
+
134
+ unit.identifier = "#{unit.identifier} ##{seen[unit.identifier]}"
135
+ end
136
+ units
137
+ end
138
+
83
139
  # Extract defaults hash from route, handling different Rails versions.
84
140
  #
85
141
  # @param route [ActionDispatch::Journey::Route]