woods 1.6.1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2035 -0
  3. data/CONTRIBUTING.md +253 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +655 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +455 -0
  19. data/docs/INTERNALS.md +418 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +213 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +171 -2
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2783 -238
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +147 -26
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +175 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +564 -87
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +236 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
  218. data/lib/woods/ruby_analyzer.rb +21 -5
  219. data/lib/woods/session_tracer/file_store.rb +138 -19
  220. data/lib/woods/session_tracer/middleware.rb +1 -2
  221. data/lib/woods/session_tracer/redis_store.rb +122 -12
  222. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  223. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  224. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  225. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  226. data/lib/woods/session_tracer/store.rb +14 -1
  227. data/lib/woods/storage/metadata_store.rb +230 -26
  228. data/lib/woods/storage/pgvector.rb +180 -22
  229. data/lib/woods/storage/qdrant.rb +367 -41
  230. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  231. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  232. data/lib/woods/storage/snapshotter.rb +23 -5
  233. data/lib/woods/storage/vector_store.rb +49 -8
  234. data/lib/woods/storage_identity.rb +28 -0
  235. data/lib/woods/tasks.rb +53 -2
  236. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  237. data/lib/woods/temporal/snapshot_store.rb +139 -42
  238. data/lib/woods/unblocked/client.rb +119 -17
  239. data/lib/woods/unblocked/document_builder.rb +34 -2
  240. data/lib/woods/unblocked/exporter.rb +63 -27
  241. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  242. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  243. data/lib/woods/update_check.rb +24 -1
  244. data/lib/woods/util/uuid5.rb +124 -0
  245. data/lib/woods/version.rb +1 -1
  246. data/lib/woods/watch/daemon.rb +1345 -0
  247. data/lib/woods/watch/listen_watcher.rb +81 -0
  248. data/lib/woods/watch/polling_watcher.rb +137 -0
  249. data/lib/woods/watch/status.rb +169 -0
  250. data/lib/woods/watch/tree_scan.rb +163 -0
  251. data/lib/woods/watch/watcher.rb +100 -0
  252. data/lib/woods.rb +138 -9
  253. data/plugin/.claude-plugin/plugin.json +18 -0
  254. data/plugin/hooks/hooks.json +29 -0
  255. data/plugin/hooks/woods-post-edit.sh +226 -0
  256. data/plugin/hooks/woods-session-start.sh +77 -0
  257. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  258. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  259. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  260. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  261. data/plugin/skills/woods-setup/SKILL.md +99 -0
  262. metadata +134 -23
  263. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  264. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  265. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  266. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  267. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  268. data/lib/woods/console/bridge.rb +0 -210
  269. data/lib/woods/formatting/claude_adapter.rb +0 -98
  270. data/lib/woods/formatting/generic_adapter.rb +0 -56
  271. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  272. data/lib/woods/notion/mapper.rb +0 -40
  273. data/lib/woods/observability/health_check.rb +0 -79
  274. data/lib/woods/observability/instrumentation.rb +0 -34
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ # lib/tasks/woods_checks.rake
4
+ #
5
+ # Deterministic checks over a published index (#280). None of these boot
6
+ # Rails: they read retained payload generations through Woods::PublishedIndex.
7
+ #
8
+ # Usage:
9
+ # bundle exec rake woods:check:moved_messages # previous vs published generation
10
+ # bundle exec rake "woods:check:moved_messages[41,42]" # explicit generations
11
+ # WOODS_CHECK_STRICT=1 ... # exit 1 on findings (still heuristic; see below)
12
+
13
+ require 'json'
14
+
15
+ namespace :woods do
16
+ namespace :check do
17
+ desc 'Candidate moves into a unit without mapped tests, between two retained generations ' \
18
+ '(heuristic: name+kind match, not a proven coverage loss; see docs/PUBLISHED_INDEX.md)'
19
+ task :moved_messages, %i[from to] do |_task, args|
20
+ Woods::CheckTasks.run_moved_messages(from: args[:from], to: args[:to])
21
+ end
22
+ end
23
+ end
24
+
25
+ module Woods
26
+ # Bodies of the `woods:check:*` tasks. A module, not bare defs, so nothing
27
+ # lands on Object in the host application.
28
+ module CheckTasks
29
+ module_function
30
+
31
+ # @param from [String, Integer, nil] older generation; defaults to the one before `to`
32
+ # @param to [String, Integer, nil] newer generation; defaults to the published one
33
+ # @return [void]
34
+ def run_moved_messages(from: nil, to: nil)
35
+ require 'woods/published_index'
36
+ require 'woods/checks/moved_messages'
37
+ require 'woods/checks/generation_resolution'
38
+
39
+ index_dir = check_index_dir
40
+ available = Woods::PublishedIndex.available_generations(index_dir)
41
+ resolved = Woods::Checks::GenerationResolution.call(available, from: from, to: to)
42
+ abort_for_insufficient_generations(index_dir, available) unless resolved
43
+
44
+ from_number, to_number = resolved
45
+ puts "Comparing generation #{from_number} to #{to_number} under #{index_dir}"
46
+
47
+ findings = compare_generations(index_dir, from_number, to_number)
48
+
49
+ print_moved_messages(findings)
50
+ exit 1 if findings.any? && ENV['WOODS_CHECK_STRICT'] == '1'
51
+ rescue Woods::PublishedIndex::CorruptPointerError => e
52
+ # Let it propagate (a corrupt install is not "0 findings"), but name the
53
+ # task first: a bare CorruptPointerError backtrace from deep inside
54
+ # PublishedIndex gives no hint this came from a rake task at all.
55
+ warn "woods:check:moved_messages: #{e.message}"
56
+ raise
57
+ end
58
+
59
+ # WOODS_OUTPUT, or tmp/woods beside the loaded Rakefile (the same
60
+ # resolution woods:watch_status uses; no Rails boot).
61
+ #
62
+ # @return [String]
63
+ def check_index_dir
64
+ ENV.fetch('WOODS_OUTPUT') do
65
+ root = respond_to?(:woods_task_root, true) ? woods_task_root : Rake.application.original_dir
66
+ File.join(root, 'tmp/woods')
67
+ end
68
+ end
69
+
70
+ # @param index_dir [String]
71
+ # @param from_number [Integer]
72
+ # @param to_number [Integer]
73
+ # @return [Array<Woods::Checks::MovedMessages::Finding>]
74
+ def compare_generations(index_dir, from_number, to_number)
75
+ Woods::PublishedIndex.open(index_dir, generation: from_number) do |before|
76
+ Woods::PublishedIndex.open(index_dir, generation: to_number) do |after|
77
+ Woods::Checks::MovedMessages.new(before: before, after: after).run
78
+ end
79
+ end
80
+ end
81
+
82
+ # @param index_dir [String]
83
+ # @param available [Array<Integer>]
84
+ # @return [void]
85
+ def abort_for_insufficient_generations(index_dir, available)
86
+ warn "ERROR: need two retained generations under #{index_dir}; found #{available.inspect}."
87
+ warn 'Run two extractions, or raise WOODS_PAYLOAD_RETENTION, or pass [from,to] explicitly.'
88
+ exit 1
89
+ end
90
+
91
+ # @param findings [Array<Woods::Checks::MovedMessages::Finding>]
92
+ # @return [void]
93
+ def print_moved_messages(findings)
94
+ if findings.empty?
95
+ puts 'No candidate moves into a unit without mapped tests.'
96
+ return
97
+ end
98
+
99
+ puts "#{findings.size} candidate move(s) into a unit without mapped tests:"
100
+ findings.each do |finding|
101
+ puts " #{finding.method} (#{finding.kind}): #{finding.from_unit} -> #{finding.to_unit} " \
102
+ "(covered before: #{finding.covered_before}, covered after: #{finding.covered_after})"
103
+ end
104
+ puts JSON.pretty_generate(findings.map(&:to_h)) if ENV['WOODS_CHECK_JSON'] == '1'
105
+ end
106
+ end
107
+ end
@@ -5,111 +5,195 @@
5
5
  # Rake tasks for evaluating retrieval quality.
6
6
  #
7
7
  # Usage:
8
- # bundle exec rake woods:evaluate # Run evaluation
9
- # bundle exec rake woods:evaluate:baseline[grep] # Run baseline comparison
8
+ # bundle exec rake woods:evaluate # Run evaluation
9
+ # bundle exec rake "woods:evaluate:baseline[grep]" # Run baseline comparison
10
+ #
11
+ # Loaded by the railtie alongside woods.rake. It previously was not loaded at
12
+ # all, so neither task existed on any host — and the code below called
13
+ # `Woods.vector_store` / `Woods.metadata_store` / `Woods.graph_store` /
14
+ # `Woods.embedding_provider`, none of which have ever existed. Both halves are
15
+ # fixed together (#212): stores now come from the MCP bootstrapper, the same
16
+ # path that hydrates persisted vector/metadata/graph artifacts for retrieval.
17
+ #
18
+ # Offline runs need no API key — set `embedding_provider = :fake` (#178).
19
+
20
+ require 'woods'
10
21
 
11
22
  namespace :woods do
12
23
  desc 'Run evaluation queries against the retrieval pipeline'
13
24
  task evaluate: :environment do
14
- require 'woods/retriever'
15
- require 'woods/evaluation/query_set'
16
- require 'woods/evaluation/evaluator'
17
- require 'woods/evaluation/report_generator'
18
-
19
- run_evaluation
25
+ Woods::EvaluationTasks.run_evaluation
20
26
  end
21
27
 
22
28
  namespace :evaluate do
23
- desc 'Run baseline comparison'
29
+ desc 'Run baseline comparison (strategy: grep, random, or file_level)'
24
30
  task :baseline, [:strategy] => :environment do |_t, args|
25
- require 'woods/evaluation/query_set'
26
- require 'woods/evaluation/baseline_runner'
27
- require 'woods/evaluation/metrics'
31
+ Woods::EvaluationTasks.run_baseline(strategy: args[:strategy])
32
+ end
28
33
 
29
- run_baseline(args)
34
+ desc 'Agent-level ablation: run a task set with the index on and off, ' \
35
+ 'each trial in a disposable worktree (no Rails boot)'
36
+ task :ablation, [:task_set] do |_t, args|
37
+ Woods::EvaluationTasks.run_ablation(task_set_path: args[:task_set])
30
38
  end
31
39
  end
32
40
  end
33
41
 
34
- def run_evaluation
35
- query_set_path = ENV.fetch('EVAL_QUERY_SET', 'config/eval_queries.json')
36
- output_path = ENV.fetch('EVAL_OUTPUT', 'tmp/eval_report.json')
37
- budget = ENV.fetch('EVAL_BUDGET', '8000').to_i
42
+ module Woods
43
+ # Bodies of the `woods:evaluate` rake tasks.
44
+ #
45
+ # A module rather than top-level `def`s: a rake file's bare methods land on
46
+ # Object and become callable on every object in the host application.
47
+ module EvaluationTasks
48
+ DEFAULT_QUERY_SET = 'config/eval_queries.json'
49
+ DEFAULT_OUTPUT = 'tmp/eval_report.json'
50
+ DEFAULT_BUDGET = 8000
51
+ DEFAULT_BASELINE_LIMIT = 10
52
+
53
+ module_function
54
+
55
+ # Evaluate the real retrieval pipeline against a ground-truth query set.
56
+ #
57
+ # @return [void]
58
+ def run_evaluation
59
+ require 'woods/mcp/bootstrapper'
60
+ require 'woods/evaluation/query_set'
61
+ require 'woods/evaluation/evaluator'
62
+ require 'woods/evaluation/report_generator'
63
+ require 'woods/evaluation/baseline'
38
64
 
39
- puts "Loading query set from: #{query_set_path}"
40
- query_set = Woods::Evaluation::QuerySet.load(query_set_path)
41
- puts "Loaded #{query_set.size} queries — building retriever..."
65
+ query_set_path, output_path, budget = eval_run_config
66
+ thresholds = build_thresholds
42
67
 
43
- evaluator = Woods::Evaluation::Evaluator.new(
44
- retriever: build_eval_retriever, query_set: query_set, budget: budget
45
- )
46
- report = evaluator.evaluate
68
+ puts "Loading query set from: #{query_set_path}"
69
+ query_set = Evaluation::QuerySet.load(query_set_path)
70
+ puts "Loaded #{query_set.size} queries — building retriever..."
47
71
 
48
- Woods::Evaluation::ReportGenerator.new
49
- .save(report, output_path, metadata: { 'query_set' => query_set_path })
72
+ report = Evaluation::Evaluator.new(
73
+ retriever: build_eval_retriever, query_set: query_set, budget: budget, thresholds: thresholds
74
+ ).evaluate
50
75
 
51
- print_eval_report(report, output_path)
52
- end
76
+ Evaluation::ReportGenerator.new
77
+ .save(report, output_path, metadata: { 'query_set' => query_set_path })
53
78
 
54
- def run_baseline(args)
55
- strategy = (args[:strategy] || ENV.fetch('EVAL_BASELINE_STRATEGY', 'grep')).to_sym
56
- query_set_path = ENV.fetch('EVAL_QUERY_SET', 'config/eval_queries.json')
57
- limit = ENV.fetch('EVAL_BASELINE_LIMIT', '10').to_i
79
+ print_eval_report(report, output_path)
80
+ exit 1 if report.threshold_report && !report.threshold_report.passed
81
+ end
58
82
 
59
- puts "Loading query set from: #{query_set_path}"
60
- query_set = Woods::Evaluation::QuerySet.load(query_set_path)
61
- puts "Running #{strategy} baseline (limit: #{limit})..."
83
+ # @return [Array(String, String, Integer)] query_set_path, output_path, budget
84
+ def eval_run_config
85
+ [ENV.fetch('EVAL_QUERY_SET', DEFAULT_QUERY_SET), ENV.fetch('EVAL_OUTPUT', DEFAULT_OUTPUT),
86
+ ENV.fetch('EVAL_BUDGET', DEFAULT_BUDGET.to_s).to_i]
87
+ end
62
88
 
63
- runner = Woods::Evaluation::BaselineRunner.new(
64
- metadata_store: Woods.metadata_store
65
- )
89
+ # Build the thresholds hash for the release-quality gate, or nil for the
90
+ # report-only default (no gate). `EVAL_BASELINE_FILE` (the versioned
91
+ # baseline format, see Evaluation::Baseline) seeds it; the three flat
92
+ # EVAL_MIN_* vars override individual metrics on top, so a one-off CI run
93
+ # can tighten a single metric without a new baseline file.
94
+ #
95
+ # @return [Hash{Symbol=>Float}, nil]
96
+ def build_thresholds
97
+ path = ENV.fetch('EVAL_BASELINE_FILE', nil)
98
+ thresholds = path ? Evaluation::Baseline.load(path).thresholds : {}
99
+ thresholds[:mean_precision_at5] = ENV['EVAL_MIN_PRECISION'].to_f if ENV['EVAL_MIN_PRECISION']
100
+ thresholds[:mean_recall] = ENV['EVAL_MIN_RECALL'].to_f if ENV['EVAL_MIN_RECALL']
101
+ thresholds[:mean_mrr] = ENV['EVAL_MIN_MRR'].to_f if ENV['EVAL_MIN_MRR']
102
+ thresholds.empty? ? nil : thresholds
103
+ end
66
104
 
67
- totals = compute_baseline_totals(query_set, runner, strategy, limit)
68
- print_baseline_report(strategy, query_set.size, totals)
69
- end
105
+ # Score a naive baseline strategy on the same query set, for comparison.
106
+ #
107
+ # @param strategy [String, nil] grep, random, or file_level
108
+ # @return [void]
109
+ def run_baseline(strategy: nil)
110
+ require 'woods/mcp/bootstrapper'
111
+ require 'woods/evaluation/query_set'
112
+ require 'woods/evaluation/baseline_runner'
113
+ require 'woods/evaluation/metrics'
70
114
 
71
- def compute_baseline_totals(query_set, runner, strategy, limit)
72
- total_mrr = 0.0
73
- total_recall = 0.0
115
+ strategy = (strategy || ENV.fetch('EVAL_BASELINE_STRATEGY', 'grep')).to_sym
116
+ query_set_path = ENV.fetch('EVAL_QUERY_SET', DEFAULT_QUERY_SET)
117
+ limit = ENV.fetch('EVAL_BASELINE_LIMIT', DEFAULT_BASELINE_LIMIT.to_s).to_i
74
118
 
75
- query_set.queries.each do |query|
76
- results = runner.run(query.query, strategy: strategy, limit: limit)
77
- total_mrr += Woods::Evaluation::Metrics.mrr(results, query.expected_units)
78
- total_recall += Woods::Evaluation::Metrics.recall(results, query.expected_units)
79
- end
119
+ puts "Loading query set from: #{query_set_path}"
120
+ query_set = Evaluation::QuerySet.load(query_set_path)
121
+ puts "Running #{strategy} baseline (limit: #{limit})..."
80
122
 
81
- { mrr: total_mrr, recall: total_recall }
82
- end
123
+ runner = Evaluation::BaselineRunner.new(metadata_store: build_eval_retriever.metadata_store)
124
+ totals = compute_baseline_totals(query_set, runner, strategy, limit)
83
125
 
84
- def print_eval_report(report, output_path)
85
- puts
86
- puts 'Evaluation complete!'
87
- puts '=' * 50
88
- report.aggregates.each do |key, value|
89
- formatted = value.is_a?(Float) ? format('%.4f', value) : value.to_s
90
- puts " #{key.to_s.ljust(25)}: #{formatted}"
91
- end
92
- puts '=' * 50
93
- puts "Report saved to: #{output_path}"
94
- end
126
+ print_baseline_report(strategy, query_set.size, totals)
127
+ end
95
128
 
96
- def print_baseline_report(strategy, count, totals)
97
- puts
98
- puts "Baseline: #{strategy}"
99
- puts '=' * 50
100
- puts " Mean MRR: #{format('%.4f', count.positive? ? totals[:mrr] / count : 0.0)}"
101
- puts " Mean Recall: #{format('%.4f', count.positive? ? totals[:recall] / count : 0.0)}"
102
- puts '=' * 50
103
- end
129
+ # @return [Hash] Summed MRR and recall across the query set
130
+ def compute_baseline_totals(query_set, runner, strategy, limit)
131
+ query_set.queries.each_with_object({ mrr: 0.0, recall: 0.0 }) do |query, totals|
132
+ results = runner.run(query.query, strategy: strategy, limit: limit)
133
+ totals[:mrr] += Evaluation::Metrics.mrr(results, query.expected_units)
134
+ totals[:recall] += Evaluation::Metrics.recall(results, query.expected_units)
135
+ end
136
+ end
104
137
 
105
- # Build a retriever for evaluation (requires Rails environment with stores configured).
106
- #
107
- # @return [Woods::Retriever]
108
- def build_eval_retriever
109
- Woods::Retriever.new(
110
- vector_store: Woods.vector_store,
111
- metadata_store: Woods.metadata_store,
112
- graph_store: Woods.graph_store,
113
- embedding_provider: Woods.embedding_provider
114
- )
138
+ # The configured retrieval pipeline, hydrated from persisted artifacts the
139
+ # same way the MCP server builds semantic search.
140
+ #
141
+ # @return [Woods::Retriever]
142
+ def build_eval_retriever
143
+ config = Woods.configuration || Woods.configure
144
+ retriever, _state = MCP::Bootstrapper.build_retriever(index_dir: config.output_dir)
145
+ retriever || raise(Woods::Error, 'Evaluation requires an embedded Woods index with a configured provider.')
146
+ end
147
+
148
+ def print_eval_report(report, output_path)
149
+ threshold_report = report.threshold_report
150
+ passed = threshold_report.nil? || threshold_report.passed
151
+ puts
152
+ puts(passed ? 'Evaluation complete!' : 'Evaluation FAILED — one or more thresholds not met.')
153
+ puts '=' * 50
154
+ report.aggregates.each do |key, value|
155
+ formatted = value.is_a?(Float) ? format('%.4f', value) : value.to_s
156
+ puts " #{key.to_s.ljust(25)}: #{formatted}"
157
+ end
158
+ puts '=' * 50
159
+ print_threshold_report(threshold_report) if threshold_report
160
+ puts "Report saved to: #{output_path}"
161
+ end
162
+
163
+ # Per-metric [PASS]/[FAIL] lines with the delta against the configured
164
+ # minimum. Absent unconditionally, same as today, when no thresholds were
165
+ # given — see the "Evaluation complete!" gate above.
166
+ def print_threshold_report(threshold_report)
167
+ puts 'Thresholds:'
168
+ threshold_report.metrics.sort.each do |key, metric|
169
+ status = metric[:passed] ? '[PASS]' : '[FAIL]'
170
+ actual = metric[:actual].nil? ? 'n/a' : format('%.4f', metric[:actual])
171
+ delta = metric[:delta].nil? ? 'n/a' : format('%+.4f', metric[:delta])
172
+ puts " #{status} #{key.to_s.ljust(25)} actual=#{actual} min=#{format('%.4f', metric[:threshold])} " \
173
+ "delta=#{delta}"
174
+ end
175
+ puts '=' * 50
176
+ end
177
+
178
+ def print_baseline_report(strategy, count, totals)
179
+ puts
180
+ puts "Baseline: #{strategy}"
181
+ puts '=' * 50
182
+ puts " Mean MRR: #{format('%.4f', count.positive? ? totals[:mrr] / count : 0.0)}"
183
+ puts " Mean Recall: #{format('%.4f', count.positive? ? totals[:recall] / count : 0.0)}"
184
+ puts '=' * 50
185
+ end
186
+
187
+ # Run the agent-level ablation harness and write a JSON report. This
188
+ # collects paired on/off runs; it is not causal evidence, see
189
+ # docs/EVALUATION.md. Deliberately has no `:environment` prerequisite:
190
+ # the harness shells out to an agent command and never boots Rails.
191
+ #
192
+ # @param task_set_path [String, nil]
193
+ # @return [void]
194
+ def run_ablation(task_set_path: nil)
195
+ require 'woods/evaluation/ablation_task'
196
+ Evaluation::AblationTask.run(task_set_path: task_set_path, workdir: Dir.pwd)
197
+ end
198
+ end
115
199
  end
@@ -53,21 +53,6 @@ module Woods
53
53
  calls.sort_by { |c| c[:line] }
54
54
  end
55
55
 
56
- # Extract only significant call sites, filtering out noise.
57
- #
58
- # @param node [Ast::Node] The AST node to search
59
- # @param known_units [Array<String>] Known unit identifiers for relevance filtering
60
- # @return [Array<Hash>] Filtered call site hashes
61
- def extract_significant(node, known_units: [])
62
- calls = extract(node)
63
- known_set = Set.new(known_units)
64
-
65
- calls.reject do |call|
66
- INSIGNIFICANT_METHODS.include?(call[:method_name]) &&
67
- (known_units.empty? || !known_set.include?(call[:receiver]))
68
- end
69
- end
70
-
71
56
  private
72
57
 
73
58
  def collect_calls(node, calls)
@@ -94,6 +79,12 @@ module Woods
94
79
  block: true
95
80
  }
96
81
  end
82
+ # Recurse into the send's own children — its receiver chain and its
83
+ # arguments — the way the non-block :send branch does (EXTB-16).
84
+ # Skipping them lost `where` from `User.where(active: true).each {}`
85
+ # while the same chain without a block recorded it. The send itself
86
+ # is emitted above, so recursing into its children adds no duplicate.
87
+ send_child.children&.each { |child| collect_calls(child, calls) } if send_child.is_a?(Ast::Node)
97
88
  # Also recurse into block body (children[1])
98
89
  node.children&.drop(1)&.each { |child| collect_calls(child, calls) }
99
90
  return # Don't double-recurse into children
@@ -38,15 +38,6 @@ module Woods
38
38
  end
39
39
  end
40
40
 
41
- # Extract all method definition nodes from source.
42
- #
43
- # @param source [String] Ruby source code
44
- # @return [Array<Ast::Node>] All :def and :defs nodes
45
- def extract_all_methods(source)
46
- root = @parser.parse(source)
47
- root.find_all(:def) + root.find_all(:defs)
48
- end
49
-
50
41
  # Extract the raw source text of a method, including def...end.
51
42
  #
52
43
  # This is the key replacement for `extract_action_source` in the controller
@@ -66,6 +57,25 @@ module Woods
66
57
  # Fallback: extract by line range
67
58
  extract_source_span(source, node.line, node.end_line)
68
59
  end
60
+
61
+ # Extract the raw source text of every instance-method definition,
62
+ # keyed by method name — one parse answers every query (P1).
63
+ #
64
+ # A name defined more than once keeps its first definition, matching
65
+ # {#extract_method_source}'s first-match lookup, so
66
+ # `extract_method_sources(source)[name]` is byte-identical to
67
+ # `extract_method_source(source, name)` for every name.
68
+ #
69
+ # @param source [String] Ruby source code
70
+ # @return [Hash{String => String}] method name => source text
71
+ def extract_method_sources(source)
72
+ root = @parser.parse(source)
73
+ root.find_all(:def).each_with_object({}) do |node, map|
74
+ next if map.key?(node.method_name)
75
+
76
+ map[node.method_name] = node.source || extract_source_span(source, node.line, node.end_line)
77
+ end
78
+ end
69
79
  end
70
80
  end
71
81
  end
@@ -277,7 +277,13 @@ module Woods
277
277
 
278
278
  # Convert receiver node so tree walking finds nested calls/constants
279
279
  receiver_node = prism_node.receiver ? convert_prism_node(prism_node.receiver, source) : nil
280
- children = [receiver_node].compact
280
+ # Argument nodes become children too (EXTB-5). Flattening arguments to
281
+ # source text alone hid every definition and call nested in an argument
282
+ # list — `private def hidden` lost the whole method, `foo(Bar.baz)` lost
283
+ # the `Bar.baz` call site. The `arguments` text field is unchanged for
284
+ # existing consumers; the children are appended *after* the receiver so
285
+ # nothing that reads children[0] as the receiver moves.
286
+ children = [receiver_node].compact + convert_prism_arguments(prism_node, source)
281
287
 
282
288
  # If there's a block, create a :block node wrapping this send
283
289
  if prism_node.block.is_a?(Prism::BlockNode)
@@ -311,8 +317,22 @@ module Woods
311
317
  )
312
318
  end
313
319
 
320
+ # Convert a call's argument nodes into walkable children.
321
+ #
322
+ # Only {Node} results are kept: literals convert to bare Strings or
323
+ # Integers, which carry nothing for a tree walker and would only make
324
+ # the children array noisier.
325
+ def convert_prism_arguments(prism_node, source)
326
+ return [] unless prism_node.arguments
327
+
328
+ prism_node.arguments.arguments.filter_map do |arg|
329
+ converted = convert_prism_node(arg, source)
330
+ converted if converted.is_a?(Node)
331
+ end
332
+ end
333
+
314
334
  def convert_prism_constant_path(prism_node, _source)
315
- parent_text = (extract_const_path_text(prism_node.parent) if prism_node.parent)
335
+ parent_text = prism_node.parent ? extract_const_path_text(prism_node.parent) : ''
316
336
  const_name = prism_node.respond_to?(:name) ? prism_node.name.to_s : prism_node.child.name.to_s
317
337
 
318
338
  Node.new(
@@ -331,13 +351,30 @@ module Woods
331
351
  condition = Node.new(**condition.to_h, source: condition_source)
332
352
  end
333
353
 
334
- then_body = prism_node.statements ? convert_prism_node(prism_node.statements, source) : nil
354
+ body = prism_node.statements ? convert_prism_node(prism_node.statements, source) : nil
335
355
  else_clause = prism_else_clause(prism_node)
336
356
  else_body = else_clause ? convert_prism_node(else_clause, source) : nil
337
357
 
358
+ # `unless` runs its body when the predicate is FALSE, so the body
359
+ # belongs in the else slot and the else clause in the then slot
360
+ # (EXTB-4). Sharing the unswapped :if shape made every flow document
361
+ # state the opposite of what the code does. After the swap the emitted
362
+ # `if <predicate>` reading is exact, so no negation marker is needed —
363
+ # the parser-gem branch below already normalizes `unless` the same way.
364
+ then_body, else_body = if prism_node.is_a?(Prism::UnlessNode)
365
+ [else_body, body]
366
+ else
367
+ [body, else_body]
368
+ end
369
+
338
370
  Node.new(
339
371
  type: :if,
340
- children: [condition, then_body, else_body].compact,
372
+ # Positional: [condition, then, else]. NO compact — a missing slot
373
+ # must stay a nil hole. A literal `nil` condition converts to nil
374
+ # (the NilNode mapping below), and compacting shifted the else body
375
+ # into children[1], which handle_conditional reported as then_ops
376
+ # (L2). Every consumer guards with `is_a?(Ast::Node)`.
377
+ children: [condition, then_body, else_body],
341
378
  line: line_for_prism(prism_node),
342
379
  end_line: end_line_for_prism(prism_node),
343
380
  source: condition_source
@@ -345,8 +382,13 @@ module Woods
345
382
  end
346
383
 
347
384
  def convert_prism_case(prism_node, source)
348
- children = []
349
- children << convert_prism_node(prism_node.predicate, source) if prism_node.predicate
385
+ # Positional: [predicate, *branches]. The predicate slot stays a nil
386
+ # hole for a predicate-less `case` (EXTB-15) — omitting it shifted the
387
+ # first `when` into children[0], where every consumer's `drop(1)` threw
388
+ # the branch away and rendered its condition as the word "when".
389
+ # Mirrors the L2 convention in {#convert_prism_if}.
390
+ predicate = (convert_prism_node(prism_node.predicate, source) if prism_node.predicate)
391
+ children = [predicate]
350
392
  prism_node.conditions.each { |c| children << convert_prism_node(c, source) }
351
393
  else_clause = prism_else_clause(prism_node)
352
394
  children << convert_prism_node(else_clause, source) if else_clause
@@ -428,7 +470,7 @@ module Woods
428
470
  when Prism::ConstantReadNode
429
471
  node.name.to_s
430
472
  when Prism::ConstantPathNode
431
- parent = node.parent ? extract_const_path_text(node.parent) : nil
473
+ parent = node.parent ? extract_const_path_text(node.parent) : ''
432
474
  const_name = node.respond_to?(:name) ? node.name.to_s : node.child.name.to_s
433
475
  [parent, const_name].compact.join('::')
434
476
  end
@@ -529,7 +571,9 @@ module Woods
529
571
  else_body = parser_node.children[2] ? convert_parser_node(parser_node.children[2], source) : nil
530
572
  Node.new(
531
573
  type: :if,
532
- children: [condition, then_body, else_body].compact,
574
+ # Positional [condition, then, else] with nil holes — see the
575
+ # Prism branch of convert_prism_if (L2).
576
+ children: [condition, then_body, else_body],
533
577
  line: parser_node.loc.line,
534
578
  end_line: parser_node.loc.expression&.last_line,
535
579
  source: condition_source
@@ -604,6 +648,8 @@ module Woods
604
648
  return nil unless node.is_a?(::Parser::AST::Node)
605
649
 
606
650
  case node.type
651
+ when :cbase
652
+ ''
607
653
  when :const
608
654
  parent = node.children[0] ? extract_parser_const_name(node.children[0]) : nil
609
655
  [parent, node.children[1].to_s].compact.join('::')