woods 1.6.1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2035 -0
  3. data/CONTRIBUTING.md +253 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +655 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +455 -0
  19. data/docs/INTERNALS.md +418 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +213 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +171 -2
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2783 -238
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +147 -26
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +175 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +564 -87
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +236 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
  218. data/lib/woods/ruby_analyzer.rb +21 -5
  219. data/lib/woods/session_tracer/file_store.rb +138 -19
  220. data/lib/woods/session_tracer/middleware.rb +1 -2
  221. data/lib/woods/session_tracer/redis_store.rb +122 -12
  222. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  223. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  224. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  225. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  226. data/lib/woods/session_tracer/store.rb +14 -1
  227. data/lib/woods/storage/metadata_store.rb +230 -26
  228. data/lib/woods/storage/pgvector.rb +180 -22
  229. data/lib/woods/storage/qdrant.rb +367 -41
  230. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  231. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  232. data/lib/woods/storage/snapshotter.rb +23 -5
  233. data/lib/woods/storage/vector_store.rb +49 -8
  234. data/lib/woods/storage_identity.rb +28 -0
  235. data/lib/woods/tasks.rb +53 -2
  236. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  237. data/lib/woods/temporal/snapshot_store.rb +139 -42
  238. data/lib/woods/unblocked/client.rb +119 -17
  239. data/lib/woods/unblocked/document_builder.rb +34 -2
  240. data/lib/woods/unblocked/exporter.rb +63 -27
  241. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  242. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  243. data/lib/woods/update_check.rb +24 -1
  244. data/lib/woods/util/uuid5.rb +124 -0
  245. data/lib/woods/version.rb +1 -1
  246. data/lib/woods/watch/daemon.rb +1345 -0
  247. data/lib/woods/watch/listen_watcher.rb +81 -0
  248. data/lib/woods/watch/polling_watcher.rb +137 -0
  249. data/lib/woods/watch/status.rb +169 -0
  250. data/lib/woods/watch/tree_scan.rb +163 -0
  251. data/lib/woods/watch/watcher.rb +100 -0
  252. data/lib/woods.rb +138 -9
  253. data/plugin/.claude-plugin/plugin.json +18 -0
  254. data/plugin/hooks/hooks.json +29 -0
  255. data/plugin/hooks/woods-post-edit.sh +226 -0
  256. data/plugin/hooks/woods-session-start.sh +77 -0
  257. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  258. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  259. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  260. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  261. data/plugin/skills/woods-setup/SKILL.md +99 -0
  262. metadata +134 -23
  263. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  264. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  265. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  266. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  267. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  268. data/lib/woods/console/bridge.rb +0 -210
  269. data/lib/woods/formatting/claude_adapter.rb +0 -98
  270. data/lib/woods/formatting/generic_adapter.rb +0 -56
  271. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  272. data/lib/woods/notion/mapper.rb +0 -40
  273. data/lib/woods/observability/health_check.rb +0 -79
  274. data/lib/woods/observability/instrumentation.rb +0 -34
@@ -0,0 +1,206 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Woods
4
+ # What a resident, booted Woods process must do before a changed path can be
5
+ # re-extracted truthfully.
6
+ #
7
+ # Extraction reads the *runtime*: `ActiveRecord::Base.descendants`,
8
+ # `Rails.application.routes`, resolved `config` values, callback chains on
9
+ # loaded classes. So "the file changed" is not the same question as "what do
10
+ # I have to do before re-reading it is worth anything". Three answers matter,
11
+ # in increasing cost:
12
+ #
13
+ # * `:reextract` — Woods reads this file's bytes, not a constant.
14
+ # Re-extract; no Rails machinery has to run first.
15
+ # * `:reload` — an autoloaded constant changed. Reload through
16
+ # `Rails.application.reloader` first, or the extractor introspects the
17
+ # class that is already gone.
18
+ # * `:restart` — boot-captured state changed. No reloader re-runs it, so
19
+ # the process itself has to go.
20
+ #
21
+ # `:ignore` is the fourth answer and the common case: most of a repository
22
+ # is not extraction input at all.
23
+ #
24
+ # ## Why `:restart` is a real category
25
+ #
26
+ # Rails' reloader replaces autoloaded constants. It does not re-run
27
+ # initializers, re-read `config/*`, re-resolve `Rails.application.config`,
28
+ # or rebuild the schema cache. Woods captures all of those:
29
+ # `BehavioralProfile` introspects resolved config, `MiddlewareExtractor`
30
+ # reads the built middleware stack, model extraction reads column
31
+ # information. `rails/spring`'s well-documented staleness bugs came from
32
+ # exactly this — under-scoping the restart set — so the boundary here is
33
+ # drawn on the generous side.
34
+ #
35
+ # ## Version notes (railties >= 6.0)
36
+ #
37
+ # The classification is deliberately version-independent. Zeitwerk is the
38
+ # only autoloader from 6.0 on, and every path class below has meant the same
39
+ # thing throughout that range. Two version-sensitive behaviours sit *behind*
40
+ # the classification rather than in it, and belong to whoever implements the
41
+ # reload step:
42
+ #
43
+ # * `ActiveSupport::DescendantsTracker` internals changed across 6.0–8.x, so
44
+ # a reload can leave stale entries in a descendants set. Discovery-based
45
+ # extraction should re-read descendants *after* the reload completes, never
46
+ # across it.
47
+ # * A schema change needs `reset_column_information` plus schema-cache
48
+ # invalidation to be visible. That is classified `:restart` here rather
49
+ # than `:reload` because getting it right in-process is subtle and the
50
+ # payoff is small — schema changes are rare and already expensive.
51
+ #
52
+ # @example Deciding what a batch of file events requires
53
+ # policy = Woods::ReloadPolicy.new
54
+ # policy.classify_all(%w[app/models/user.rb config/initializers/redis.rb])
55
+ # # => :restart
56
+ #
57
+ class ReloadPolicy
58
+ # Ordered least to most disruptive. {#classify_all} returns the last one
59
+ # any path in the set demands.
60
+ ACTIONS = %i[ignore reextract reload restart].freeze
61
+
62
+ # Exact paths whose change invalidates boot-captured state.
63
+ RESTART_PATHS = %w[
64
+ Gemfile
65
+ Gemfile.lock
66
+ .ruby-version
67
+ config/application.rb
68
+ config/boot.rb
69
+ config/environment.rb
70
+ config/database.yml
71
+ config/credentials.yml.enc
72
+ config/master.key
73
+ db/schema.rb
74
+ db/structure.sql
75
+ ].freeze
76
+
77
+ # Boot-captured configuration that is not under an initializer directory.
78
+ #
79
+ # `BehavioralProfile` introspects *resolved* `Rails.application.config`, and
80
+ # these are the files that feed it outside `config/initializers`: the
81
+ # `config` gem's settings, and the per-adapter YAML Rails reads at boot.
82
+ # They were classified `:ignore`, which is precisely the Spring-style
83
+ # under-scoping this file's own commentary warns against — a daemon that
84
+ # ignores them keeps serving a profile derived from the previous values.
85
+ #
86
+ # `.env` files are matched by prefix rather than listed, because dotenv
87
+ # conventionally ships `.env`, `.env.local`, `.env.development` and friends.
88
+ # They also have to survive the watcher's dotfile filter to be seen at all.
89
+ # Named rather than wildcarded, deliberately. `config/*.yml` also catches
90
+ # files Woods reads as *bytes* — `config/recurring.yml` and
91
+ # `config/sidekiq_cron.yml` are scheduled-job sources, and escalating
92
+ # either to `:restart` would stop the daemon for a change it could simply
93
+ # re-extract. The boot-captured set is small and knowable; guessing at it
94
+ # costs more than listing it.
95
+ RESTART_PATH_PATTERNS = [
96
+ %r{\Aconfig/settings\.ya?ml\z},
97
+ %r{\Aconfig/settings/[^/]+\.ya?ml\z},
98
+ # Note what is absent: `config/recurring.yml` and `config/sidekiq_cron.yml`
99
+ # are scheduled-job *sources* that Woods reads as bytes
100
+ # (`ScheduledJobExtractor::SCHEDULE_FILES`), so they stay `:reextract`.
101
+ # `spec/reload_policy_spec.rb` asserts that, which is how the first draft
102
+ # of this list got caught.
103
+ %r{\Aconfig/(cable|storage|sidekiq|puma|cache|queue)\.ya?ml\z},
104
+ /\A\.env(\..+)?\z/
105
+ ].freeze
106
+
107
+ # Directory prefixes whose change invalidates boot-captured state.
108
+ RESTART_DIRECTORIES = %w[
109
+ config/initializers
110
+ config/environments
111
+ config/credentials
112
+ ].freeze
113
+
114
+ # Directories holding autoloaded application code. A `.rb` change here
115
+ # redefines a constant the extractors introspect.
116
+ RELOAD_DIRECTORIES = %w[app lib].freeze
117
+
118
+ # Route definitions. Reloading the route set is cheaper than a full
119
+ # reload (`Rails.application.reload_routes!`), but it is still a runtime
120
+ # refresh rather than a file re-read, so it lands in the same bucket.
121
+ ROUTE_PATHS = %w[config/routes.rb].freeze
122
+ ROUTE_DIRECTORIES = %w[config/routes].freeze
123
+
124
+ # Directories Woods reads as bytes. No constant is involved, so no Rails
125
+ # machinery has to run before re-extraction.
126
+ REEXTRACT_DIRECTORIES = %w[
127
+ app/views
128
+ config/locales
129
+ db/migrate
130
+ db/views
131
+ lib/tasks
132
+ spec
133
+ test
134
+ ].freeze
135
+
136
+ # Schedule files, which are data rather than code.
137
+ REEXTRACT_PATHS = %w[
138
+ config/recurring.yml
139
+ config/sidekiq_cron.yml
140
+ config/schedule.rb
141
+ ].freeze
142
+
143
+ # Packwerk boundary files, read as bytes by PackageExtractor (#280).
144
+ REEXTRACT_BASENAMES = %w[package.yml packwerk.yml].freeze
145
+
146
+ # What has to happen before this path can be re-extracted truthfully.
147
+ #
148
+ # @param relative_path [String] Rails.root-relative path
149
+ # @return [Symbol] one of {ACTIONS}
150
+ def classify(relative_path)
151
+ path = relative_path.to_s
152
+ return :restart if restart?(path)
153
+ return :reload if reload?(path)
154
+ return :reextract if reextract?(path)
155
+
156
+ :ignore
157
+ end
158
+
159
+ # The strongest action any path in the set demands.
160
+ #
161
+ # @param relative_paths [Enumerable<String>]
162
+ # @return [Symbol] one of {ACTIONS}
163
+ def classify_all(relative_paths)
164
+ Array(relative_paths).reduce(:ignore) do |strongest, path|
165
+ stronger_of(strongest, classify(path))
166
+ end
167
+ end
168
+
169
+ # Paths in the set that demand a given action.
170
+ #
171
+ # @param relative_paths [Enumerable<String>]
172
+ # @param action [Symbol] one of {ACTIONS}
173
+ # @return [Array<String>]
174
+ def paths_requiring(relative_paths, action)
175
+ Array(relative_paths).select { |path| classify(path) == action }
176
+ end
177
+
178
+ private
179
+
180
+ def stronger_of(left, right)
181
+ ACTIONS.index(left) >= ACTIONS.index(right) ? left : right
182
+ end
183
+
184
+ def restart?(path)
185
+ RESTART_PATHS.include?(path) || under?(path, RESTART_DIRECTORIES) ||
186
+ RESTART_PATH_PATTERNS.any? { |pattern| pattern.match?(path) }
187
+ end
188
+
189
+ def reload?(path)
190
+ return true if ROUTE_PATHS.include?(path) || under?(path, ROUTE_DIRECTORIES)
191
+ # lib/tasks and lib/generators are read as files, never autoloaded.
192
+ return false if under?(path, %w[lib/tasks lib/generators])
193
+
194
+ path.end_with?('.rb') && under?(path, RELOAD_DIRECTORIES)
195
+ end
196
+
197
+ def reextract?(path)
198
+ REEXTRACT_PATHS.include?(path) || under?(path, REEXTRACT_DIRECTORIES) ||
199
+ REEXTRACT_BASENAMES.include?(File.basename(path))
200
+ end
201
+
202
+ def under?(path, directories)
203
+ directories.any? { |dir| path.start_with?("#{dir}/") }
204
+ end
205
+ end
206
+ end
@@ -1,6 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Woods
4
+ # Same conditional-define pattern used elsewhere in the gem (pgvector,
5
+ # metadata_store) so this file can be required in isolation without
6
+ # tripping NameError on the CircuitOpenError superclass below.
7
+ class Error < StandardError; end unless defined?(Woods::Error)
8
+
4
9
  module Resilience
5
10
  # Raised when the circuit breaker is open and calls are being rejected.
6
11
  #
@@ -65,30 +70,64 @@ module Woods
65
70
  # @raise [StandardError] re-raises any error from the block
66
71
  def call(&block)
67
72
  probing = admit_call!
73
+ handled = false
68
74
 
69
75
  begin
70
76
  result = block.call
77
+ handled = true
71
78
  rescue CircuitOpenError
72
79
  # A nested breaker tripped — release our probe slot but don't count
73
80
  # it as this breaker's own failure.
74
- @mutex.synchronize { @half_open_probe_in_flight = false if probing }
81
+ handled = true
82
+ release_probe(probing)
75
83
  raise
76
84
  rescue StandardError => e
77
- @mutex.synchronize do
78
- @half_open_probe_in_flight = false if probing
79
- record_failure(probing)
80
- end
85
+ handled = true
86
+ finish_failure(probing)
81
87
  raise e
88
+ ensure
89
+ # A non-StandardError hits neither rescue above (Interrupt,
90
+ # Thread#raise-delivered exceptions, a custom Exception subclass).
91
+ # Release only the probe slot here — deliberately no
92
+ # record_failure, matching pre-existing state-transition rules.
93
+ release_probe(probing) unless handled
82
94
  end
83
95
 
96
+ finish_success(probing)
97
+ result
98
+ end
99
+
100
+ private
101
+
102
+ # Release the half_open probe slot without recording an outcome.
103
+ # Used by the CircuitOpenError rescue (a nested breaker's failure
104
+ # isn't this breaker's own) and the ensure fallback (a non-StandardError
105
+ # that skipped both rescue clauses).
106
+ #
107
+ # @param probing [Boolean] whether this call was admitted as the probe
108
+ def release_probe(probing)
109
+ @mutex.synchronize { @half_open_probe_in_flight = false if probing }
110
+ end
111
+
112
+ # Release the probe slot and record a failure, atomically.
113
+ #
114
+ # @param probing [Boolean] whether this call was admitted as the probe
115
+ def finish_failure(probing)
84
116
  @mutex.synchronize do
85
117
  @half_open_probe_in_flight = false if probing
86
- record_success(probing)
118
+ record_failure(probing)
87
119
  end
88
- result
89
120
  end
90
121
 
91
- private
122
+ # Release the probe slot and record a success, atomically.
123
+ #
124
+ # @param probing [Boolean] whether this call was admitted as the probe
125
+ def finish_success(probing)
126
+ @mutex.synchronize do
127
+ @half_open_probe_in_flight = false if probing
128
+ record_success(probing)
129
+ end
130
+ end
92
131
 
93
132
  # Decide whether this call may proceed, transitioning open→half_open when
94
133
  # the reset timeout has elapsed. Runs entirely under the mutex.
@@ -1,7 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'json'
4
+ require 'set'
4
5
  require_relative '../filename_utils'
6
+ require_relative '../atomic_file'
7
+
8
+ require_relative '../generation'
5
9
 
6
10
  module Woods
7
11
  module Resilience
@@ -13,11 +17,22 @@ module Woods
13
17
  # - Content hashes (source_hash) match the actual source_code
14
18
  # - No stale unit files exist that aren't listed in the index
15
19
  #
20
+ # **This class knows nothing about vectors or embedding dimensions.** Six
21
+ # documents used to credit it with detecting dimension mismatches; it never
22
+ # did (#214). That check is {Woods::MCP::DimensionMismatch}, raised by
23
+ # `Tasks.verify_store_dimensions!` before a durable embed run and by
24
+ # {Woods::Storage::Snapshotter::Vector} at MCP boot.
25
+ #
26
+ # Consumed by `spec/integration/multi_worktree_spec.rb` as a per-worktree
27
+ # integrity oracle. The `woods:validate` rake task performs an overlapping
28
+ # check inline rather than calling this — deliberate duplication left alone
29
+ # for now, since the task's output format is user-facing.
30
+ #
16
31
  # @example
17
32
  # validator = IndexValidator.new(index_dir: "tmp/woods")
18
33
  # report = validator.validate
19
34
  # puts report.errors if !report.valid?
20
- class IndexValidator
35
+ class IndexValidator # rubocop:disable Metrics/ClassLength
21
36
  include Woods::FilenameUtils
22
37
 
23
38
  # Report produced by {#validate}.
@@ -30,9 +45,31 @@ module Woods
30
45
  # @return [Array<String>] fatal integrity issues
31
46
  ValidationReport = Struct.new(:valid?, :warnings, :errors, keyword_init: true)
32
47
 
48
+ # The shared unit-type-directory contract, loadable without Rails:
49
+ # exactly the directories extraction publishes unit types into,
50
+ # derived from `Extractor::EXTRACTORS`. `woods/extractor` loads clean
51
+ # without a booted Rails app (the unit suite proves it), so this stays
52
+ # a plain require. A failure raises to the caller — it must convert
53
+ # the failure into a validation error rather than degrade to a
54
+ # silently empty allowlist, which would disable every structural
55
+ # type-directory check without saying so.
56
+ #
57
+ # @return [Array<String>]
58
+ # @raise [StandardError] when the extraction contract cannot be loaded
59
+ def self.unit_type_directories
60
+ require_relative '../extractor' unless defined?(Woods::Extractor::EXTRACTORS)
61
+
62
+ Woods::Extractor::EXTRACTORS.keys.map(&:to_s).freeze
63
+ end
64
+
33
65
  # @param index_dir [String] Path to the codebase index output directory
34
- def initialize(index_dir:)
66
+ # @param app_root [String, nil] the host application root; when given,
67
+ # a unit whose +file_path+ resolves neither as written nor under it
68
+ # is reported (the #169 staleness class: extracted elsewhere, or the
69
+ # source has since vanished)
70
+ def initialize(index_dir:, app_root: nil)
35
71
  @index_dir = index_dir
72
+ @app_root = app_root
36
73
  end
37
74
 
38
75
  # Validate the index directory and return a report.
@@ -47,20 +84,269 @@ module Woods
47
84
  return ValidationReport.new(valid?: false, warnings: warnings, errors: errors)
48
85
  end
49
86
 
50
- type_dirs = Dir.children(@index_dir).filter_map do |name|
51
- full_path = File.join(@index_dir, name)
52
- full_path if File.directory?(full_path)
53
- end
54
-
55
- type_dirs.each do |type_dir|
87
+ payload_type_dirs(errors).each do |type_dir|
56
88
  validate_type_directory(type_dir, warnings, errors)
57
89
  end
90
+ validate_flow_artifacts(errors)
91
+ validate_against_manifest(warnings, errors)
58
92
 
59
93
  ValidationReport.new(valid?: errors.empty?, warnings: warnings, errors: errors)
60
94
  end
61
95
 
96
+ # The checks `woods:validate` used to carry inline: manifest counts
97
+ # against the files on disk, every unit file parseable with an
98
+ # identifier and source, file paths that resolve, and a parseable
99
+ # dependency graph. Skipped when there is no manifest, so a bare
100
+ # type-directory tree (older fixtures, partial writes) still validates
101
+ # on the structural checks alone.
102
+ #
103
+ # @param warnings [Array<String>]
104
+ # @param errors [Array<String>]
105
+ def validate_against_manifest(warnings, errors)
106
+ payload = payload_dir
107
+ manifest_path = File.join(payload, 'manifest.json')
108
+ return unless File.exist?(manifest_path)
109
+
110
+ manifest = JSON.parse(Woods::AtomicFile.read(manifest_path))
111
+ unresolvable = Hash.new { |hash, key| hash[key] = [] }
112
+
113
+ (manifest['counts'] || {}).each do |type, expected_count|
114
+ validate_manifest_type(payload, type, expected_count, unresolvable, warnings, errors)
115
+ end
116
+
117
+ warn_unresolvable_paths(warnings, unresolvable)
118
+ validate_dependency_graph(payload, errors)
119
+ end
120
+
121
+ # Split each type's unresolvable units into app-tree paths and gem-owned
122
+ # paths, since only the first has a remedy the operator can act on.
123
+ #
124
+ # @param warnings [Array<String>]
125
+ # @param unresolvable [Hash{String => Array<Array(String, Boolean)>}]
126
+ def warn_unresolvable_paths(warnings, unresolvable)
127
+ unresolvable.each do |type, entries|
128
+ inside, outside = entries.partition { |_identifier, outside_root| !outside_root }
129
+ warn_unresolvable_app_paths(warnings, type, inside.map(&:first))
130
+ warn_unresolvable_gem_paths(warnings, type, outside.map(&:first))
131
+ end
132
+ end
133
+
134
+ # App-tree paths that do not exist under the app root: the index was
135
+ # extracted from a different tree, and re-extracting here fixes it.
136
+ def warn_unresolvable_app_paths(warnings, type, identifiers)
137
+ return if identifiers.empty?
138
+
139
+ warnings << "#{type}: #{identifiers.size} unit(s) whose file_path resolves nowhere " \
140
+ "(e.g. #{identifiers.first(3).join(', ')}). Extracted in a different environment? " \
141
+ 'Re-run extraction.'
142
+ end
143
+
144
+ # Absolute paths outside the app root belong to a gem — an engine model,
145
+ # a framework source — and resolve only where that gem is installed at
146
+ # the extracting path. Re-running extraction here cannot change that,
147
+ # so the remedy is not offered.
148
+ def warn_unresolvable_gem_paths(warnings, type, identifiers)
149
+ return if identifiers.empty?
150
+
151
+ warnings << "#{type}: #{identifiers.size} unit(s) whose file_path lies outside the app root " \
152
+ "and is absent here (e.g. #{identifiers.first(3).join(', ')}). Gem-owned units " \
153
+ '(engine models, framework sources) resolve only where that gem is installed at ' \
154
+ 'the extracting path.'
155
+ end
156
+
157
+ # rubocop:disable-next Metrics/ParameterLists
158
+ def validate_manifest_type(payload, type, expected_count, unresolvable, warnings, errors)
159
+ type_dir = File.join(payload, type)
160
+ unless File.directory?(type_dir)
161
+ errors << "Missing directory: #{type}"
162
+ return
163
+ end
164
+
165
+ unit_files = Dir[File.join(type_dir, '*.json')].reject { |f| f.end_with?('_index.json') }
166
+ warnings << "#{type}: expected #{expected_count}, found #{unit_files.size}" if unit_files.size != expected_count
167
+ unit_files.each { |file| validate_unit_file(file, type, unresolvable, errors) }
168
+ end
169
+
170
+ # @param file [String] unit JSON path
171
+ # @param type [String] type directory name
172
+ # @param unresolvable [Hash{String => Array<Array(String, Boolean)>}]
173
+ # `[identifier, outside_app_root]` pairs whose file_path resolves
174
+ # nowhere, keyed by type
175
+ # @param errors [Array<String>]
176
+ def validate_unit_file(file, type, unresolvable, errors)
177
+ data = JSON.parse(Woods::AtomicFile.read(file))
178
+ errors << "#{file}: missing identifier" unless data['identifier']
179
+ errors << "#{file}: missing source_code" unless data['source_code']
180
+ file_path = data['file_path']
181
+ return if path_resolvable?(file_path)
182
+
183
+ unresolvable[type] << [data['identifier'] || File.basename(file), outside_app_root?(file_path)]
184
+ rescue JSON::ParserError => e
185
+ errors << "#{file}: invalid JSON - #{e.message}"
186
+ end
187
+
188
+ # True when the check is off (no +app_root+), the unit has no path, or
189
+ # the path exists as written or under the app root.
190
+ def path_resolvable?(file_path)
191
+ return true if @app_root.nil? || file_path.nil?
192
+
193
+ File.exist?(file_path) || File.exist?(File.join(@app_root, file_path))
194
+ rescue StandardError
195
+ false
196
+ end
197
+
198
+ # True for an absolute path that is not under the app root. Extraction
199
+ # relativizes every path under Rails.root, so an absolute path in the
200
+ # index is one that was never in the app tree: a gem's.
201
+ def outside_app_root?(file_path)
202
+ return false unless file_path.to_s.start_with?('/')
203
+
204
+ !file_path.start_with?("#{@app_root.to_s.chomp('/')}/")
205
+ end
206
+
207
+ def validate_dependency_graph(payload, errors)
208
+ graph_path = File.join(payload, 'dependency_graph.json')
209
+ unless File.exist?(graph_path)
210
+ errors << 'Missing dependency_graph.json'
211
+ return
212
+ end
213
+
214
+ JSON.parse(Woods::AtomicFile.read(graph_path))
215
+ rescue JSON::ParserError
216
+ errors << 'dependency_graph.json: invalid JSON'
217
+ end
218
+
219
+ def payload_dir
220
+ Woods::Generation.new(output_dir: @index_dir).payload_dir.to_s
221
+ end
222
+
62
223
  private
63
224
 
225
+ # Resolve the published generation's payload and list its unit-type
226
+ # directories (e.g. models/, controllers/). An index that publishes
227
+ # per-generation payloads keeps `payloads/`, `dumps/` and `tasks/`
228
+ # beside them at the root, none of which are type directories.
229
+ #
230
+ # The list is bounded by a type-directory allowlist (G-2): a directory
231
+ # the allowlist does not claim — `flows/` above all — is not a unit-type
232
+ # directory and never reaches {#validate_type_directory}, which demands
233
+ # an `_index.json` no auxiliary artifact can satisfy. Before the
234
+ # allowlist, every directory under the payload was treated as a unit
235
+ # type, so any index published with flow precomputation enabled failed
236
+ # validation with "Missing _index.json in flows/".
237
+ #
238
+ # @param errors [Array<String>] accumulated errors; appended to if the
239
+ # payload directory named by the published generation isn't on disk,
240
+ # or the shared type-directory allowlist cannot be derived (a
241
+ # silently empty allowlist would disable every structural
242
+ # type-directory check without saying so)
243
+ # @return [Array<String>] absolute paths to type directories
244
+ def payload_type_dirs(errors)
245
+ payload = payload_dir
246
+ allowlist = derive_type_directory_allowlist(errors)
247
+ return [] if allowlist.nil?
248
+
249
+ Dir.children(payload).filter_map do |name|
250
+ full_path = File.join(payload, name)
251
+ next unless File.directory?(full_path)
252
+ next unless allowlist.include?(name)
253
+
254
+ full_path
255
+ end
256
+ rescue Errno::ENOENT
257
+ # A published `generation.json` pointing at a payload directory
258
+ # that isn't actually on disk (e.g. a generation bump raced a
259
+ # promote, or the payload was manually removed) is an index
260
+ # integrity problem this validator exists to report — not a
261
+ # crash for its caller to catch.
262
+ errors << "Payload directory does not exist: #{payload}"
263
+ []
264
+ end
265
+
266
+ # The allowlist, or nil — with a validation error recorded — when the
267
+ # shared contract cannot be derived. A silently empty allowlist would
268
+ # disable every structural type-directory check without saying so.
269
+ #
270
+ # @param errors [Array<String>] accumulated errors
271
+ # @return [Array<String>, nil]
272
+ def derive_type_directory_allowlist(errors)
273
+ type_directory_allowlist
274
+ rescue StandardError, ScriptError => e
275
+ # ScriptError too: a require failure surfaces as LoadError, which
276
+ # `rescue StandardError` does not catch.
277
+ errors << 'Could not derive the unit-type directory allowlist ' \
278
+ "(#{e.class}: #{e.message}); structural checks disabled"
279
+ nil
280
+ end
281
+
282
+ # @param name [String] directory basename under the payload
283
+ # @return [Boolean] true when extraction publishes unit types here
284
+ def unit_type_directory?(name)
285
+ type_directory_allowlist.include?(name)
286
+ end
287
+
288
+ # The shared allowlist: exactly the directories extraction publishes
289
+ # unit types into, derived from `Extractor::EXTRACTORS` so the two
290
+ # cannot drift. Required lazily — {IndexValidator} deliberately loads
291
+ # without Rails, and `woods/extractor` also loads clean. A derivation
292
+ # failure RAISES: {#payload_type_dirs} converts it to a validation
293
+ # error rather than degrading to a silently empty allowlist.
294
+ #
295
+ # `flows/` is deliberately absent: it holds `flow_index.json` and
296
+ # per-flow documents, which {#validate_flow_artifacts} owns.
297
+ #
298
+ # @return [Array<String>]
299
+ def type_directory_allowlist
300
+ @type_directory_allowlist ||= self.class.unit_type_directories
301
+ end
302
+
303
+ # Validate the flows/ artifact family (G-2): `flow_index.json` parses,
304
+ # and every entry points at a flow document that exists and parses.
305
+ #
306
+ # A payload from a run that never enabled flow precomputation has no
307
+ # flows directory at all, and an empty one holds nothing — both are
308
+ # absences. A POPULATED family with no index is corruption: the index
309
+ # is what defines which documents are live, so documents without it
310
+ # are unaccounted artifacts and are reported, not accepted.
311
+ #
312
+ # @param errors [Array<String>] accumulated errors
313
+ def validate_flow_artifacts(errors)
314
+ flows_dir = File.join(payload_dir, 'flows')
315
+ return unless File.directory?(flows_dir)
316
+ return if Dir.empty?(flows_dir)
317
+
318
+ index_path = File.join(flows_dir, 'flow_index.json')
319
+ unless File.exist?(index_path)
320
+ errors << 'flows/ is populated but flow_index.json is missing'
321
+ return
322
+ end
323
+
324
+ index = parse_artifact(index_path, 'flows/flow_index.json', errors)
325
+ return unless index.is_a?(Hash)
326
+
327
+ index.each_value do |relative|
328
+ filename = File.basename(relative.to_s)
329
+ document = File.join(flows_dir, filename)
330
+ unless File.exist?(document)
331
+ errors << "flow_index.json references missing document: flows/#{filename}"
332
+ next
333
+ end
334
+
335
+ parse_artifact(document, "flows/#{filename}", errors)
336
+ end
337
+ end
338
+
339
+ # @param path [String] artifact path
340
+ # @param label [String] how to name the artifact in an error
341
+ # @param errors [Array<String>] accumulated errors
342
+ # @return [Object, nil] parsed JSON, or nil when it does not parse
343
+ def parse_artifact(path, label, errors)
344
+ JSON.parse(Woods::AtomicFile.read(path))
345
+ rescue JSON::ParserError => e
346
+ errors << "#{label}: invalid JSON - #{e.message}"
347
+ nil
348
+ end
349
+
64
350
  # Validate a single type directory (e.g., models/, controllers/).
65
351
  #
66
352
  # @param type_dir [String] Absolute path to the type directory
@@ -75,7 +361,7 @@ module Woods
75
361
  return
76
362
  end
77
363
 
78
- index_entries = JSON.parse(File.read(index_path))
364
+ index_entries = JSON.parse(Woods::AtomicFile.read(index_path))
79
365
  indexed_identifiers = Set.new
80
366
 
81
367
  index_entries.each do |entry|
@@ -126,7 +412,7 @@ module Woods
126
412
  # @param identifier [String] The unit identifier (for error messages)
127
413
  # @param errors [Array<String>] Accumulated errors
128
414
  def validate_content_hash(unit_file, identifier, errors)
129
- data = JSON.parse(File.read(unit_file))
415
+ data = JSON.parse(Woods::AtomicFile.read(unit_file))
130
416
  source_code = data['source_code']
131
417
  stored_hash = data['source_hash']
132
418