woods 2.0.0.beta2 → 2.0.0.beta4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +339 -1
  3. data/CONTRIBUTING.md +188 -12
  4. data/README.md +93 -174
  5. data/SECURITY.md +9 -6
  6. data/docs/AGENT_GUIDE.md +109 -8
  7. data/docs/AGENT_SETUP.md +98 -7
  8. data/docs/BACKEND_MATRIX.md +25 -0
  9. data/docs/CLIENT_HOOKS.md +111 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +267 -16
  11. data/docs/CONSOLE_MCP_SETUP.md +80 -7
  12. data/docs/DOCKER_SETUP.md +22 -3
  13. data/docs/EVALUATION.md +464 -1
  14. data/docs/EXTRACTOR_REFERENCE.md +45 -6
  15. data/docs/FAQ.md +11 -12
  16. data/docs/GETTING_STARTED.md +17 -5
  17. data/docs/INCREMENTAL_EXTRACTION.md +147 -7
  18. data/docs/INDEX_LAYOUT.md +382 -0
  19. data/docs/INTERNALS.md +7 -2
  20. data/docs/MCP_SERVERS.md +276 -5
  21. data/docs/MCP_TOOL_COOKBOOK.md +37 -22
  22. data/docs/MCP_WORKTREE_SETUP.md +43 -83
  23. data/docs/NOTION_INTEGRATION.md +13 -0
  24. data/docs/OBSIDIAN_INTEGRATION.md +57 -9
  25. data/docs/PUBLISHED_INDEX.md +72 -0
  26. data/docs/README.md +7 -0
  27. data/docs/RETRIEVAL_GUIDE.md +273 -12
  28. data/docs/RUNTIME_TRACING.md +71 -0
  29. data/docs/SOURCE_FRESHNESS.md +143 -0
  30. data/docs/TROUBLESHOOTING.md +129 -18
  31. data/docs/UNBLOCKED_INTEGRATION.md +25 -0
  32. data/docs/UPGRADING_TO_2.md +48 -22
  33. data/docs/WATCH_DAEMON.md +277 -67
  34. data/exe/woods-agent-config +6 -0
  35. data/exe/woods-extract +5 -0
  36. data/exe/woods-hook-context +6 -0
  37. data/exe/woods-mcp-start +14 -9
  38. data/lib/generators/woods/pgvector_generator.rb +8 -2
  39. data/lib/generators/woods/templates/woods.rb.tt +1 -3
  40. data/lib/tasks/woods.rake +47 -397
  41. data/lib/woods/agent_configuration/applier.rb +135 -0
  42. data/lib/woods/agent_configuration/cli.rb +101 -0
  43. data/lib/woods/agent_configuration/cli_options.rb +29 -0
  44. data/lib/woods/agent_configuration/document.rb +105 -0
  45. data/lib/woods/agent_configuration/error.rb +7 -0
  46. data/lib/woods/agent_configuration/launcher.rb +75 -0
  47. data/lib/woods/agent_configuration/layout.rb +72 -0
  48. data/lib/woods/agent_configuration/managed_section.rb +62 -0
  49. data/lib/woods/agent_configuration/plan.rb +98 -0
  50. data/lib/woods/agent_configuration/plan_diff.rb +38 -0
  51. data/lib/woods/agent_configuration/planned_files.rb +61 -0
  52. data/lib/woods/agent_configuration/planner.rb +63 -0
  53. data/lib/woods/agent_configuration/planner_validation.rb +77 -0
  54. data/lib/woods/agent_configuration/preflight.rb +100 -0
  55. data/lib/woods/agent_configuration/recovery.rb +49 -0
  56. data/lib/woods/ast/node.rb +2 -0
  57. data/lib/woods/ast/parser.rb +38 -5
  58. data/lib/woods/builder.rb +21 -5
  59. data/lib/woods/cache/cache_middleware.rb +28 -7
  60. data/lib/woods/cache/cache_store.rb +4 -5
  61. data/lib/woods/change_set.rb +5 -4
  62. data/lib/woods/console/credential_index.rb +20 -2
  63. data/lib/woods/console/credential_scanner.rb +18 -17
  64. data/lib/woods/console/credential_scanner_registry.rb +36 -0
  65. data/lib/woods/console/dispatch_pipeline.rb +7 -0
  66. data/lib/woods/console/embedded_executor.rb +32 -10
  67. data/lib/woods/console/encrypted_credential_snapshot.rb +16 -0
  68. data/lib/woods/console/rack_middleware.rb +22 -13
  69. data/lib/woods/console/server.rb +18 -16
  70. data/lib/woods/console/sql_noise_stripper.rb +9 -7
  71. data/lib/woods/console/sql_table_scanner.rb +47 -7
  72. data/lib/woods/console/sql_validator.rb +49 -9
  73. data/lib/woods/console/sqlite_read_guard.rb +46 -0
  74. data/lib/woods/coordination/pipeline_lock.rb +3 -2
  75. data/lib/woods/dependency_graph.rb +65 -13
  76. data/lib/woods/embedding/corpus.rb +94 -0
  77. data/lib/woods/embedding/indexer.rb +114 -60
  78. data/lib/woods/embedding/openai.rb +17 -6
  79. data/lib/woods/evaluation/ablation_executor.rb +6 -1
  80. data/lib/woods/evaluation/ablation_timed_executor.rb +22 -4
  81. data/lib/woods/export/typed_reader.rb +56 -0
  82. data/lib/woods/extractor.rb +277 -149
  83. data/lib/woods/extractors/action_cable_extractor.rb +3 -1
  84. data/lib/woods/extractors/behavioral_profile.rb +9 -7
  85. data/lib/woods/extractors/caching_extractor.rb +3 -1
  86. data/lib/woods/extractors/concern_extractor.rb +64 -6
  87. data/lib/woods/extractors/configuration_extractor.rb +7 -3
  88. data/lib/woods/extractors/controller_extractor.rb +13 -4
  89. data/lib/woods/extractors/database_view_extractor.rb +3 -1
  90. data/lib/woods/extractors/declared_parent.rb +55 -0
  91. data/lib/woods/extractors/decorator_extractor.rb +3 -1
  92. data/lib/woods/extractors/engine_extractor.rb +3 -1
  93. data/lib/woods/extractors/event_extractor.rb +4 -2
  94. data/lib/woods/extractors/factory_extractor.rb +3 -1
  95. data/lib/woods/extractors/graphql_extractor.rb +10 -13
  96. data/lib/woods/extractors/i18n_extractor.rb +3 -1
  97. data/lib/woods/extractors/job_extractor.rb +6 -19
  98. data/lib/woods/extractors/lib_extractor.rb +13 -9
  99. data/lib/woods/extractors/mailer_extractor.rb +26 -15
  100. data/lib/woods/extractors/manager_extractor.rb +3 -1
  101. data/lib/woods/extractors/method_parameters.rb +53 -0
  102. data/lib/woods/extractors/middleware_argument.rb +65 -0
  103. data/lib/woods/extractors/middleware_extractor.rb +9 -3
  104. data/lib/woods/extractors/migration_extractor.rb +3 -1
  105. data/lib/woods/extractors/model_extractor.rb +26 -34
  106. data/lib/woods/extractors/package_extractor.rb +24 -4
  107. data/lib/woods/extractors/phlex_extractor.rb +3 -1
  108. data/lib/woods/extractors/policy_extractor.rb +3 -1
  109. data/lib/woods/extractors/poro_extractor.rb +13 -9
  110. data/lib/woods/extractors/pundit_extractor.rb +3 -1
  111. data/lib/woods/extractors/rails_source_extractor.rb +4 -2
  112. data/lib/woods/extractors/rake_task_extractor.rb +4 -2
  113. data/lib/woods/extractors/route_extractor.rb +3 -1
  114. data/lib/woods/extractors/route_helper_resolver.rb +10 -33
  115. data/lib/woods/extractors/scheduled_job_extractor.rb +41 -15
  116. data/lib/woods/extractors/serializer_extractor.rb +4 -2
  117. data/lib/woods/extractors/service_extractor.rb +3 -1
  118. data/lib/woods/extractors/shared_dependency_scanner.rb +2 -2
  119. data/lib/woods/extractors/shared_utility_methods.rb +48 -19
  120. data/lib/woods/extractors/source_nesting.rb +1 -1
  121. data/lib/woods/extractors/state_machine_extractor.rb +3 -1
  122. data/lib/woods/extractors/test_mapping_extractor.rb +3 -1
  123. data/lib/woods/extractors/validator_extractor.rb +3 -1
  124. data/lib/woods/extractors/view_component_extractor.rb +3 -1
  125. data/lib/woods/extractors/view_template_extractor.rb +3 -1
  126. data/lib/woods/gem_mapper.rb +2 -0
  127. data/lib/woods/git_history.rb +116 -0
  128. data/lib/woods/graph_analyzer.rb +35 -6
  129. data/lib/woods/hooks/context_cli.rb +54 -0
  130. data/lib/woods/hooks/context_event.rb +88 -0
  131. data/lib/woods/hooks/context_hint.rb +73 -0
  132. data/lib/woods/hooks/context_impact.rb +77 -0
  133. data/lib/woods/hooks/context_output.rb +47 -0
  134. data/lib/woods/hooks/context_state.rb +102 -0
  135. data/lib/woods/hooks/refresh.rb +79 -0
  136. data/lib/woods/hooks/rule_projection.rb +78 -0
  137. data/lib/woods/input_rules.rb +19 -0
  138. data/lib/woods/mcp/bearer_auth.rb +22 -13
  139. data/lib/woods/mcp/bootstrapper.rb +79 -4
  140. data/lib/woods/mcp/config_resolver.rb +2 -1
  141. data/lib/woods/mcp/index_reader.rb +334 -162
  142. data/lib/woods/mcp/initialization_guidance.rb +27 -0
  143. data/lib/woods/mcp/origin_guard.rb +17 -9
  144. data/lib/woods/mcp/published_lexical_retriever.rb +115 -0
  145. data/lib/woods/mcp/renderers/markdown_renderer.rb +22 -9
  146. data/lib/woods/mcp/renderers/plain_renderer.rb +18 -8
  147. data/lib/woods/mcp/search_results.rb +74 -0
  148. data/lib/woods/mcp/server.rb +178 -63
  149. data/lib/woods/mcp/tool_contract.rb +3 -1
  150. data/lib/woods/mcp/tool_response_renderer.rb +41 -0
  151. data/lib/woods/mcp/traversal_evidence.rb +113 -0
  152. data/lib/woods/mcp/traversal_evidence_index.rb +100 -0
  153. data/lib/woods/mcp/traversal_evidence_page.rb +41 -0
  154. data/lib/woods/mcp/traversal_evidence_text.rb +52 -0
  155. data/lib/woods/mcp/traversal_response.rb +22 -0
  156. data/lib/woods/notion/exporter.rb +56 -17
  157. data/lib/woods/obsidian/destination_plan.rb +98 -0
  158. data/lib/woods/obsidian/name_mapper.rb +19 -3
  159. data/lib/woods/obsidian/note_builder.rb +19 -10
  160. data/lib/woods/obsidian/vault_exporter.rb +88 -32
  161. data/lib/woods/operator/pipeline_guard.rb +18 -13
  162. data/lib/woods/path_dispatcher.rb +13 -6
  163. data/lib/woods/payload_store.rb +27 -26
  164. data/lib/woods/published_index/typed_unit_reader.rb +40 -3
  165. data/lib/woods/published_index.rb +2 -2
  166. data/lib/woods/railtie.rb +3 -3
  167. data/lib/woods/railtie_support.rb +12 -12
  168. data/lib/woods/rake_helpers.rb +382 -0
  169. data/lib/woods/resilience/graph_invariant_validator/membership_checks.rb +71 -0
  170. data/lib/woods/resilience/graph_invariant_validator/node_checks.rb +61 -0
  171. data/lib/woods/resilience/graph_invariant_validator/reverse_relationship_checks.rb +46 -0
  172. data/lib/woods/resilience/graph_invariant_validator.rb +119 -0
  173. data/lib/woods/resilience/index_validator/graph_checks.rb +80 -0
  174. data/lib/woods/resilience/index_validator.rb +112 -23
  175. data/lib/woods/retrieval/context_assembler.rb +50 -15
  176. data/lib/woods/retrieval/lexical_assembler.rb +84 -0
  177. data/lib/woods/retrieval/lexical_index.rb +120 -0
  178. data/lib/woods/retrieval/ranker.rb +4 -2
  179. data/lib/woods/retrieval/scope.rb +108 -0
  180. data/lib/woods/retrieval/scoped_graph_store.rb +32 -0
  181. data/lib/woods/retrieval/scoped_vector_store.rb +55 -0
  182. data/lib/woods/retrieval/search_executor.rb +86 -27
  183. data/lib/woods/retrieval/source_evidence.rb +200 -0
  184. data/lib/woods/retriever.rb +98 -22
  185. data/lib/woods/ruby_analyzer/trace_enricher.rb +77 -38
  186. data/lib/woods/session_tracer/file_store.rb +6 -1
  187. data/lib/woods/session_tracer/middleware.rb +10 -12
  188. data/lib/woods/session_tracer/redis_store.rb +22 -6
  189. data/lib/woods/session_tracer/session_flow_assembler.rb +23 -17
  190. data/lib/woods/session_tracer/solid_cache_coordination.rb +6 -4
  191. data/lib/woods/session_tracer/unit_resolver.rb +63 -0
  192. data/lib/woods/source_inputs/consumer_errors.rb +31 -0
  193. data/lib/woods/source_inputs/handoff.rb +102 -0
  194. data/lib/woods/source_inputs/launcher.rb +157 -0
  195. data/lib/woods/source_inputs/manifest.rb +124 -0
  196. data/lib/woods/source_inputs/private_key.rb +55 -0
  197. data/lib/woods/source_inputs/scanner.rb +171 -0
  198. data/lib/woods/source_inputs/scopes.rb +71 -0
  199. data/lib/woods/source_inputs/session.rb +214 -0
  200. data/lib/woods/source_inputs/status.rb +84 -0
  201. data/lib/woods/source_inputs/verifier.rb +107 -0
  202. data/lib/woods/storage/metadata_store.rb +25 -25
  203. data/lib/woods/storage/pgvector.rb +35 -10
  204. data/lib/woods/storage/qdrant.rb +17 -7
  205. data/lib/woods/storage/vector_store.rb +18 -6
  206. data/lib/woods/tasks.rb +3 -2
  207. data/lib/woods/temporal/json_snapshot_store.rb +58 -9
  208. data/lib/woods/unblocked/exporter.rb +59 -70
  209. data/lib/woods/version.rb +1 -1
  210. data/lib/woods/watch/boot_snapshot.rb +52 -0
  211. data/lib/woods/watch/daemon.rb +154 -32
  212. data/lib/woods/watch/listen_watcher.rb +4 -0
  213. data/lib/woods/watch/polling_watcher.rb +5 -1
  214. data/lib/woods/watch/status.rb +20 -15
  215. data/lib/woods/watch/tree_scan.rb +21 -13
  216. data/lib/woods/watch/watcher.rb +4 -1
  217. data/lib/woods.rb +50 -11
  218. data/plugin/.claude-plugin/plugin.json +1 -1
  219. data/plugin/hooks/adapters/normalize.jq +15 -0
  220. data/plugin/hooks/adapters/normalize.rb +63 -0
  221. data/plugin/hooks/hooks.json +20 -0
  222. data/plugin/hooks/woods-context.sh +50 -0
  223. data/plugin/hooks/woods-input-rules.sh +159 -0
  224. data/plugin/hooks/woods-opencode.mjs +65 -0
  225. data/plugin/hooks/woods-post-edit.sh +2 -225
  226. data/plugin/hooks/woods-refresh.sh +260 -0
  227. data/plugin/hooks/woods-session-start.sh +47 -55
  228. data/plugin/skills/woods-agent-enable/SKILL.md +19 -0
  229. data/plugin/skills/woods-diagnose/SKILL.md +319 -1
  230. data/plugin/skills/woods-investigate/SKILL.md +145 -0
  231. data/plugin/skills/woods-mcp-config/SKILL.md +90 -2
  232. data/plugin/skills/woods-setup/SKILL.md +110 -6
  233. metadata +87 -5
@@ -8,6 +8,7 @@ require_relative 'client'
8
8
  require_relative 'rate_limiter'
9
9
  require_relative 'document_builder'
10
10
  require_relative 'sync_manifest'
11
+ require_relative '../export/typed_reader'
11
12
 
12
13
  module Woods
13
14
  module Unblocked
@@ -75,6 +76,7 @@ module Woods
75
76
 
76
77
  @client = client || Client.new(api_token: api_token, rate_limiter: limiter)
77
78
  @reader = reader || build_reader(index_dir)
79
+ @typed_reader = Export::TypedReader.new(@reader)
78
80
  # Cite the ref the index was actually extracted from. `main` was
79
81
  # hardcoded, so citations on a `master`-default repo pointed at a
80
82
  # branch that need not exist.
@@ -96,10 +98,11 @@ module Woods
96
98
  #
97
99
  # @return [Hash] { synced:, skipped:, deleted:, errors: }
98
100
  def sync_all
99
- with_pinned_index do
101
+ prepared = false
102
+ with_prepared_index do
103
+ prepared = true
100
104
  @current_uris = Set.new
101
105
  @budget_exhausted = false
102
- build_uri_index
103
106
  reconcile_from_remote if @manifest.empty?
104
107
 
105
108
  synced = 0
@@ -108,6 +111,7 @@ module Woods
108
111
 
109
112
  FULL_SYNC_TYPES.each do |type|
110
113
  break if @budget_exhausted
114
+ next if type.start_with?('graphql_') # the family already includes these actual types
111
115
 
112
116
  result = sync_type(type)
113
117
  synced += result[:synced]
@@ -124,11 +128,11 @@ module Woods
124
128
  errors.concat(result[:errors])
125
129
  end
126
130
 
127
- deleted = @budget_exhausted ? 0 : purge_stale(errors)
131
+ deleted = @budget_exhausted || @ambiguous_uris.any? ? 0 : purge_stale(errors)
128
132
  { synced: synced, skipped: skipped, deleted: deleted, errors: cap_errors(errors) }
129
133
  end
130
134
  ensure
131
- save_manifest
135
+ save_manifest if prepared
132
136
  end
133
137
 
134
138
  # Sync all units of a given type.
@@ -136,10 +140,12 @@ module Woods
136
140
  # @param type [String] Unit type (e.g. "model", "controller")
137
141
  # @return [Hash] { synced:, skipped:, errors: }
138
142
  def sync_type(type)
139
- units = @reader.list_units(type: type)
140
- log " #{type}: #{units.size} units"
143
+ with_prepared_index do
144
+ units = units_for(type)
145
+ log " #{type}: #{units.size} units"
141
146
 
142
- sync_units(units)
147
+ sync_unit_data(units.map { |unit| [unit, unit] })
148
+ end
143
149
  end
144
150
 
145
151
  # Sync the top N most-connected units of a type (by dependent count).
@@ -148,35 +154,42 @@ module Woods
148
154
  # @param max_count [Integer] Maximum units to sync
149
155
  # @return [Hash] { synced:, skipped:, errors: }
150
156
  def sync_type_partial(type, max_count)
151
- units = @reader.list_units(type: type)
152
- return empty_stats if units.empty?
153
-
154
- # Load full data to sort by dependent count
155
- units_with_data = units.filter_map do |entry|
156
- data = @reader.find_unit(entry['identifier'])
157
- next unless data
158
-
159
- dep_count = (data['dependents'] || []).size
160
- { entry: entry, data: data, dep_count: dep_count }
157
+ with_prepared_index do
158
+ units = units_for(type)
159
+ units.each { |unit| track_uri(unit) }
160
+ top = units.sort_by { |unit| -(unit['dependents'] || []).size }.first(max_count)
161
+ log " #{type}: #{top.size}/#{units.size} units (top by dependents)"
162
+ result = sync_unit_data(top.map { |unit| [unit, unit] })
163
+ result[:skipped] += units.size - top.size
164
+ result
161
165
  end
166
+ end
162
167
 
163
- # Every unit of this type still exists — track its URI so partial units
164
- # that fall *out* of the top-N are never mistaken for deletions.
165
- units_with_data.each { |u| track_uri(u[:data]) }
166
-
167
- top_units = units_with_data.sort_by { |u| -u[:dep_count] }.first(max_count)
168
- # Count against what was actually synced — units.size includes entries
169
- # whose unit data was missing (dropped by the filter_map above).
170
- skipped_count = units.size - top_units.size
168
+ private
171
169
 
172
- log " #{type}: #{top_units.size}/#{units.size} units (top by dependents)"
170
+ # Complete preflight precedes all remote writes and destructive pruning.
171
+ # Nested sync methods share this snapshot; standalone methods pin as well.
172
+ def with_prepared_index
173
+ return yield if @prepared_index
173
174
 
174
- result = sync_unit_data(top_units.map { |u| [u[:entry], u[:data]] })
175
- result[:skipped] += skipped_count
176
- result
175
+ with_pinned_index do
176
+ @published_units = @typed_reader.all
177
+ build_uri_index
178
+ @prepared_index = true
179
+ begin
180
+ yield
181
+ ensure
182
+ @prepared_index = false
183
+ @published_units = nil
184
+ end
185
+ end
177
186
  end
178
187
 
179
- private
188
+ def units_for(type)
189
+ # The historical graphql family includes all four actual subtypes.
190
+ types = type == 'graphql' ? MCP::IndexReader::UNIT_TYPES_BY_DIR.fetch('graphql') : [type]
191
+ @published_units.select { |unit| types.include?(unit['type']) }
192
+ end
180
193
 
181
194
  # Run a multi-read export body against one index generation.
182
195
  #
@@ -197,36 +210,6 @@ module Woods
197
210
  @reader.with_pinned_generation(&block)
198
211
  end
199
212
 
200
- def sync_units(units)
201
- synced = 0
202
- skipped = 0
203
- errors = []
204
-
205
- units.each do |entry|
206
- unit_data = @reader.find_unit(entry['identifier'])
207
- unless unit_data
208
- skipped += 1
209
- next
210
- end
211
-
212
- track_uri(unit_data)
213
- if push_document(unit_data) == :skipped
214
- skipped += 1
215
- else
216
- synced += 1
217
- end
218
- rescue Woods::Error => e
219
- errors << "#{entry['identifier']}: #{e.message}"
220
- break if note_budget_exhaustion(e)
221
- rescue StandardError => e
222
- # Include the class — "undefined method for nil" without it is
223
- # unactionable in CI logs.
224
- errors << "#{entry['identifier']}: #{e.class}: #{e.message}"
225
- end
226
-
227
- { synced: synced, skipped: skipped, errors: errors }
228
- end
229
-
230
213
  def sync_unit_data(entries_with_data)
231
214
  synced = 0
232
215
  skipped = 0
@@ -256,6 +239,10 @@ module Woods
256
239
  #
257
240
  # @return [Symbol] :synced or :skipped
258
241
  def push_document(unit_data)
242
+ if @ambiguous_uris.include?(@builder.uri_for(unit_data))
243
+ raise ExtractionError,
244
+ "ambiguous export URI for #{unit_data['type']}:#{unit_data['identifier']} — push skipped"
245
+ end
259
246
  # No file_path → the URI falls back to the bare repo URL, which every
260
247
  # such unit would share: they'd overwrite each other remotely and
261
248
  # ping-pong the manifest hash forever. Skip them.
@@ -411,24 +398,26 @@ module Woods
411
398
  "#{base}?unit=#{URI.encode_www_form_component(unit_data['identifier'])}"
412
399
  end
413
400
 
414
- # One cheap pass over the type indexes (entries already carry file_path,
415
- # and read_index is cached) to find files that define more than one synced
416
- # unit. For each such base URI, the lexically-smallest identifier — the
401
+ # Inspect the complete validated snapshot, including excluded types, for
402
+ # same-name cross-type collisions. For files with distinct synced names,
403
+ # the lexically-smallest identifier — the
417
404
  # outer/top-level class — keeps the bare URI; siblings are suffixed. Solo
418
405
  # files (the overwhelming majority) are absent from the map and unchanged,
419
406
  # so this introduces no churn for them.
420
407
  def build_uri_index
421
408
  groups = Hash.new { |h, k| h[k] = [] }
422
- synced_types.each do |type|
423
- @reader.list_units(type: type).each do |entry|
424
- next unless entry['file_path']
409
+ @published_units.each do |unit|
410
+ next unless unit['file_path']
425
411
 
426
- groups[@builder.uri_for(entry)] << entry['identifier']
427
- end
412
+ groups[@builder.uri_for(unit)] << unit
428
413
  end
429
414
 
430
- @uri_primary = groups.each_with_object({}) do |(uri, identifiers), primary|
431
- unique = identifiers.uniq
415
+ @ambiguous_uris = groups.each_with_object(Set.new) do |(uri, units), ambiguous|
416
+ identities = units.map { |unit| [unit['identifier'], unit['type']] }.uniq
417
+ ambiguous << uri if identities.group_by(&:first).any? { |_, variants| variants.size > 1 }
418
+ end
419
+ @uri_primary = groups.each_with_object({}) do |(uri, units), primary|
420
+ unique = units.select { |unit| synced_types.include?(unit['type']) }.map { |unit| unit['identifier'] }.uniq
432
421
  primary[uri] = unique.min if unique.size > 1
433
422
  end
434
423
  end
data/lib/woods/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Woods
4
- VERSION = '2.0.0.beta2'
4
+ VERSION = '2.0.0.beta4'
5
5
  end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../reload_policy'
4
+ require_relative 'tree_scan'
5
+ require_relative 'watcher'
6
+
7
+ module Woods
8
+ module Watch
9
+ # Files observed before the watch task invokes Rails' environment task.
10
+ # This bounds environment initialization, not earlier Bundler/application
11
+ # loading. Hosts embedding the daemon may provide the same explicit boundary.
12
+ class BootSnapshot
13
+ def initialize(root:, policy: ReloadPolicy.new)
14
+ @root = File.expand_path(root.to_s)
15
+ @policy = policy
16
+ @files = scan
17
+ end
18
+
19
+ # True for unchanged files, including a carried path absent before and
20
+ # after boot. A deleted initializer must not demand another restart.
21
+ def covers?(path)
22
+ absolute = File.expand_path(path, @root)
23
+ @files[absolute] == fingerprint(absolute)
24
+ end
25
+
26
+ # Includes additions and deletions, even if another writer has advanced
27
+ # the index watermark while Rails was booting.
28
+ def changed_paths
29
+ current = scan
30
+ (@files.keys | current.keys).reject { |path| @files[path] == current[path] }
31
+ end
32
+
33
+ private
34
+
35
+ def scan
36
+ TreeScan.files(root: @root, ignored: Watcher::DEFAULT_IGNORED_DIRECTORIES).each_with_object({}) do |path, files|
37
+ next unless %i[restart reload].include?(@policy.classify(path.delete_prefix("#{@root}/")))
38
+
39
+ stamp = fingerprint(path)
40
+ files[path] = stamp if stamp
41
+ end
42
+ end
43
+
44
+ def fingerprint(path)
45
+ stat = File.stat(path)
46
+ [stat.ino, stat.size, stat.mtime, stat.ctime]
47
+ rescue Errno::ENOENT, Errno::ENOTDIR
48
+ nil
49
+ end
50
+ end
51
+ end
52
+ end
@@ -9,9 +9,11 @@ require_relative '../reload_policy'
9
9
  require_relative 'status'
10
10
  require_relative 'tree_scan'
11
11
  require_relative 'watcher'
12
+ require_relative 'boot_snapshot'
12
13
  require 'json'
13
14
  require 'set'
14
15
  require 'securerandom'
16
+ require 'timeout'
15
17
 
16
18
  module Woods
17
19
  module Watch
@@ -138,6 +140,9 @@ module Woods
138
140
  # which most daemon cycles are (a single-file cycle is milliseconds).
139
141
  HEARTBEAT_SHUTDOWN_TIMEOUT = 2
140
142
 
143
+ # Native registration or a polling baseline must finish before catch-up.
144
+ WATCHER_STARTUP_TIMEOUT = 30
145
+
141
146
  # @return [Woods::Generation]
142
147
  attr_reader :generation
143
148
 
@@ -156,6 +161,9 @@ module Woods
156
161
  # @param lock [Woods::Coordination::PipelineLock, nil] writer lock for
157
162
  # this index; built from {LOCK_NAME} when nil
158
163
  # @param catch_up [Boolean] reconcile changes that predate startup
164
+ # @param poll_interval [Float] finite, positive seconds between polling scans
165
+ # @param boot_snapshot [BootSnapshot, nil] captured before environment
166
+ # initialization; permits full reconciliation of unchanged startup inputs
159
167
  # @param force_polling [Boolean] never use the `listen` backend — the
160
168
  # right choice across a container bind mount, where native FS events
161
169
  # do not propagate
@@ -166,7 +174,9 @@ module Woods
166
174
  def initialize(output_dir:, root: nil, extractor_factory: nil, reloader: nil, watcher: nil,
167
175
  policy: ReloadPolicy.new, debounce: DEFAULT_DEBOUNCE,
168
176
  full_extraction_threshold: DEFAULT_FULL_EXTRACTION_THRESHOLD,
169
- idle_timeout: nil, lock: nil, catch_up: true, force_polling: false, logger: nil)
177
+ idle_timeout: nil, lock: nil, catch_up: true, force_polling: false,
178
+ boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil)
179
+ @poll_interval = validated_poll_interval(poll_interval)
170
180
  @output_dir = output_dir.to_s
171
181
  @root = (root || (defined?(Rails) ? Rails.root : Dir.pwd)).to_s
172
182
  @extractor_factory = extractor_factory || -> { Woods::Extractor.new(output_dir: @output_dir) }
@@ -177,6 +187,7 @@ module Woods
177
187
  @full_extraction_threshold = full_extraction_threshold
178
188
  @idle_timeout = idle_timeout
179
189
  @catch_up = catch_up
190
+ @boot_snapshot = boot_snapshot
180
191
  @force_polling = force_polling
181
192
  @logger = logger || default_logger
182
193
  @generation = Generation.new(output_dir: @output_dir)
@@ -231,11 +242,19 @@ module Woods
231
242
  # @return [Hash] `{ action:, state:, generation:, reason:, count:,
232
243
  # duration_ms: }`
233
244
  def process(paths = [])
245
+ process_batch(paths)
246
+ end
247
+
248
+ private
249
+
250
+ # Only the resident loop can discharge work covered by its explicit
251
+ # environment-boot snapshot. Direct #process calls remain conservative.
252
+ def process_batch(paths = [], startup: false)
234
253
  change_set = ChangeSet.new(paths: drain_with(paths), root: @root)
235
254
  cycle_completed = false
236
255
 
237
256
  begin
238
- result = case required_action(change_set)
257
+ result = case required_action(change_set, startup: startup)
239
258
  when :ignore then nothing_to_do
240
259
  when :restart then require_restart(change_set)
241
260
  when :reload then attempt_reload ? extract(change_set) : degraded_reload(change_set)
@@ -249,13 +268,20 @@ module Woods
249
268
  # shutdown's heartbeat.kill lands on a thread doing real work) would
250
269
  # otherwise lose the batch #drain_with already popped from @pending.
251
270
  # Every branch above that finishes normally already carries its own
252
- # paths forward on failure (or intentionally doesn't, e.g. :restart);
271
+ # paths forward on failure or restart;
253
272
  # this only fires for the abnormal case none of them can catch.
254
273
  carry_forward(change_set) unless cycle_completed
255
274
  end
256
275
  end
257
276
 
258
- private
277
+ def validated_poll_interval(value)
278
+ interval = Float(value)
279
+ unless interval.finite? && interval.positive?
280
+ raise ArgumentError, 'poll_interval (WOODS_WATCH_POLL_INTERVAL) must be finite and greater than zero'
281
+ end
282
+
283
+ interval
284
+ end
259
285
 
260
286
  # The watch loop proper, split from {#run} so the shutdown `ensure` —
261
287
  # publish `stopped`, persist carried paths — can only ever fire for a
@@ -273,11 +299,19 @@ module Woods
273
299
  @last_event_at = monotonic_now
274
300
  heartbeat = start_heartbeat
275
301
 
276
- watcher_thread = launch_watcher do |paths|
277
- enqueue(paths)
278
- drain
302
+ # Watch immediately, but establish startup obligations before any
303
+ # callback can drain them. Live callbacks enqueue under a separate mutex.
304
+ watcher_thread = nil
305
+ @drain_mutex.synchronize do
306
+ watcher_thread = launch_watcher do |paths|
307
+ enqueue(paths)
308
+ drain
309
+ end
310
+ await_watcher_ready
311
+ catch_up unless @stop_reason
312
+ drain_cycles
313
+ @last_event_at = monotonic_now
279
314
  end
280
- catch_up
281
315
  watcher_thread.join
282
316
  raise @watcher_failure if @watcher_failure
283
317
 
@@ -308,7 +342,7 @@ module Woods
308
342
  # plausibly) before reaching that line, and it must still happen
309
343
  # before #persist_pending — otherwise a watcher thread still draining
310
344
  # a batch races the shutdown snapshot of @pending.
311
- watcher_thread&.join
345
+ watcher_thread&.join(HEARTBEAT_SHUTDOWN_TIMEOUT) || watcher_thread&.kill&.join
312
346
  stop_heartbeat(heartbeat)
313
347
  # Before #persist_pending, for the same reason the watcher join is: a
314
348
  # retry drain still running would race the shutdown snapshot of
@@ -346,9 +380,18 @@ module Woods
346
380
  start_watching(&on_change)
347
381
  rescue StandardError => e
348
382
  @watcher_failure = e
383
+ ensure
384
+ @watcher_ready << :finished
349
385
  end
350
386
  end
351
387
 
388
+ def await_watcher_ready
389
+ Timeout.timeout(WATCHER_STARTUP_TIMEOUT) { @watcher_ready.pop }
390
+ raise @watcher_failure if @watcher_failure
391
+ rescue Timeout::Error
392
+ raise WatcherError, 'watcher did not establish its baseline before the startup timeout'
393
+ end
394
+
352
395
  # @param heartbeat [Thread, nil]
353
396
  # @return [void]
354
397
  def stop_heartbeat(heartbeat)
@@ -363,8 +406,12 @@ module Woods
363
406
  # `:reload` by restarting, since extracting against constants that no
364
407
  # longer match their source is the thing the classification exists to
365
408
  # prevent.
366
- def required_action(change_set)
367
- action = @policy.classify_all(change_set.relative_paths)
409
+ def required_action(change_set, startup: false)
410
+ paths = change_set.absolute_paths
411
+ paths = paths.reject { |path| startup_covered?(path) } if startup
412
+ relative = ChangeSet.new(paths: paths, root: @root).relative_paths
413
+ action = @policy.classify_all(relative)
414
+ action = :reextract if action == :ignore && @startup_full
368
415
  return :restart if action == :reload && !@reloader.enabled?
369
416
 
370
417
  action
@@ -386,10 +433,14 @@ module Woods
386
433
 
387
434
  def reset_cycle_state
388
435
  @pending = Set.new
436
+ @startup_paths = Set.new
437
+ @live_paths = Set.new
438
+ @startup_full = false
389
439
  @pending_mutex = Mutex.new
390
440
  @stop_reason = nil
391
441
  @drain_mutex = Mutex.new
392
442
  @retry_thread = nil
443
+ @watcher_ready = Queue.new
393
444
  end
394
445
 
395
446
  # Wait out the debounce window so events that land during it join the
@@ -405,9 +456,12 @@ module Woods
405
456
  end
406
457
 
407
458
  # Merge a watcher batch into the pending set without processing it.
408
- def enqueue(paths)
459
+ def enqueue(paths, startup: false)
409
460
  absolute = ChangeSet.new(paths: paths, root: @root).absolute_paths
410
- @pending_mutex.synchronize { @pending.merge(absolute) }
461
+ @pending_mutex.synchronize do
462
+ @pending.merge(absolute)
463
+ @live_paths.merge(absolute) unless startup
464
+ end
411
465
  @last_event_at = monotonic_now
412
466
  end
413
467
 
@@ -457,7 +511,7 @@ module Woods
457
511
  def drain_cycles
458
512
  until pending_empty? || @stop_reason
459
513
  settle
460
- result = process
514
+ result = process_batch(startup: true)
461
515
 
462
516
  if result[:action] == :restart
463
517
  # @watcher, not a captured local — start_watching's polling
@@ -529,7 +583,11 @@ module Woods
529
583
  return unless @catch_up
530
584
 
531
585
  carried = restore_pending
532
- paths = (uncovered_paths + carried).uniq
586
+ paths = (uncovered_paths + carried + vanished_restart_paths).uniq
587
+ boot_changes = @boot_snapshot ? @boot_snapshot.changed_paths : []
588
+ enqueue(boot_changes) unless boot_changes.empty?
589
+ prepare_startup_reconciliation(paths)
590
+ paths |= boot_changes
533
591
  if paths.empty?
534
592
  return reconcile_deletions if stale_deletions?
535
593
 
@@ -537,17 +595,36 @@ module Woods
537
595
  end
538
596
 
539
597
  @logger.info("[Woods] watch: #{paths.size} path(s) changed before startup — catching up")
540
- enqueue(paths)
598
+ enqueue(paths, startup: true)
541
599
  drain
542
600
  end
543
601
 
602
+ def prepare_startup_reconciliation(paths)
603
+ return unless @boot_snapshot
604
+
605
+ restart_paths = paths.select do |path|
606
+ set = ChangeSet.new(paths: [path], root: @root)
607
+ required_action(set) == :restart && @boot_snapshot.covers?(path)
608
+ end
609
+ @pending_mutex.synchronize do
610
+ @startup_paths.merge(restart_paths)
611
+ @startup_full = restart_paths.any?
612
+ end
613
+ end
614
+
615
+ def startup_covered?(path)
616
+ eligible = @pending_mutex.synchronize { @startup_paths.include?(path) && !@live_paths.include?(path) }
617
+ eligible && @boot_snapshot.covers?(path)
618
+ end
619
+
544
620
  # A file deleted while nothing was watching leaves no mtime for the scan
545
621
  # to see — {TreeScan} only walks files that exist — so a deletion-only
546
622
  # downtime would log "index is current" while ghost units survive. The
547
623
  # graph knows every path it attributed a unit to; any of those gone from
548
624
  # disk means the extractor's sweep has reconciling to do.
549
625
  #
550
- # The daemon only *detects*; it does not name the paths. Naming them
626
+ # Except for boot inputs handled by vanished_restart_paths, the daemon
627
+ # only detects; it does not name the paths. Naming them
551
628
  # would put them in the change set, whose deletions are authoritative for
552
629
  # any unit type — and some registered paths are nominal (on Rails < 7.1,
553
630
  # `ActiveRecord::SchemaMigration` registers a convention path no app
@@ -555,9 +632,22 @@ module Woods
555
632
  # still produces. An empty-change-set run reaches the same ghosts through
556
633
  # the sweep, which carries the bounds that make it safe.
557
634
  def stale_deletions?
558
- root_prefix = "#{@root}/"
635
+ vanished_registered_paths.any?
636
+ end
559
637
 
560
- persisted_registered_paths.any? do |path|
638
+ # Deleted boot inputs have no mtime and are absent from a fresh snapshot.
639
+ # Preserve their restart obligation so a boot that already omitted them
640
+ # performs a full extraction. Do not promote nominal model paths to
641
+ # authoritative deletions, even when application reloading is disabled.
642
+ def vanished_restart_paths
643
+ vanished_registered_paths.select do |path|
644
+ @policy.classify(path.delete_prefix("#{@root}/")) == :restart
645
+ end
646
+ end
647
+
648
+ def vanished_registered_paths
649
+ root_prefix = "#{@root}/"
650
+ persisted_registered_paths.select do |path|
561
651
  path.start_with?(root_prefix) && !File.exist?(path)
562
652
  end
563
653
  end
@@ -759,7 +849,7 @@ module Woods
759
849
  # start at all (inotify exhaustion being the usual reason). A daemon that
760
850
  # costs some CPU beats one that silently never fires.
761
851
  def start_watching(&on_change)
762
- @watcher.start(&on_change)
852
+ start_backend(&on_change)
763
853
  rescue WatcherError => e
764
854
  raise if @polling_fallback
765
855
 
@@ -769,8 +859,20 @@ module Woods
769
859
  # the output-directory feedback loop {#ignored_directories} exists to
770
860
  # break, on precisely the path a large tree reaches.
771
861
  @watcher = Watcher.build(
772
- root: @root, ignored: ignored_directories, logger: @logger, force_polling: true
862
+ root: @root, ignored: ignored_directories, logger: @logger, force_polling: true,
863
+ poll_interval: @poll_interval
773
864
  )
865
+ start_backend(&on_change)
866
+ end
867
+
868
+ def start_backend(&on_change)
869
+ if @watcher.respond_to?(:ready_callback=)
870
+ @watcher.ready_callback = -> { @watcher_ready << :ready }
871
+ else
872
+ # Existing injected watchers own their startup contract. Built-in
873
+ # backends signal only after the actual baseline/registration.
874
+ @watcher_ready << :ready
875
+ end
774
876
  @watcher.start(&on_change)
775
877
  end
776
878
 
@@ -800,6 +902,7 @@ module Woods
800
902
  end
801
903
 
802
904
  def require_restart(change_set)
905
+ carry_forward(change_set)
803
906
  triggers = @policy.paths_requiring(change_set.relative_paths, :restart)
804
907
  reason = "restart required: #{triggers.first(5).join(', ')}"
805
908
  @logger.warn("[Woods] watch: #{reason}")
@@ -864,13 +967,7 @@ module Woods
864
967
  end
865
968
 
866
969
  def run_extraction(change_set, started)
867
- # Count only paths that imply extraction work. Sixty edited markdown
868
- # files plus one model is a one-model change, and reading it as a storm
869
- # would trade a millisecond cycle for a full extraction.
870
- actionable = actionable_count(change_set)
871
- full = actionable > @full_extraction_threshold
872
- log_storm(actionable) if full
873
-
970
+ full = full_extraction?(change_set)
874
971
  before = @generation.current.number
875
972
  extractor = @extractor_factory.call
876
973
  touched = if full
@@ -883,7 +980,7 @@ module Woods
883
980
  action = full ? :full : :incremental
884
981
  return unpublished(action, change_set, started) if wrote_without_publishing?(touched, before)
885
982
 
886
- publish(action, change_set, touched, started)
983
+ finish_extraction(action, change_set, touched, started)
887
984
  rescue ScriptError, StandardError => e
888
985
  # Extraction failed, so nothing landed — the generation stays where it
889
986
  # was and the index keeps serving its last good state.
@@ -896,6 +993,28 @@ module Woods
896
993
  duration_ms: elapsed_ms(started))
897
994
  end
898
995
 
996
+ def full_extraction?(change_set)
997
+ if @startup_full
998
+ @logger.info('[Woods] watch: environment boot covers startup changes — full extraction')
999
+ return true
1000
+ end
1001
+
1002
+ # Ignorable paths do not make a storm out of a single model edit.
1003
+ actionable = actionable_count(change_set)
1004
+ full = actionable > @full_extraction_threshold
1005
+ log_storm(actionable) if full
1006
+ full
1007
+ end
1008
+
1009
+ def finish_extraction(action, change_set, touched, started)
1010
+ result = publish(action, change_set, touched, started)
1011
+ if action == :full
1012
+ @startup_full = false
1013
+ @pending_mutex.synchronize { @startup_paths.clear }
1014
+ end
1015
+ result
1016
+ end
1017
+
899
1018
  def actionable_count(change_set)
900
1019
  change_set.relative_paths.count { |path| @policy.classify(path) != :ignore }
901
1020
  end
@@ -992,8 +1111,10 @@ module Woods
992
1111
  return false if ENV['WOODS_IGNORE_WATCH'] == '1'
993
1112
  return false unless @status.alive?
994
1113
 
995
- other = @status.read['pid']
996
- return false if other.nil? || other.to_i == Process.pid
1114
+ record = @status.read
1115
+ other = record['pid']
1116
+ return false if other.nil?
1117
+ return false if same_claim_host?(record['host']) && other.to_i == Process.pid
997
1118
 
998
1119
  @logger.warn(
999
1120
  "[Woods] a watch daemon (pid #{other}) is already maintaining #{@output_dir} — standing down. " \
@@ -1225,7 +1346,8 @@ module Woods
1225
1346
 
1226
1347
  def build_watcher
1227
1348
  @watcher = Watcher.build(
1228
- root: @root, ignored: ignored_directories, logger: @logger, force_polling: @force_polling
1349
+ root: @root, ignored: ignored_directories, logger: @logger, force_polling: @force_polling,
1350
+ poll_interval: @poll_interval
1229
1351
  )
1230
1352
  end
1231
1353
 
@@ -15,6 +15,9 @@ module Woods
15
15
  # silent while files change under it. Hosts in that position should force
16
16
  # polling rather than trust this backend.
17
17
  class ListenWatcher
18
+ # Optional daemon handshake; called once detection is established.
19
+ attr_writer :ready_callback
20
+
18
21
  # @param root [String, Pathname] directory to watch
19
22
  # @param ignored [Array<String>] directory names/prefixes to skip
20
23
  # @param listen_class [Class] injectable for specs
@@ -50,6 +53,7 @@ module Woods
50
53
  on_change.call(batch) if batch.any?
51
54
  end
52
55
  @listener.start
56
+ @ready_callback&.call
53
57
  rescue StandardError => e
54
58
  # inotify watch exhaustion (ENOSPC) lands here, and it is the most
55
59
  # likely failure on a large tree. {Watcher.build}'s caller falls back