woods 1.6.1 → 2.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1897 -0
  3. data/CONTRIBUTING.md +202 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +620 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +415 -0
  19. data/docs/INTERNALS.md +415 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +197 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +40 -1
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2503 -192
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +63 -21
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +138 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +408 -34
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +223 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer.rb +21 -5
  218. data/lib/woods/session_tracer/file_store.rb +138 -19
  219. data/lib/woods/session_tracer/middleware.rb +1 -2
  220. data/lib/woods/session_tracer/redis_store.rb +122 -12
  221. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  222. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  223. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  224. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  225. data/lib/woods/session_tracer/store.rb +14 -1
  226. data/lib/woods/storage/metadata_store.rb +230 -26
  227. data/lib/woods/storage/pgvector.rb +180 -22
  228. data/lib/woods/storage/qdrant.rb +367 -41
  229. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  230. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  231. data/lib/woods/storage/snapshotter.rb +23 -5
  232. data/lib/woods/storage/vector_store.rb +49 -8
  233. data/lib/woods/storage_identity.rb +28 -0
  234. data/lib/woods/tasks.rb +53 -2
  235. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  236. data/lib/woods/temporal/snapshot_store.rb +139 -42
  237. data/lib/woods/unblocked/client.rb +119 -17
  238. data/lib/woods/unblocked/document_builder.rb +34 -2
  239. data/lib/woods/unblocked/exporter.rb +63 -27
  240. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  241. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  242. data/lib/woods/update_check.rb +24 -1
  243. data/lib/woods/util/uuid5.rb +124 -0
  244. data/lib/woods/version.rb +1 -1
  245. data/lib/woods/watch/daemon.rb +1345 -0
  246. data/lib/woods/watch/listen_watcher.rb +81 -0
  247. data/lib/woods/watch/polling_watcher.rb +137 -0
  248. data/lib/woods/watch/status.rb +169 -0
  249. data/lib/woods/watch/tree_scan.rb +163 -0
  250. data/lib/woods/watch/watcher.rb +100 -0
  251. data/lib/woods.rb +53 -9
  252. data/plugin/.claude-plugin/plugin.json +18 -0
  253. data/plugin/hooks/hooks.json +29 -0
  254. data/plugin/hooks/woods-post-edit.sh +226 -0
  255. data/plugin/hooks/woods-session-start.sh +77 -0
  256. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  257. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  258. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  259. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  260. data/plugin/skills/woods-setup/SKILL.md +99 -0
  261. metadata +134 -23
  262. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  263. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  264. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  265. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  266. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  267. data/lib/woods/console/bridge.rb +0 -210
  268. data/lib/woods/formatting/claude_adapter.rb +0 -98
  269. data/lib/woods/formatting/generic_adapter.rb +0 -56
  270. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  271. data/lib/woods/notion/mapper.rb +0 -40
  272. data/lib/woods/observability/health_check.rb +0 -79
  273. data/lib/woods/observability/instrumentation.rb +0 -34
data/lib/woods/builder.rb CHANGED
@@ -8,6 +8,8 @@ require_relative 'storage/metadata_store'
8
8
  require_relative 'storage/graph_store'
9
9
  require_relative 'embedding/provider'
10
10
  require_relative 'embedding/openai'
11
+ require_relative 'embedding/fake'
12
+ require_relative 'resilience/retryable_provider'
11
13
  require_relative 'embedding/text_preparer'
12
14
  require_relative 'embedding/token_counter'
13
15
  require_relative 'token_utils'
@@ -69,7 +71,8 @@ module Woods
69
71
 
70
72
  # Build a {Configuration} populated with the named preset's adapter types.
71
73
  #
72
- # @param name [Symbol] Preset name — one of :local, :postgresql, or :production
74
+ # @param name [Symbol] Preset name — one of :local, :shared_filesystem,
75
+ # :postgresql, or :production
73
76
  # @return [Configuration] A new Configuration with preset values applied
74
77
  # @raise [ArgumentError] if the preset name is not recognized
75
78
  def self.preset_config(name)
@@ -107,13 +110,13 @@ module Woods
107
110
  # on disk and passes the populated store here.
108
111
  # @return [Retriever, Cache::CachedRetriever] A fully wired retriever
109
112
  def build_retriever(vector_store: nil, metadata_store: nil, graph_store: nil)
110
- provider = build_embedding_provider
113
+ provider = build_resilient_embedding_provider
111
114
  cache = build_cache_store
112
115
 
113
116
  provider = wrap_with_embedding_cache(provider, cache) if cache
114
117
 
115
118
  retriever = Retriever.new(
116
- vector_store: vector_store || build_vector_store,
119
+ vector_store: vector_store || build_vector_store(dimensions: vector_dimensions(provider)),
117
120
  metadata_store: metadata_store || build_metadata_store,
118
121
  graph_store: graph_store || build_graph_store,
119
122
  embedding_provider: provider
@@ -124,19 +127,56 @@ module Woods
124
127
 
125
128
  # Instantiate the vector store adapter specified by the configuration.
126
129
  #
130
+ # The :pgvector branch also ensures the adapter's own schema exists —
131
+ # see {#build_pgvector_store}.
132
+ #
127
133
  # @return [Storage::VectorStore::Interface] Vector store adapter instance
128
134
  # @raise [ArgumentError] if the configured type is not recognized
129
- def build_vector_store
135
+ # @raise [Woods::Error] if the pgvector schema cannot be created
136
+ def build_vector_store(dimensions: nil)
130
137
  case @config.vector_store
131
138
  when :in_memory then Storage::VectorStore::InMemory.new
132
- when :pgvector then Storage::VectorStore::Pgvector.new(**(@config.vector_store_options || {}))
133
- when :qdrant then Storage::VectorStore::Qdrant.new(**(@config.vector_store_options || {}))
139
+ when :pgvector then build_pgvector_store(dimensions)
140
+ when :qdrant then build_qdrant_store(dimensions)
134
141
  else raise ArgumentError, "Unknown vector_store: #{@config.vector_store}"
135
142
  end
136
143
  end
137
144
 
145
+ # The provider's advertised dimension count, when a durable store needs
146
+ # one to size its column or collection.
147
+ #
148
+ # Probed exactly like {#safe_max_input_tokens}: {#provider_object?}'s
149
+ # contract is that only +#embed+/+#embed_batch+ are required of an
150
+ # injected provider and the rest of the interface is checked at each call
151
+ # site, and a provider that merely includes {Embedding::Provider::Interface}
152
+ # answers +respond_to?+ with a +NotImplementedError+ stub (B-108). Either
153
+ # way the answer is "no dimensions", which lets
154
+ # {#resolve_pgvector_dimensions}/{#resolve_qdrant_dimensions} fall back to
155
+ # the explicit +vector_store_options[:dimensions]+ they document (STO-6).
156
+ #
157
+ # @param provider [Embedding::Provider::Interface]
158
+ # @return [Integer, nil]
159
+ def vector_dimensions(provider)
160
+ return nil unless %i[pgvector qdrant].include?(@config.vector_store)
161
+ return nil unless provider.respond_to?(:dimensions)
162
+
163
+ provider.dimensions
164
+ rescue NotImplementedError
165
+ nil
166
+ end
167
+ private :vector_dimensions
168
+
138
169
  # Instantiate the embedding provider specified by the configuration.
139
170
  #
171
+ # `embedding_provider` accepts three shapes (#178):
172
+ # - +:openai+ / +:ollama+ — the network-backed adapters.
173
+ # - +:fake+ — {Embedding::Provider::Fake}: deterministic, offline, for
174
+ # CI/smoke runs. See {#build_fake_provider} for dimension resolution.
175
+ # - an already-constructed provider *object* — anything responding to
176
+ # +#embed+ and +#embed_batch+ is returned as-is, so hosts can plug in
177
+ # their own implementation without patching the Builder. It flows
178
+ # through {#build_resilient_embedding_provider} like the built-ins.
179
+ #
140
180
  # Strips `embedding_options` keys that belong to the ResolvedConfig layer
141
181
  # (like `:dimension`) before splatting into the provider's constructor —
142
182
  # those keys are useful for the Snapshotter's schema header but
@@ -145,27 +185,148 @@ module Woods
145
185
  # @return [Embedding::Provider::Interface] Embedding provider instance
146
186
  # @raise [ArgumentError] if the configured type is not recognized
147
187
  def build_embedding_provider
148
- opts = provider_kwargs
149
- case @config.embedding_provider
188
+ configured = @config.embedding_provider
189
+ return configured if provider_object?(configured)
190
+
191
+ opts = provider_kwargs(configured)
192
+ case configured
150
193
  when :openai then Embedding::Provider::OpenAI.new(**opts)
151
194
  when :ollama then Embedding::Provider::Ollama.new(**opts)
152
- else raise ArgumentError, "Unknown embedding_provider: #{@config.embedding_provider}"
195
+ when :fake then build_fake_provider(opts)
196
+ else
197
+ raise ArgumentError,
198
+ "Unknown embedding_provider: #{configured}. Valid: :openai, :ollama, :fake, " \
199
+ 'or a provider object responding to #embed and #embed_batch'
153
200
  end
154
201
  end
155
202
 
156
- # Kwargs accepted by embedding provider constructors — everything in
157
- # `embedding_options` except metadata fields that live there for
158
- # ResolvedConfig bookkeeping.
159
- SNAPSHOT_ONLY_KEYS = %i[dimension].freeze
160
- private_constant :SNAPSHOT_ONLY_KEYS
203
+ # Wrap an embedding provider in the resilience stack: retry with
204
+ # full-jitter exponential backoff (Retry-After aware) plus a dedicated
205
+ # {Resilience::CircuitBreaker}.
206
+ #
207
+ # This is the provider every pipeline entry point must hand to the
208
+ # Indexer or Retriever — with it, a transient 429/5xx burst degrades a
209
+ # run instead of aborting it (#188 / B-076). {#build_embedding_provider}
210
+ # deliberately keeps returning the *raw* provider: the MCP boot path
211
+ # ({MCP::ProviderProbe}) dispatches on the provider's concrete class and
212
+ # reads its internals, so the wrap happens here, one layer up.
213
+ #
214
+ # Each call constructs a fresh breaker — breaker state is per-instance
215
+ # and must never be shared across unrelated components.
216
+ #
217
+ # The wrap is harmless for providers that never raise transient
218
+ # failures ({Embedding::Provider::Fake}, most injected provider
219
+ # objects): nothing retryable ever fires, so the wrapper is a
220
+ # transparent pass-through.
221
+ #
222
+ # @param provider [Embedding::Provider::Interface] raw provider to wrap;
223
+ # defaults to a freshly built one from the configuration
224
+ # @return [Resilience::RetryableProvider] the wrapped provider
225
+ def build_resilient_embedding_provider(provider = build_embedding_provider)
226
+ Resilience::RetryableProvider.new(
227
+ provider: provider,
228
+ circuit_breaker: Resilience::CircuitBreaker.new
229
+ )
230
+ end
231
+
232
+ PROVIDER_OPTION_KEYS = {
233
+ openai: %i[api_key model dimension dimensions],
234
+ ollama: %i[model host num_ctx read_timeout dimension dimensions],
235
+ fake: %i[model dims dimension dimensions]
236
+ }.freeze
237
+ private_constant :PROVIDER_OPTION_KEYS
161
238
 
162
- def provider_kwargs
239
+ def provider_kwargs(configured)
163
240
  opts = (@config.embedding_options || {}).transform_keys(&:to_sym)
164
- SNAPSHOT_ONLY_KEYS.each { |k| opts.delete(k) }
241
+ validate_provider_options!(configured, opts)
242
+ apply_embedding_model!(opts)
243
+ normalize_dimension_option!(configured, opts)
244
+ validate_required_provider_options!(configured, opts)
165
245
  opts
166
246
  end
167
247
  private :provider_kwargs
168
248
 
249
+ def validate_provider_options!(configured, opts)
250
+ allowed = PROVIDER_OPTION_KEYS[configured]
251
+ return unless allowed
252
+
253
+ unknown = opts.keys - allowed
254
+ return if unknown.empty?
255
+
256
+ provider_name = configured.to_s.capitalize
257
+ noun = unknown.one? ? 'option' : 'options'
258
+ raise ConfigurationError,
259
+ "Unknown #{provider_name} embedding #{noun}: #{unknown.join(', ')}. " \
260
+ "Valid options: #{allowed.join(', ')}"
261
+ end
262
+ private :validate_provider_options!
263
+
264
+ def validate_required_provider_options!(configured, opts)
265
+ return unless configured == :openai
266
+ return unless opts[:api_key].nil? || opts[:api_key].to_s.empty?
267
+
268
+ raise ConfigurationError,
269
+ 'OpenAI requires embedding_options[:api_key]. Set it explicitly, typically from OPENAI_API_KEY.'
270
+ end
271
+ private :validate_required_provider_options!
272
+
273
+ def apply_embedding_model!(opts)
274
+ return if opts.key?(:model)
275
+ return unless @config.respond_to?(:embedding_model_explicit?) && @config.embedding_model_explicit?
276
+
277
+ opts[:model] = @config.embedding_model
278
+ end
279
+ private :apply_embedding_model!
280
+
281
+ def normalize_dimension_option!(configured, opts)
282
+ legacy_dimension = opts.delete(:dimension)
283
+ dimensions = opts.delete(:dimensions)
284
+ if conflicting_dimensions?(legacy_dimension, dimensions)
285
+ raise ConfigurationError, 'embedding_options dimension and dimensions must match when both are provided'
286
+ end
287
+
288
+ dimension = dimensions || legacy_dimension
289
+ return unless dimension
290
+
291
+ opts[configured == :fake ? :dims : :dimensions] ||= dimension
292
+ end
293
+ private :normalize_dimension_option!
294
+
295
+ def conflicting_dimensions?(legacy_dimension, dimensions)
296
+ !legacy_dimension.nil? && !dimensions.nil? && legacy_dimension != dimensions
297
+ end
298
+ private :conflicting_dimensions?
299
+
300
+ # True when the configured `embedding_provider` is not a Symbol naming a
301
+ # built-in adapter but an already-constructed provider object (#178).
302
+ # Duck-typed on the two methods every pipeline consumer calls; the rest
303
+ # of {Embedding::Provider::Interface} (+#dimensions+, +#model_name+,
304
+ # +#max_input_tokens+) is probed with +respond_to?+ at each call site,
305
+ # so an object that omits them still works where they are optional.
306
+ #
307
+ # @param candidate [Object]
308
+ # @return [Boolean]
309
+ def provider_object?(candidate)
310
+ !candidate.is_a?(Symbol) && candidate.respond_to?(:embed) && candidate.respond_to?(:embed_batch)
311
+ end
312
+ private :provider_object?
313
+
314
+ # Build the deterministic fake provider (#178).
315
+ #
316
+ # Dimension resolution: `embedding_options[:dims]` maps directly onto
317
+ # the {Embedding::Provider::Fake} constructor; failing that, the
318
+ # ResolvedConfig-level `embedding_options[:dimension]` key — normally
319
+ # snapshot-only bookkeeping stripped by {#provider_kwargs} — is
320
+ # honoured, so hosts that declare their dimension there (and the MCP
321
+ # boot path, which restores exactly that key from woods.json) get
322
+ # vectors of the recorded dimension.
323
+ #
324
+ # @return [Embedding::Provider::Fake]
325
+ def build_fake_provider(opts)
326
+ Embedding::Provider::Fake.new(**opts)
327
+ end
328
+ private :build_fake_provider
329
+
169
330
  # Build a {Embedding::TextPreparer} calibrated to a given provider.
170
331
  #
171
332
  # OpenAI embedders use tiktoken (cl100k_base) — 4.0 chars/token is a
@@ -184,7 +345,7 @@ module Woods
184
345
  # @return [Embedding::TextPreparer]
185
346
  def build_text_preparer(provider)
186
347
  chars_per_token = chars_per_token_for(provider)
187
- budget = provider.respond_to?(:max_input_tokens) ? provider.max_input_tokens : nil
348
+ budget = safe_max_input_tokens(provider)
188
349
  max_tokens = budget || Embedding::TextPreparer::DEFAULT_MAX_TOKENS
189
350
 
190
351
  Embedding::TextPreparer.new(max_tokens: max_tokens, chars_per_token: chars_per_token)
@@ -215,7 +376,7 @@ module Woods
215
376
  # @param provider [Embedding::Provider::Interface]
216
377
  # @return [Chunking::SemanticChunker]
217
378
  def build_chunker(provider)
218
- budget = provider.respond_to?(:max_input_tokens) ? provider.max_input_tokens : nil
379
+ budget = safe_max_input_tokens(provider)
219
380
  max_chars = ((budget * chars_per_token_for(provider)).floor - CHUNKER_PREFIX_ALLOWANCE if budget)
220
381
 
221
382
  # Guard against a budget so small that the prefix allowance leaves
@@ -256,7 +417,7 @@ module Woods
256
417
  # @param provider [Embedding::Provider::Interface]
257
418
  # @return [Embedding::TokenCounter, nil]
258
419
  def token_counter_for(provider)
259
- return unless provider.is_a?(Embedding::Provider::Ollama)
420
+ return unless unwrap_provider(provider).is_a?(Embedding::Provider::Ollama)
260
421
 
261
422
  Embedding::TokenCounter.new
262
423
  end
@@ -268,16 +429,50 @@ module Woods
268
429
  # @param provider [Embedding::Provider::Interface]
269
430
  # @return [Float]
270
431
  def chars_per_token_for(provider)
271
- symbol = case provider
432
+ symbol = case unwrap_provider(provider)
272
433
  when Embedding::Provider::Ollama then :ollama
273
434
  else :openai
274
435
  end
275
436
  TokenUtils.chars_per_token_for(symbol)
276
437
  end
277
438
 
439
+ # Provider input-token budget, or nil when the provider has none.
440
+ # `respond_to?` alone is the wrong guard here: {Embedding::Provider::Interface}
441
+ # *defines* +max_input_tokens+ as a +NotImplementedError+ stub, so a
442
+ # provider that merely includes the interface without overriding it
443
+ # still answers +respond_to?+ with +true+ (B-108) and raises when
444
+ # called. A provider with no such method at all still needs the
445
+ # +respond_to?+ guard to avoid a bare +NoMethodError+.
446
+ #
447
+ # @param provider [Embedding::Provider::Interface]
448
+ # @return [Integer, nil]
449
+ def safe_max_input_tokens(provider)
450
+ return nil unless provider.respond_to?(:max_input_tokens)
451
+
452
+ provider.max_input_tokens
453
+ rescue NotImplementedError
454
+ nil
455
+ end
456
+
457
+ # Reach the concrete provider through the resilience wrapper.
458
+ # Tokenizer calibration dispatches on the provider's real class, so a
459
+ # {Resilience::RetryableProvider} handed to {#build_text_preparer} or
460
+ # {#build_chunker} must calibrate exactly like its inner provider —
461
+ # without this, a wrapped Ollama silently got OpenAI ratios.
462
+ #
463
+ # @param provider [Embedding::Provider::Interface]
464
+ # @return [Embedding::Provider::Interface] the innermost provider
465
+ def unwrap_provider(provider)
466
+ provider.is_a?(Resilience::RetryableProvider) ? provider.provider : provider
467
+ end
468
+
278
469
  # Diagnostic for the build_chunker budget guard.
470
+ #
471
+ # @param provider [Embedding::Provider::Interface]
472
+ # @param budget [Integer]
473
+ # @return [String]
279
474
  def chunker_budget_message(provider, budget)
280
- "embedding model '#{provider.respond_to?(:model) ? provider.model : provider.class}' " \
475
+ "embedding model '#{provider.respond_to?(:model_name) ? provider.model_name : provider.class}' " \
281
476
  "reports a max_input_tokens of #{budget}, which leaves no room for " \
282
477
  "the chunk prefix (#{CHUNKER_PREFIX_ALLOWANCE} chars). Configure a " \
283
478
  'model with a larger native context, or set num_ctx explicitly.'
@@ -292,11 +487,18 @@ module Woods
292
487
  def build_metadata_store
293
488
  case @config.metadata_store
294
489
  when :in_memory then Storage::MetadataStore::InMemory.new
295
- when :sqlite then Storage::MetadataStore::SQLite.new(**(@config.metadata_store_options || {}))
490
+ when :sqlite then Storage::MetadataStore::SQLite.new(**sqlite_metadata_options)
296
491
  else raise ArgumentError, "Unknown metadata_store: #{@config.metadata_store}"
297
492
  end
298
493
  end
299
494
 
495
+ def sqlite_metadata_options
496
+ opts = (@config.metadata_store_options || {}).transform_keys(&:to_sym)
497
+ opts[:database] ||= File.join(@config.output_dir.to_s, 'metadata.sqlite3')
498
+ opts
499
+ end
500
+ private :sqlite_metadata_options
501
+
300
502
  # Instantiate the graph store adapter specified by the configuration.
301
503
  #
302
504
  # @return [Storage::GraphStore::Interface] Graph store adapter instance
@@ -310,6 +512,92 @@ module Woods
310
512
 
311
513
  private
312
514
 
515
+ # Construct the pgvector adapter and ensure its schema exists.
516
+ #
517
+ # The adapter reads and writes its own `woods_vectors` table. The
518
+ # `woods:pgvector` generator can create it via a migration, but nothing
519
+ # guarantees that migration ran — so the builder calls the adapter's
520
+ # idempotent {Storage::VectorStore::Pgvector#ensure_schema!}
521
+ # (CREATE ... IF NOT EXISTS DDL) after construction. Without this, the
522
+ # first embed against a bare database fails with PG::UndefinedTable
523
+ # (#187 / B-075). Schema/connection failures are re-raised as
524
+ # {Woods::Error} with the original error preserved as the cause.
525
+ #
526
+ # @return [Storage::VectorStore::Pgvector]
527
+ # @raise [Woods::Error] when the schema cannot be created
528
+ def build_pgvector_store(provider_dimensions)
529
+ opts = (@config.vector_store_options || {}).transform_keys(&:to_sym)
530
+ validate_required_store_options!(:pgvector, opts, :connection)
531
+ opts[:dimensions] = resolve_pgvector_dimensions(provider_dimensions, opts[:dimensions])
532
+ store = Storage::VectorStore::Pgvector.new(**opts)
533
+ begin
534
+ store.ensure_schema!
535
+ verify_pgvector_dimensions!(store, opts[:dimensions])
536
+ rescue ConfigurationError
537
+ raise
538
+ rescue StandardError => e
539
+ raise Woods::Error,
540
+ "pgvector schema setup failed (#{e.class}: #{e.message}). " \
541
+ 'Verify vector_store_options[:connection] is a live PostgreSQL ' \
542
+ 'connection and that the pgvector extension is available ' \
543
+ '(`rails generate woods:pgvector && rails db:migrate` sets it up via migration).'
544
+ end
545
+ store
546
+ end
547
+
548
+ def verify_pgvector_dimensions!(store, expected)
549
+ actual = store.stored_dimensions
550
+ return if actual.nil? || actual == expected
551
+
552
+ raise ConfigurationError,
553
+ "Stored pgvector dimensions #{actual} do not match embedding provider dimensions #{expected}. " \
554
+ 'Use a compatible table or rebuild the index.'
555
+ end
556
+
557
+ def resolve_pgvector_dimensions(provider_dimensions, configured_dimensions)
558
+ if provider_dimensions && configured_dimensions && provider_dimensions != configured_dimensions
559
+ raise ConfigurationError,
560
+ "pgvector dimensions #{configured_dimensions} do not match embedding provider dimensions " \
561
+ "#{provider_dimensions}"
562
+ end
563
+
564
+ provider_dimensions || configured_dimensions || raise(
565
+ ConfigurationError,
566
+ 'pgvector requires vector_store_options[:dimensions] when built without an embedding provider'
567
+ )
568
+ end
569
+
570
+ def build_qdrant_store(provider_dimensions)
571
+ opts = (@config.vector_store_options || {}).transform_keys(&:to_sym)
572
+ validate_required_store_options!(:qdrant, opts, :url, :collection)
573
+ dimensions = resolve_qdrant_dimensions(provider_dimensions, opts[:dimensions])
574
+ opts[:dimensions] = dimensions
575
+ store = Storage::VectorStore::Qdrant.new(**opts)
576
+ store.ensure_collection!(dimensions: dimensions)
577
+ store
578
+ end
579
+
580
+ def validate_required_store_options!(adapter, opts, *keys)
581
+ missing = keys.select { |key| opts[key].nil? || opts[key].to_s.empty? }
582
+ return if missing.empty?
583
+
584
+ requirements = missing.map { |key| "vector_store_options[:#{key}]" }.join(' and ')
585
+ raise ConfigurationError, "#{adapter} requires #{requirements}"
586
+ end
587
+
588
+ def resolve_qdrant_dimensions(provider_dimensions, configured_dimensions)
589
+ if provider_dimensions && configured_dimensions && provider_dimensions != configured_dimensions
590
+ raise ConfigurationError,
591
+ "Qdrant dimensions #{configured_dimensions} do not match embedding provider dimensions " \
592
+ "#{provider_dimensions}"
593
+ end
594
+
595
+ provider_dimensions || configured_dimensions || raise(
596
+ ConfigurationError,
597
+ 'Qdrant requires vector_store_options[:dimensions] when built without an embedding provider'
598
+ )
599
+ end
600
+
313
601
  # Build a cache store from configuration, or nil if caching is disabled.
314
602
  #
315
603
  # @return [Cache::CacheStore, nil]
@@ -2,6 +2,11 @@
2
2
 
3
3
  require 'digest'
4
4
  require_relative 'cache_store'
5
+ # CachedEmbeddingProvider includes Embedding::Provider::Interface at load
6
+ # time, so the interface must be defined before this file's class bodies run
7
+ # — otherwise a narrow `require 'woods/cache/cache_middleware'` raises
8
+ # NameError (STO-5).
9
+ require_relative '../embedding/provider'
5
10
 
6
11
  module Woods
7
12
  module Cache
@@ -364,8 +369,8 @@ module Woods
364
369
  # persistent shared backend (Redis/SolidCache) returns the previous
365
370
  # model's vector after a model switch or upgrade — different dimensions
366
371
  # error mid-batch, same dimensions silently corrupt similarity scores
367
- # (which IndexValidator can't detect, since it checks provider-vs-store
368
- # dims, not cache contents).
372
+ # (which the provider-vs-store dimension check can't detect, since it
373
+ # compares declared widths, not cache contents).
369
374
  #
370
375
  # model_name (a plain attribute) is used rather than dimensions on
371
376
  # purpose: for every supported provider the model uniquely determines
@@ -25,7 +25,11 @@ module Woods
25
25
  # @param parts [Array<String>] Key components (will be SHA256-hashed if long)
26
26
  # @return [String] Namespaced key
27
27
  def self.cache_key(domain, *parts)
28
- raw = parts.join(':')
28
+ raw = if parts.one?
29
+ parts.first.to_s
30
+ else
31
+ parts.map { |part| "#{part.to_s.bytesize}:#{part}" }.join
32
+ end
29
33
  suffix = raw.length > 64 ? Digest::SHA256.hexdigest(raw) : raw
30
34
  "woods:cache:#{domain}:#{suffix}"
31
35
  end
@@ -148,6 +152,10 @@ module Woods
148
152
  # @param max_entries [Integer] Maximum cached entries before LRU eviction
149
153
  def initialize(max_entries: 500)
150
154
  super()
155
+ unless max_entries.is_a?(Integer) && max_entries.positive?
156
+ raise ArgumentError, 'max_entries must be a positive Integer'
157
+ end
158
+
151
159
  @max_entries = max_entries
152
160
  @entries = {}
153
161
  @access_order = []
@@ -56,10 +56,12 @@ module Woods
56
56
  effective_ttl = ttl || @default_ttl
57
57
 
58
58
  opts = effective_ttl ? { expires_in: effective_ttl } : {}
59
- @cache.write(key, serialized, **opts)
60
- rescue StandardError => e
61
- logger.warn("[Woods] SolidCacheStore#write failed for #{key}: #{e.message}")
62
- nil
59
+ begin
60
+ @cache.write(key, serialized, **opts)
61
+ rescue StandardError => e
62
+ logger.warn("[Woods] SolidCacheStore#write failed for #{key}: #{e.message}")
63
+ nil
64
+ end
63
65
  end
64
66
 
65
67
  # Delete a key from the cache.
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'pathname'
4
+
5
+ module Woods
6
+ # A normalized set of changed paths, shared by every entry point that has
7
+ # to answer "what changed?".
8
+ #
9
+ # Two callers feed it: the git-diff-driven `woods:incremental` rake task, and
10
+ # {Woods::Watch::Daemon}, which builds one per batch of FS events. Two code
11
+ # paths computing "what changed" independently is exactly how a watcher and a
12
+ # CI chain drift apart, so the normalization — absolutizing, de-duplicating,
13
+ # and splitting present-from-vanished — lives here rather than in either
14
+ # caller.
15
+ #
16
+ # Paths are held absolute (matching how {DependencyGraph} registers file
17
+ # paths) and exposed relative on demand (matching how {PathDispatcher}
18
+ # rules are written).
19
+ #
20
+ # @example
21
+ # cs = Woods::ChangeSet.new(paths: %w[app/models/user.rb app/models/gone.rb], root: Rails.root)
22
+ # cs.existing_paths # => ["/app/app/models/user.rb"]
23
+ # cs.missing_paths # => ["/app/app/models/gone.rb"]
24
+ #
25
+ class ChangeSet
26
+ # @return [Pathname] the application root paths are resolved against
27
+ attr_reader :root
28
+
29
+ # @param paths [Array<String>] changed paths, absolute or root-relative
30
+ # @param root [String, Pathname] application root (usually Rails.root)
31
+ def initialize(paths:, root:)
32
+ @root = Pathname.new(root.to_s)
33
+ @absolute_paths = Array(paths).filter_map { |p| absolutize(p) }.uniq.freeze
34
+ end
35
+
36
+ # @return [Array<String>] every changed path, absolute
37
+ attr_reader :absolute_paths
38
+
39
+ # @return [Array<String>] every changed path, relative to {#root}
40
+ def relative_paths
41
+ @relative_paths ||= @absolute_paths.map { |p| relativize(p) }.freeze
42
+ end
43
+
44
+ # Changed paths that still exist on disk (created or modified).
45
+ #
46
+ # @return [Array<String>] absolute paths
47
+ def existing_paths
48
+ @existing_paths ||= @absolute_paths.select { |p| File.exist?(p) }.freeze
49
+ end
50
+
51
+ # Changed paths that no longer exist (deleted, or the old side of a
52
+ # rename — git's `--no-renames` semantics, which is what the deletion
53
+ # sweep expects).
54
+ #
55
+ # @return [Array<String>] absolute paths
56
+ def missing_paths
57
+ @missing_paths ||= (@absolute_paths - existing_paths).freeze
58
+ end
59
+
60
+ # @return [Boolean] true when nothing changed
61
+ def empty?
62
+ @absolute_paths.empty?
63
+ end
64
+
65
+ # @return [Integer] number of changed paths
66
+ def size
67
+ @absolute_paths.size
68
+ end
69
+
70
+ # Convert an absolute path back to a root-relative one.
71
+ #
72
+ # @param path [String] absolute path
73
+ # @return [String] root-relative path, or the input if outside {#root}
74
+ def relativize(path)
75
+ prefix = "#{@root}/"
76
+ path.start_with?(prefix) ? path.delete_prefix(prefix) : path
77
+ end
78
+
79
+ private
80
+
81
+ def absolutize(path)
82
+ str = path.to_s.strip
83
+ return nil if str.empty?
84
+
85
+ Pathname.new(str).absolute? ? str : @root.join(str).to_s
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Woods
4
+ module Checks
5
+ # Turns "which two generations should `woods:check:moved_messages`
6
+ # compare" into a pair of numbers, given the generations a
7
+ # {Woods::PublishedIndex} currently has published (#280 / M14).
8
+ #
9
+ # A pure function over its arguments: it never touches the filesystem or
10
+ # opens an index. `to` defaults to the newest published generation,
11
+ # `from` defaults to the newest one strictly older than `to`. Either side
12
+ # given explicitly is used as-is (converted with `Integer()`, so rake's
13
+ # string arguments work); explicit values are not checked against
14
+ # +available+ here, {Woods::PublishedIndex.new} already raises a clear
15
+ # `ArgumentError` for a generation that is not published.
16
+ module GenerationResolution
17
+ # @param available [Array<Integer>] published generation numbers, ascending
18
+ # @param from [String, Integer, nil] explicit older generation
19
+ # @param to [String, Integer, nil] explicit newer generation
20
+ # @return [Array(Integer, Integer), nil] `[from, to]`, or nil when there
21
+ # are not two generations to compare (fewer than two retained, and
22
+ # neither side was given explicitly)
23
+ def self.call(available, from: nil, to: nil)
24
+ to_number = to ? Integer(to) : available.last
25
+ return nil if to_number.nil?
26
+
27
+ from_number = from ? Integer(from) : available.select { |number| number < to_number }.max
28
+ return nil if from_number.nil?
29
+
30
+ [from_number, to_number]
31
+ end
32
+ end
33
+ end
34
+ end