woods 1.6.1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2035 -0
  3. data/CONTRIBUTING.md +253 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +655 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +455 -0
  19. data/docs/INTERNALS.md +418 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +213 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +171 -2
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2783 -238
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +147 -26
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +175 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +564 -87
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +236 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
  218. data/lib/woods/ruby_analyzer.rb +21 -5
  219. data/lib/woods/session_tracer/file_store.rb +138 -19
  220. data/lib/woods/session_tracer/middleware.rb +1 -2
  221. data/lib/woods/session_tracer/redis_store.rb +122 -12
  222. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  223. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  224. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  225. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  226. data/lib/woods/session_tracer/store.rb +14 -1
  227. data/lib/woods/storage/metadata_store.rb +230 -26
  228. data/lib/woods/storage/pgvector.rb +180 -22
  229. data/lib/woods/storage/qdrant.rb +367 -41
  230. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  231. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  232. data/lib/woods/storage/snapshotter.rb +23 -5
  233. data/lib/woods/storage/vector_store.rb +49 -8
  234. data/lib/woods/storage_identity.rb +28 -0
  235. data/lib/woods/tasks.rb +53 -2
  236. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  237. data/lib/woods/temporal/snapshot_store.rb +139 -42
  238. data/lib/woods/unblocked/client.rb +119 -17
  239. data/lib/woods/unblocked/document_builder.rb +34 -2
  240. data/lib/woods/unblocked/exporter.rb +63 -27
  241. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  242. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  243. data/lib/woods/update_check.rb +24 -1
  244. data/lib/woods/util/uuid5.rb +124 -0
  245. data/lib/woods/version.rb +1 -1
  246. data/lib/woods/watch/daemon.rb +1345 -0
  247. data/lib/woods/watch/listen_watcher.rb +81 -0
  248. data/lib/woods/watch/polling_watcher.rb +137 -0
  249. data/lib/woods/watch/status.rb +169 -0
  250. data/lib/woods/watch/tree_scan.rb +163 -0
  251. data/lib/woods/watch/watcher.rb +100 -0
  252. data/lib/woods.rb +138 -9
  253. data/plugin/.claude-plugin/plugin.json +18 -0
  254. data/plugin/hooks/hooks.json +29 -0
  255. data/plugin/hooks/woods-post-edit.sh +226 -0
  256. data/plugin/hooks/woods-session-start.sh +77 -0
  257. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  258. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  259. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  260. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  261. data/plugin/skills/woods-setup/SKILL.md +99 -0
  262. metadata +134 -23
  263. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  264. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  265. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  266. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  267. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  268. data/lib/woods/console/bridge.rb +0 -210
  269. data/lib/woods/formatting/claude_adapter.rb +0 -98
  270. data/lib/woods/formatting/generic_adapter.rb +0 -56
  271. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  272. data/lib/woods/notion/mapper.rb +0 -40
  273. data/lib/woods/observability/health_check.rb +0 -79
  274. data/lib/woods/observability/instrumentation.rb +0 -34
@@ -28,7 +28,6 @@ module Woods
28
28
  # @param columns [Array<String>, nil] Columns to include
29
29
  # @return [Hash] Bridge request
30
30
  def console_sample(model:, scope: nil, limit: 5, columns: nil)
31
- limit = [limit, 25].min
32
31
  { tool: 'sample', params: { model: model, scope: scope, limit: limit, columns: columns }.compact }
33
32
  end
34
33
 
@@ -52,7 +51,6 @@ module Woods
52
51
  # @param distinct [Boolean] Return unique values only
53
52
  # @return [Hash] Bridge request
54
53
  def console_pluck(model:, columns:, scope: nil, limit: 100, distinct: false)
55
- limit = [limit, 1000].min
56
54
  { tool: 'pluck', params: { model: model, columns: columns, scope: scope,
57
55
  limit: limit, distinct: distinct }.compact }
58
56
  end
@@ -98,13 +96,11 @@ module Woods
98
96
  # @param scope [Hash, nil] Filter conditions
99
97
  # @param columns [Array<String>, nil] Columns to include
100
98
  # @return [Hash] Bridge request
101
- # rubocop:disable Metrics/ParameterLists
99
+ # rubocop:disable-next Metrics/ParameterLists
102
100
  def console_recent(model:, order_by: 'created_at', direction: 'desc', limit: 10, scope: nil, columns: nil)
103
- limit = [limit, 50].min
104
101
  { tool: 'recent', params: { model: model, order_by: order_by, direction: direction,
105
102
  limit: limit, scope: scope, columns: columns }.compact }
106
103
  end
107
- # rubocop:enable Metrics/ParameterLists
108
104
 
109
105
  # System health check.
110
106
  #
@@ -37,16 +37,27 @@ module Woods
37
37
  { tool: 'eval', params: { code: code, timeout: timeout } }
38
38
  end
39
39
 
40
- # Read-only SQL execution with validation.
40
+ # Read-only SQL execution.
41
+ #
42
+ # SQL validation belongs to whoever knows the host's dialect. The
43
+ # embedded executor re-validates with `SqlValidator.new(dialect:
44
+ # sql_dialect)` from the live adapter and raises
45
+ # {Woods::Console::SqlValidationError}, so the registered handler
46
+ # passes no validator: a handler-stage `SqlValidator.new` is the
47
+ # conservative postgres+mysql union, and on a MySQL host its
48
+ # PostgreSQL view of `\'` rejected dialect-valid statements before the
49
+ # executor could accept them (CON-2). `validator:` stays available for
50
+ # callers that own their own gate — the bridge path, and specs that
51
+ # exercise the validator in isolation.
41
52
  #
42
53
  # @param sql [String] SQL query (must be SELECT or WITH...SELECT)
43
- # @param validator [SqlValidator] SQL validator instance
54
+ # @param validator [SqlValidator, nil] Optional pre-dispatch validator.
55
+ # Leave nil to let the executor validate with the adapter's dialect.
44
56
  # @param limit [Integer, nil] Optional row limit (max 10000)
45
57
  # @return [Hash] Bridge request
46
- # @raise [SqlValidationError] if SQL is not read-only
47
- def console_sql(sql:, validator:, limit: nil)
48
- validator.validate!(sql)
49
- limit = [limit, MAX_SQL_LIMIT].min if limit
58
+ # @raise [SqlValidationError] if a validator was supplied and refuses
59
+ def console_sql(sql:, validator: nil, limit: nil)
60
+ validator&.validate!(sql)
50
61
  { tool: 'sql', params: { sql: sql, limit: limit }.compact }
51
62
  end
52
63
 
@@ -61,9 +72,8 @@ module Woods
61
72
  # @param scope [Hash, nil] Filter conditions
62
73
  # @param limit [Integer, nil] Row limit (max 10000)
63
74
  # @return [Hash] Bridge request
64
- # rubocop:disable Metrics/ParameterLists
75
+ # rubocop:disable-next Metrics/ParameterLists
65
76
  def console_query(model:, select:, joins: nil, group_by: nil, having: nil, order: nil, scope: nil, limit: nil)
66
- limit = [limit, MAX_QUERY_LIMIT].min if limit
67
77
  {
68
78
  tool: 'query',
69
79
  params: {
@@ -78,7 +88,6 @@ module Woods
78
88
  }.compact
79
89
  }
80
90
  end
81
- # rubocop:enable Metrics/ParameterLists
82
91
  end
83
92
  end
84
93
  end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'pipeline_lock'
4
+
5
+ module Woods
6
+ module Coordination
7
+ # Keeps a held {PipelineLock} fresh for the duration of a block.
8
+ #
9
+ # {PipelineLock} staleness is mtime-only, so a holder that goes quiet for
10
+ # longer than `stale_timeout` has its lock retired by the next writer —
11
+ # producing exactly the two-writer clobber the lock exists to prevent,
12
+ # silently, with the generation bumped afterwards so the clobbered index
13
+ # reads as fresh.
14
+ #
15
+ # The daemon refreshes its lock at every cycle boundary. The rake writers
16
+ # (`woods:extract`, `woods:incremental`, `woods:refresh`) have no such
17
+ # boundary — the work is one opaque block — so they need a thread.
18
+ #
19
+ # Extracted from `lib/tasks/woods.rake` rather than left as a private
20
+ # method on `main`: a helper in a task file cannot be driven by a spec, and
21
+ # the properties worth guarding here are its own (the block's value
22
+ # survives, its exception survives, the thread stops, the pacing follows
23
+ # the lock) rather than {PipelineLock}'s.
24
+ class LockHeartbeat
25
+ # Fraction of the stale window to wait between refreshes. A third means
26
+ # two consecutive misses still leave the lock fresh — the same ratio
27
+ # {Woods::Watch::Status} uses for its own heartbeat.
28
+ INTERVAL_RATIO = 3.0
29
+
30
+ # How often the loop wakes to re-check the stop flag. Independent of the
31
+ # refresh interval so a short run's end is noticed promptly rather than a
32
+ # third of the window later.
33
+ TICK = 1.0
34
+
35
+ # Run a block with the lock kept fresh underneath it.
36
+ #
37
+ # @param lock [PipelineLock] a lock this process currently holds
38
+ # @param tick [Numeric] seconds between wakeups (injectable for specs)
39
+ # @return [Object] whatever the block returns
40
+ def self.run(lock, tick: TICK, &block)
41
+ new(lock, tick: tick).run(&block)
42
+ end
43
+
44
+ def initialize(lock, tick: TICK)
45
+ @lock = lock
46
+ @tick = tick
47
+ @interval = lock.stale_timeout / INTERVAL_RATIO
48
+ @stop = false
49
+ end
50
+
51
+ # @return [Object] the block's return value
52
+ def run
53
+ thread = start_thread
54
+ yield
55
+ ensure
56
+ @stop = true
57
+ thread&.join(@tick + 1)
58
+ end
59
+
60
+ private
61
+
62
+ def start_thread
63
+ # A raising heartbeat must never become the caller's error. `join`
64
+ # re-raises a thread's exception, and this `join` sits in an `ensure` —
65
+ # so without this rescue a successful extraction would fail with the
66
+ # heartbeat's error, and a failing one would have its real error
67
+ # replaced by it, precisely when the real message matters most.
68
+ thread = Thread.new do
69
+ beat
70
+ rescue StandardError => e
71
+ # Swallowed so it cannot become the caller's error — but not silently.
72
+ # A dead heartbeat leaves the run unprotected for the rest of its
73
+ # life, which is the same silent-failure class this class exists to
74
+ # prevent.
75
+ warn "[Woods] lock heartbeat stopped: #{e.class}: #{e.message}"
76
+ end
77
+ thread.report_on_exception = false
78
+ thread
79
+ end
80
+
81
+ def beat
82
+ # Monotonic, not a count of wakeups. The acquire loop in woods.rake uses
83
+ # CLOCK_MONOTONIC for the same reason: across a laptop suspend the wall
84
+ # clock advances while a wakeup counter does not, so a counted heartbeat
85
+ # would still believe it had time left while the lock aged past the
86
+ # window. Suspend is what actually produces multi-minute quiet periods
87
+ # on a dev machine.
88
+ last = now
89
+ until @stop
90
+ sleep(@tick)
91
+ next if @stop || now - last < @interval
92
+
93
+ @lock.touch
94
+ last = now
95
+ end
96
+ end
97
+
98
+ def now
99
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
100
+ end
101
+ end
102
+ end
103
+ end
@@ -4,6 +4,11 @@ require 'fileutils'
4
4
  require 'json'
5
5
  require 'securerandom'
6
6
 
7
+ # Woods::Error is defined in the entry point; required here so this file can
8
+ # be loaded directly (the watch daemon reaches for it without loading the
9
+ # whole gem first).
10
+ require 'woods'
11
+
7
12
  module Woods
8
13
  module Coordination
9
14
  class LockError < Woods::Error; end
@@ -22,9 +27,28 @@ module Woods
22
27
  class PipelineLock
23
28
  DEFAULT_STALE_TIMEOUT = 3600 # 1 hour
24
29
 
30
+ # The transaction-guard filename for a lock of +name+, exposed so
31
+ # cleanup code that empties a lock directory (woods:clean) can skip the
32
+ # guard during its sweep — deleting a flock'd guard out from under a
33
+ # contender's critical section would split the flock across two inodes —
34
+ # and remove it only after the lock is released.
35
+ #
36
+ # @param name [String]
37
+ # @return [String]
38
+ def self.guard_filename(name)
39
+ ".#{name}.lock.guard"
40
+ end
41
+
25
42
  # @param lock_dir [String] Directory for lock files
26
43
  # @param name [String] Lock name (used as filename prefix)
27
44
  # @param stale_timeout [Integer] Seconds after which a lock is considered stale
45
+ # The window after which an untouched lock is considered abandoned.
46
+ # Exposed so a heartbeat paces itself from the lock it is refreshing
47
+ # rather than from a constant that only happens to match.
48
+ #
49
+ # @return [Numeric]
50
+ attr_reader :stale_timeout
51
+
28
52
  def initialize(lock_dir:, name:, stale_timeout: DEFAULT_STALE_TIMEOUT)
29
53
  @lock_dir = lock_dir
30
54
  @name = name
@@ -37,27 +61,47 @@ module Woods
37
61
  #
38
62
  # @return [Boolean] true if lock acquired, false if already held
39
63
  def acquire
40
- FileUtils.mkdir_p(@lock_dir)
64
+ with_path_guard do
65
+ if File.exist?(@lock_path)
66
+ return false unless stale?
67
+ # Retire the stale lock atomically. A bare rm_f + create here is
68
+ # a TOCTOU race: two processes passing the stale check together
69
+ # could each delete-and-create, the second deleting the first's
70
+ # FRESH lock — both would then "hold" it.
71
+ return false unless retire_stale_unlocked == :cleared
72
+ end
41
73
 
42
- if File.exist?(@lock_path)
43
- return false unless stale?
44
- # Retire the stale lock atomically. A bare rm_f + create here is
45
- # a TOCTOU race: two processes passing the stale check together
46
- # could each delete-and-create, the second deleting the first's
47
- # FRESH lock — both would then "hold" it.
48
- return false unless retire_stale_lock
74
+ # Keep O_EXCL even though cooperating implementations hold the guard:
75
+ # it still fails closed if an external writer creates the path.
76
+ begin
77
+ File.open(@lock_path, File::WRONLY | File::CREAT | File::EXCL) do |file|
78
+ file.write(lock_content)
79
+ end
80
+ rescue Errno::EEXIST
81
+ raise
82
+ rescue ScriptError, StandardError
83
+ discard_partial_lock
84
+ raise
85
+ end
86
+ @held = true
87
+ true
49
88
  end
50
-
51
- # Atomic lock creation: File::EXCL ensures this fails if file already exists
52
- File.open(@lock_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
53
- f.write(lock_content)
54
- end
55
- @held = true
56
- true
57
89
  rescue Errno::EEXIST
58
90
  false
59
91
  end
60
92
 
93
+ # Retire the lock only when the file captured atomically from the public
94
+ # lock path is genuinely stale.
95
+ #
96
+ # @return [Symbol] `:cleared`, `:not_stale`, or `:missing`
97
+ def retire_stale
98
+ return :missing unless File.directory?(@lock_dir)
99
+
100
+ with_path_guard { retire_stale_unlocked }
101
+ rescue Errno::ENOENT
102
+ :missing
103
+ end
104
+
61
105
  # Release the lock.
62
106
  #
63
107
  # Deletes the lock file only if it still carries this instance's
@@ -69,28 +113,18 @@ module Woods
69
113
  def release
70
114
  return unless @held
71
115
 
72
- # Clear @held up front so no later failure can leave this instance
73
- # believing it still holds the lock.
74
- @held = false
116
+ with_path_guard do
117
+ next unless @held
75
118
 
76
- # Rename first, then inspect: a plain read-then-unlink is a TOCTOU —
77
- # after we read our own token a takeover could replace the file, and
78
- # our unlink would then delete the NEW holder's lock. Renaming
79
- # atomically captures whatever is at the path.
80
- graveyard = "#{@lock_path}.release.#{Process.pid}.#{SecureRandom.hex(4)}"
81
- begin
82
- File.rename(@lock_path, graveyard)
83
- rescue Errno::ENOENT
84
- return # already gone
85
- end
86
-
87
- if own_lock?(graveyard)
88
- FileUtils.rm_f(graveyard)
89
- else
90
- # We were legitimately taken over — put the successor's lock back
91
- # without clobbering a still-newer holder (see {#restore_lock}).
92
- restore_lock(graveyard)
119
+ # Clear @held before touching the ownership path so another thread
120
+ # using this instance cannot begin a heartbeat behind this release.
121
+ @held = false
122
+ release_unlocked
93
123
  end
124
+ rescue Errno::ENOENT
125
+ nil
126
+ ensure
127
+ @held = false
94
128
  end
95
129
 
96
130
  # Execute a block while holding the lock.
@@ -112,21 +146,104 @@ module Woods
112
146
  #
113
147
  # @return [Boolean]
114
148
  def locked?
115
- @held && File.exist?(@lock_path)
149
+ return false unless @held
150
+
151
+ with_path_guard { @held && File.exist?(@lock_path) }
152
+ rescue SystemCallError
153
+ false
154
+ end
155
+
156
+ # Refresh the held lock's mtime so a long but healthy run is not mistaken
157
+ # for a crashed one.
158
+ #
159
+ # Staleness is measured from mtime, and nothing was updating it while the
160
+ # lock was held. A full extraction on a large host app can exceed
161
+ # `stale_timeout`, at which point any contender retires the lock of a run
162
+ # that is still going — producing exactly the two-writer clobber the lock
163
+ # exists to prevent. A holder that outlives the window must therefore say
164
+ # so periodically.
165
+ #
166
+ # No-op unless this instance holds the lock, so a caller can heartbeat
167
+ # unconditionally.
168
+ #
169
+ # @return [Boolean] true when the mtime was refreshed
170
+ def touch
171
+ return false unless @held
172
+
173
+ with_path_guard do
174
+ next false unless @held && File.exist?(@lock_path)
175
+
176
+ touch_unlocked?
177
+ end
178
+ rescue SystemCallError
179
+ # The lock vanished (retired by a non-cooperating contender, or the
180
+ # directory went away). The next acquire/release resolves it.
181
+ false
116
182
  end
117
183
 
118
184
  private
119
185
 
120
- # Check if the existing lock file is stale.
186
+ # Remove a lock file this {#acquire} call created but could not finish
187
+ # writing (INF-7).
121
188
  #
122
- # @return [Boolean]
123
- def stale?
124
- return false unless File.exist?(@lock_path)
189
+ # A write that fails after the O_EXCL create — a full or read-only disk —
190
+ # would otherwise leave a 0-byte lock: fresh (mtime now, so never stale),
191
+ # unparseable (so {#lock_ownership} reads +:unknown+ for every process,
192
+ # including this one), and never released because +@held+ was never set.
193
+ # That artifact blocks every writer for the whole stale window; {#touch}
194
+ # was already fixed never to create one.
195
+ #
196
+ # Safe unconditionally: O_EXCL proves *this* call created the path, and
197
+ # the path guard is still held. Errors from the unlink itself are
198
+ # swallowed so they cannot replace the real failure being re-raised.
199
+ #
200
+ # @return [void]
201
+ def discard_partial_lock
202
+ File.unlink(@lock_path)
203
+ rescue SystemCallError
204
+ nil
205
+ end
125
206
 
126
- age = Time.now - File.mtime(@lock_path)
127
- age > @stale_timeout
207
+ def with_path_guard
208
+ # Ensure the lock directory itself exists — never its parent. A guard
209
+ # that lived beside @lock_dir needed the parent writable too, which a
210
+ # deployment that only grants write access to the lock directory
211
+ # can't satisfy.
212
+ FileUtils.mkdir_p(@lock_dir)
213
+ File.open(guard_path, File::RDWR | File::CREAT, 0o600) do |guard|
214
+ guard.flock(File::LOCK_EX)
215
+ yield
216
+ end
217
+ end
218
+
219
+ # The transaction guard, kept inside the owned lock directory so it
220
+ # resolves through the same symlink the lock file itself does. A guard
221
+ # computed from `File.expand_path(lock_dir)` split in two: expand_path
222
+ # normalizes `.`/`..` but never dereferences a symlink, so a real path
223
+ # and a symlinked alias of the same directory produced two different
224
+ # sibling guard files — flocks that never contended, defeating every
225
+ # TOCTOU protection `with_path_guard` exists to provide. Deriving the
226
+ # guard from `File.realpath` and placing it inside the directory
227
+ # resolves both aliases to the identical path, the same way `@lock_path`
228
+ # already does implicitly by living inside `lock_dir`.
229
+ #
230
+ # Memoized: only the first caller (which has just ensured the directory
231
+ # exists via `mkdir_p`) needs to resolve it; every later access reuses
232
+ # that resolution rather than re-touching the filesystem.
233
+ #
234
+ # @return [String]
235
+ def guard_path
236
+ @guard_path ||= File.join(canonical_lock_dir, self.class.guard_filename(@name))
237
+ end
238
+
239
+ # @return [String] the lock directory's real path when it exists,
240
+ # falling back to a lexical expansion when it does not — realpath
241
+ # requires an existing path and callers must never be made to create
242
+ # one just to compute a guard location.
243
+ def canonical_lock_dir
244
+ File.realpath(@lock_dir)
128
245
  rescue Errno::ENOENT
129
- true
246
+ File.expand_path(@lock_dir)
130
247
  end
131
248
 
132
249
  # Atomically retire a stale lock file via rename. Rename is atomic on
@@ -144,23 +261,116 @@ module Woods
144
261
  # fresh (someone beat us to the takeover), we put it back and lose the
145
262
  # race instead of clobbering a live holder.
146
263
  #
147
- # @return [Boolean] true if this process retired a genuinely stale lock
148
- def retire_stale_lock
264
+ # @return [Symbol] `:cleared` when genuinely stale, `:not_stale` when a
265
+ # fresher successor was restored instead, `:missing` when the lock
266
+ # was already gone
267
+ def retire_stale_unlocked
149
268
  graveyard = "#{@lock_path}.stale.#{Process.pid}.#{SecureRandom.hex(4)}"
150
269
  File.rename(@lock_path, graveyard)
151
270
 
152
271
  unless stale_file?(graveyard)
153
- # We grabbed a lock that is no longer stale — a competitor already
154
- # took over. Restore it (without clobbering a still-newer holder)
155
- # and back off.
156
- restore_lock(graveyard)
157
- return false
272
+ restore_lock_unlocked(graveyard)
273
+ return :not_stale
158
274
  end
159
275
 
160
276
  FileUtils.rm_f(graveyard)
161
- true
277
+ :cleared
162
278
  rescue Errno::ENOENT
163
- false
279
+ :missing
280
+ end
281
+
282
+ def release_unlocked
283
+ # Rename first, then inspect: a plain read-then-unlink is a TOCTOU —
284
+ # after we read our own token a takeover could replace the file, and
285
+ # our unlink would then delete the NEW holder's lock. Renaming
286
+ # atomically captures whatever is at the path.
287
+ graveyard = "#{@lock_path}.release.#{Process.pid}.#{SecureRandom.hex(4)}"
288
+ begin
289
+ File.rename(@lock_path, graveyard)
290
+ rescue Errno::ENOENT
291
+ return # already gone
292
+ end
293
+
294
+ if own_lock?(graveyard)
295
+ FileUtils.rm_f(graveyard)
296
+ else
297
+ # We were legitimately taken over — put the successor's lock back
298
+ # without clobbering a still-newer holder (see
299
+ # {#restore_lock_unlocked}).
300
+ restore_lock_unlocked(graveyard)
301
+ end
302
+ end
303
+
304
+ def touch_unlocked?
305
+ # Ownership, not just presence. `locked?` asks whether *a* lock file
306
+ # exists and this instance thinks it holds one — which stays true after
307
+ # a contender has retired us and put its own lock at the same path. A
308
+ # retired holder would then refresh the **successor's** mtime, and if
309
+ # that successor crashed its lock would never age out while the retired
310
+ # process lived, blocking every writer until it exited.
311
+ #
312
+ # That is not hypothetical for a heartbeat: being retired mid-run is the
313
+ # exact scenario a heartbeat exists to prevent, so it is also the state a
314
+ # heartbeat is most likely to find itself in when it fails to.
315
+ #
316
+ # Clearing @held is the honest response — we do not hold it, and
317
+ # continuing to believe we do would let `release` act on a lock that is
318
+ # someone else's.
319
+ case lock_ownership
320
+ when :ours
321
+ # Refresh with `File.utime`, never `FileUtils.touch`: touch CREATES a
322
+ # missing file, and the lock can vanish between the ownership read
323
+ # and the refresh — a heartbeat racing its own process's `release` is
324
+ # the ordinary way. Recreating it leaves an empty 0-byte lock no
325
+ # process owns and no release will delete, blocking every writer for
326
+ # the full stale window. `utime` raises ENOENT instead, which the
327
+ # rescue below turns into the same plain false as any other vanished
328
+ # lock; the next acquire/release resolves it.
329
+ now = Time.now
330
+ File.utime(now, now, @lock_path)
331
+ true
332
+ when :foreign
333
+ # Proven someone else's: stop believing we hold it, or `release` would
334
+ # act on their lock.
335
+ @held = false
336
+ false
337
+ else
338
+ # Unreadable — a torn write, most likely. Refuse to refresh, but stay
339
+ # the owner: "I cannot prove this is mine" is not "this is not mine",
340
+ # and disowning it here would strand the file. `release` opens with
341
+ # `return unless @held`, so a disowned lock is never cleaned up and
342
+ # every writer blocks until the stale window expires.
343
+ false
344
+ end
345
+ end
346
+
347
+ # Check if the existing lock file is stale.
348
+ #
349
+ # @return [Boolean]
350
+ def stale?
351
+ return false unless File.exist?(@lock_path)
352
+
353
+ age = Time.now - File.mtime(@lock_path)
354
+ age > @stale_timeout
355
+ rescue Errno::ENOENT
356
+ true
357
+ end
358
+
359
+ # Three-state, deliberately — a boolean here conflates the two states
360
+ # that need different handling.
361
+ #
362
+ # `:unknown` is not `:foreign`. Both refuse a refresh, because touching a
363
+ # lock we cannot prove is ours would extend a stranger's claim. Only
364
+ # `:foreign` justifies disowning, because only that means someone else
365
+ # holds it. Collapsing them made an unreadable-but-ours lock leak: `touch`
366
+ # cleared `@held`, and `release` returns early on `@held`, so nothing ever
367
+ # removed the file and every writer blocked for the full stale window.
368
+ #
369
+ # @return [Symbol] `:ours`, `:foreign`, or `:unknown`
370
+ def lock_ownership
371
+ JSON.parse(File.read(@lock_path))['token'] == @token ? :ours : :foreign
372
+ rescue JSON::ParserError, SystemCallError
373
+ :unknown
164
374
  end
165
375
 
166
376
  # Whether the lock file at +path+ carries this instance's token.
@@ -184,7 +394,7 @@ module Woods
184
394
  #
185
395
  # @param graveyard [String] path of the renamed-aside lock file
186
396
  # @return [void]
187
- def restore_lock(graveyard)
397
+ def restore_lock_unlocked(graveyard)
188
398
  File.link(graveyard, @lock_path)
189
399
  rescue Errno::EEXIST
190
400
  # A newer holder already claimed the path — our copy is obsolete.
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Woods
4
+ module Db
5
+ module Migrations
6
+ # Preserve the full identity of units sharing a name across extractor types.
7
+ module TypedSnapshotUnits
8
+ VERSION = 7
9
+
10
+ def self.up(connection)
11
+ sql = connection.get_first_value("SELECT sql FROM sqlite_master WHERE name = 'woods_snapshot_units'")
12
+ CreateSnapshotUnits.up(connection) if sql.nil?
13
+ return if sql&.match?(/UNIQUE\s*\(snapshot_id, identifier, unit_type\)/i)
14
+
15
+ connection.transaction do
16
+ create_table(connection)
17
+ connection.execute('INSERT INTO woods_snapshot_units_typed SELECT * FROM woods_snapshot_units')
18
+ connection.execute('DROP TABLE woods_snapshot_units')
19
+ connection.execute('ALTER TABLE woods_snapshot_units_typed RENAME TO woods_snapshot_units')
20
+ connection.execute('CREATE INDEX idx_snapshot_units_identifier ON woods_snapshot_units(identifier)')
21
+ connection.execute('CREATE INDEX idx_snapshot_units_snapshot ON woods_snapshot_units(snapshot_id)')
22
+ end
23
+ end
24
+
25
+ def self.create_table(connection)
26
+ connection.execute(<<~SQL)
27
+ CREATE TABLE woods_snapshot_units_typed (
28
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
29
+ snapshot_id INTEGER NOT NULL,
30
+ identifier TEXT NOT NULL,
31
+ unit_type TEXT NOT NULL,
32
+ source_hash TEXT,
33
+ metadata_hash TEXT,
34
+ dependencies_hash TEXT,
35
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
36
+ FOREIGN KEY (snapshot_id) REFERENCES woods_snapshots(id),
37
+ UNIQUE(snapshot_id, identifier, unit_type)
38
+ )
39
+ SQL
40
+ end
41
+ private_class_method :create_table
42
+ end
43
+ end
44
+ end
45
+ end
@@ -7,6 +7,7 @@ require_relative 'migrations/003_create_embeddings'
7
7
  require_relative 'migrations/004_create_snapshots'
8
8
  require_relative 'migrations/005_create_snapshot_units'
9
9
  require_relative 'migrations/006_rename_tables'
10
+ require_relative 'migrations/007_typed_snapshot_units'
10
11
 
11
12
  module Woods
12
13
  module Db
@@ -28,7 +29,8 @@ module Woods
28
29
  Migrations::CreateEmbeddings,
29
30
  Migrations::CreateSnapshots,
30
31
  Migrations::CreateSnapshotUnits,
31
- Migrations::RenameTables
32
+ Migrations::RenameTables,
33
+ Migrations::TypedSnapshotUnits
32
34
  ].freeze
33
35
 
34
36
  attr_reader :schema_version
@@ -53,14 +55,6 @@ module Woods
53
55
  applied
54
56
  end
55
57
 
56
- # List version numbers of pending (unapplied) migrations.
57
- #
58
- # @return [Array<Integer>]
59
- def pending_versions
60
- applied = @schema_version.applied_versions
61
- MIGRATIONS.map { |m| m::VERSION }.reject { |v| applied.include?(v) }
62
- end
63
-
64
58
  private
65
59
 
66
60
  # @return [Array<Module>] Pending migration modules