woods 2.0.0.beta2 → 2.0.0.beta4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +339 -1
  3. data/CONTRIBUTING.md +188 -12
  4. data/README.md +93 -174
  5. data/SECURITY.md +9 -6
  6. data/docs/AGENT_GUIDE.md +109 -8
  7. data/docs/AGENT_SETUP.md +98 -7
  8. data/docs/BACKEND_MATRIX.md +25 -0
  9. data/docs/CLIENT_HOOKS.md +111 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +267 -16
  11. data/docs/CONSOLE_MCP_SETUP.md +80 -7
  12. data/docs/DOCKER_SETUP.md +22 -3
  13. data/docs/EVALUATION.md +464 -1
  14. data/docs/EXTRACTOR_REFERENCE.md +45 -6
  15. data/docs/FAQ.md +11 -12
  16. data/docs/GETTING_STARTED.md +17 -5
  17. data/docs/INCREMENTAL_EXTRACTION.md +147 -7
  18. data/docs/INDEX_LAYOUT.md +382 -0
  19. data/docs/INTERNALS.md +7 -2
  20. data/docs/MCP_SERVERS.md +276 -5
  21. data/docs/MCP_TOOL_COOKBOOK.md +37 -22
  22. data/docs/MCP_WORKTREE_SETUP.md +43 -83
  23. data/docs/NOTION_INTEGRATION.md +13 -0
  24. data/docs/OBSIDIAN_INTEGRATION.md +57 -9
  25. data/docs/PUBLISHED_INDEX.md +72 -0
  26. data/docs/README.md +7 -0
  27. data/docs/RETRIEVAL_GUIDE.md +273 -12
  28. data/docs/RUNTIME_TRACING.md +71 -0
  29. data/docs/SOURCE_FRESHNESS.md +143 -0
  30. data/docs/TROUBLESHOOTING.md +129 -18
  31. data/docs/UNBLOCKED_INTEGRATION.md +25 -0
  32. data/docs/UPGRADING_TO_2.md +48 -22
  33. data/docs/WATCH_DAEMON.md +277 -67
  34. data/exe/woods-agent-config +6 -0
  35. data/exe/woods-extract +5 -0
  36. data/exe/woods-hook-context +6 -0
  37. data/exe/woods-mcp-start +14 -9
  38. data/lib/generators/woods/pgvector_generator.rb +8 -2
  39. data/lib/generators/woods/templates/woods.rb.tt +1 -3
  40. data/lib/tasks/woods.rake +47 -397
  41. data/lib/woods/agent_configuration/applier.rb +135 -0
  42. data/lib/woods/agent_configuration/cli.rb +101 -0
  43. data/lib/woods/agent_configuration/cli_options.rb +29 -0
  44. data/lib/woods/agent_configuration/document.rb +105 -0
  45. data/lib/woods/agent_configuration/error.rb +7 -0
  46. data/lib/woods/agent_configuration/launcher.rb +75 -0
  47. data/lib/woods/agent_configuration/layout.rb +72 -0
  48. data/lib/woods/agent_configuration/managed_section.rb +62 -0
  49. data/lib/woods/agent_configuration/plan.rb +98 -0
  50. data/lib/woods/agent_configuration/plan_diff.rb +38 -0
  51. data/lib/woods/agent_configuration/planned_files.rb +61 -0
  52. data/lib/woods/agent_configuration/planner.rb +63 -0
  53. data/lib/woods/agent_configuration/planner_validation.rb +77 -0
  54. data/lib/woods/agent_configuration/preflight.rb +100 -0
  55. data/lib/woods/agent_configuration/recovery.rb +49 -0
  56. data/lib/woods/ast/node.rb +2 -0
  57. data/lib/woods/ast/parser.rb +38 -5
  58. data/lib/woods/builder.rb +21 -5
  59. data/lib/woods/cache/cache_middleware.rb +28 -7
  60. data/lib/woods/cache/cache_store.rb +4 -5
  61. data/lib/woods/change_set.rb +5 -4
  62. data/lib/woods/console/credential_index.rb +20 -2
  63. data/lib/woods/console/credential_scanner.rb +18 -17
  64. data/lib/woods/console/credential_scanner_registry.rb +36 -0
  65. data/lib/woods/console/dispatch_pipeline.rb +7 -0
  66. data/lib/woods/console/embedded_executor.rb +32 -10
  67. data/lib/woods/console/encrypted_credential_snapshot.rb +16 -0
  68. data/lib/woods/console/rack_middleware.rb +22 -13
  69. data/lib/woods/console/server.rb +18 -16
  70. data/lib/woods/console/sql_noise_stripper.rb +9 -7
  71. data/lib/woods/console/sql_table_scanner.rb +47 -7
  72. data/lib/woods/console/sql_validator.rb +49 -9
  73. data/lib/woods/console/sqlite_read_guard.rb +46 -0
  74. data/lib/woods/coordination/pipeline_lock.rb +3 -2
  75. data/lib/woods/dependency_graph.rb +65 -13
  76. data/lib/woods/embedding/corpus.rb +94 -0
  77. data/lib/woods/embedding/indexer.rb +114 -60
  78. data/lib/woods/embedding/openai.rb +17 -6
  79. data/lib/woods/evaluation/ablation_executor.rb +6 -1
  80. data/lib/woods/evaluation/ablation_timed_executor.rb +22 -4
  81. data/lib/woods/export/typed_reader.rb +56 -0
  82. data/lib/woods/extractor.rb +277 -149
  83. data/lib/woods/extractors/action_cable_extractor.rb +3 -1
  84. data/lib/woods/extractors/behavioral_profile.rb +9 -7
  85. data/lib/woods/extractors/caching_extractor.rb +3 -1
  86. data/lib/woods/extractors/concern_extractor.rb +64 -6
  87. data/lib/woods/extractors/configuration_extractor.rb +7 -3
  88. data/lib/woods/extractors/controller_extractor.rb +13 -4
  89. data/lib/woods/extractors/database_view_extractor.rb +3 -1
  90. data/lib/woods/extractors/declared_parent.rb +55 -0
  91. data/lib/woods/extractors/decorator_extractor.rb +3 -1
  92. data/lib/woods/extractors/engine_extractor.rb +3 -1
  93. data/lib/woods/extractors/event_extractor.rb +4 -2
  94. data/lib/woods/extractors/factory_extractor.rb +3 -1
  95. data/lib/woods/extractors/graphql_extractor.rb +10 -13
  96. data/lib/woods/extractors/i18n_extractor.rb +3 -1
  97. data/lib/woods/extractors/job_extractor.rb +6 -19
  98. data/lib/woods/extractors/lib_extractor.rb +13 -9
  99. data/lib/woods/extractors/mailer_extractor.rb +26 -15
  100. data/lib/woods/extractors/manager_extractor.rb +3 -1
  101. data/lib/woods/extractors/method_parameters.rb +53 -0
  102. data/lib/woods/extractors/middleware_argument.rb +65 -0
  103. data/lib/woods/extractors/middleware_extractor.rb +9 -3
  104. data/lib/woods/extractors/migration_extractor.rb +3 -1
  105. data/lib/woods/extractors/model_extractor.rb +26 -34
  106. data/lib/woods/extractors/package_extractor.rb +24 -4
  107. data/lib/woods/extractors/phlex_extractor.rb +3 -1
  108. data/lib/woods/extractors/policy_extractor.rb +3 -1
  109. data/lib/woods/extractors/poro_extractor.rb +13 -9
  110. data/lib/woods/extractors/pundit_extractor.rb +3 -1
  111. data/lib/woods/extractors/rails_source_extractor.rb +4 -2
  112. data/lib/woods/extractors/rake_task_extractor.rb +4 -2
  113. data/lib/woods/extractors/route_extractor.rb +3 -1
  114. data/lib/woods/extractors/route_helper_resolver.rb +10 -33
  115. data/lib/woods/extractors/scheduled_job_extractor.rb +41 -15
  116. data/lib/woods/extractors/serializer_extractor.rb +4 -2
  117. data/lib/woods/extractors/service_extractor.rb +3 -1
  118. data/lib/woods/extractors/shared_dependency_scanner.rb +2 -2
  119. data/lib/woods/extractors/shared_utility_methods.rb +48 -19
  120. data/lib/woods/extractors/source_nesting.rb +1 -1
  121. data/lib/woods/extractors/state_machine_extractor.rb +3 -1
  122. data/lib/woods/extractors/test_mapping_extractor.rb +3 -1
  123. data/lib/woods/extractors/validator_extractor.rb +3 -1
  124. data/lib/woods/extractors/view_component_extractor.rb +3 -1
  125. data/lib/woods/extractors/view_template_extractor.rb +3 -1
  126. data/lib/woods/gem_mapper.rb +2 -0
  127. data/lib/woods/git_history.rb +116 -0
  128. data/lib/woods/graph_analyzer.rb +35 -6
  129. data/lib/woods/hooks/context_cli.rb +54 -0
  130. data/lib/woods/hooks/context_event.rb +88 -0
  131. data/lib/woods/hooks/context_hint.rb +73 -0
  132. data/lib/woods/hooks/context_impact.rb +77 -0
  133. data/lib/woods/hooks/context_output.rb +47 -0
  134. data/lib/woods/hooks/context_state.rb +102 -0
  135. data/lib/woods/hooks/refresh.rb +79 -0
  136. data/lib/woods/hooks/rule_projection.rb +78 -0
  137. data/lib/woods/input_rules.rb +19 -0
  138. data/lib/woods/mcp/bearer_auth.rb +22 -13
  139. data/lib/woods/mcp/bootstrapper.rb +79 -4
  140. data/lib/woods/mcp/config_resolver.rb +2 -1
  141. data/lib/woods/mcp/index_reader.rb +334 -162
  142. data/lib/woods/mcp/initialization_guidance.rb +27 -0
  143. data/lib/woods/mcp/origin_guard.rb +17 -9
  144. data/lib/woods/mcp/published_lexical_retriever.rb +115 -0
  145. data/lib/woods/mcp/renderers/markdown_renderer.rb +22 -9
  146. data/lib/woods/mcp/renderers/plain_renderer.rb +18 -8
  147. data/lib/woods/mcp/search_results.rb +74 -0
  148. data/lib/woods/mcp/server.rb +178 -63
  149. data/lib/woods/mcp/tool_contract.rb +3 -1
  150. data/lib/woods/mcp/tool_response_renderer.rb +41 -0
  151. data/lib/woods/mcp/traversal_evidence.rb +113 -0
  152. data/lib/woods/mcp/traversal_evidence_index.rb +100 -0
  153. data/lib/woods/mcp/traversal_evidence_page.rb +41 -0
  154. data/lib/woods/mcp/traversal_evidence_text.rb +52 -0
  155. data/lib/woods/mcp/traversal_response.rb +22 -0
  156. data/lib/woods/notion/exporter.rb +56 -17
  157. data/lib/woods/obsidian/destination_plan.rb +98 -0
  158. data/lib/woods/obsidian/name_mapper.rb +19 -3
  159. data/lib/woods/obsidian/note_builder.rb +19 -10
  160. data/lib/woods/obsidian/vault_exporter.rb +88 -32
  161. data/lib/woods/operator/pipeline_guard.rb +18 -13
  162. data/lib/woods/path_dispatcher.rb +13 -6
  163. data/lib/woods/payload_store.rb +27 -26
  164. data/lib/woods/published_index/typed_unit_reader.rb +40 -3
  165. data/lib/woods/published_index.rb +2 -2
  166. data/lib/woods/railtie.rb +3 -3
  167. data/lib/woods/railtie_support.rb +12 -12
  168. data/lib/woods/rake_helpers.rb +382 -0
  169. data/lib/woods/resilience/graph_invariant_validator/membership_checks.rb +71 -0
  170. data/lib/woods/resilience/graph_invariant_validator/node_checks.rb +61 -0
  171. data/lib/woods/resilience/graph_invariant_validator/reverse_relationship_checks.rb +46 -0
  172. data/lib/woods/resilience/graph_invariant_validator.rb +119 -0
  173. data/lib/woods/resilience/index_validator/graph_checks.rb +80 -0
  174. data/lib/woods/resilience/index_validator.rb +112 -23
  175. data/lib/woods/retrieval/context_assembler.rb +50 -15
  176. data/lib/woods/retrieval/lexical_assembler.rb +84 -0
  177. data/lib/woods/retrieval/lexical_index.rb +120 -0
  178. data/lib/woods/retrieval/ranker.rb +4 -2
  179. data/lib/woods/retrieval/scope.rb +108 -0
  180. data/lib/woods/retrieval/scoped_graph_store.rb +32 -0
  181. data/lib/woods/retrieval/scoped_vector_store.rb +55 -0
  182. data/lib/woods/retrieval/search_executor.rb +86 -27
  183. data/lib/woods/retrieval/source_evidence.rb +200 -0
  184. data/lib/woods/retriever.rb +98 -22
  185. data/lib/woods/ruby_analyzer/trace_enricher.rb +77 -38
  186. data/lib/woods/session_tracer/file_store.rb +6 -1
  187. data/lib/woods/session_tracer/middleware.rb +10 -12
  188. data/lib/woods/session_tracer/redis_store.rb +22 -6
  189. data/lib/woods/session_tracer/session_flow_assembler.rb +23 -17
  190. data/lib/woods/session_tracer/solid_cache_coordination.rb +6 -4
  191. data/lib/woods/session_tracer/unit_resolver.rb +63 -0
  192. data/lib/woods/source_inputs/consumer_errors.rb +31 -0
  193. data/lib/woods/source_inputs/handoff.rb +102 -0
  194. data/lib/woods/source_inputs/launcher.rb +157 -0
  195. data/lib/woods/source_inputs/manifest.rb +124 -0
  196. data/lib/woods/source_inputs/private_key.rb +55 -0
  197. data/lib/woods/source_inputs/scanner.rb +171 -0
  198. data/lib/woods/source_inputs/scopes.rb +71 -0
  199. data/lib/woods/source_inputs/session.rb +214 -0
  200. data/lib/woods/source_inputs/status.rb +84 -0
  201. data/lib/woods/source_inputs/verifier.rb +107 -0
  202. data/lib/woods/storage/metadata_store.rb +25 -25
  203. data/lib/woods/storage/pgvector.rb +35 -10
  204. data/lib/woods/storage/qdrant.rb +17 -7
  205. data/lib/woods/storage/vector_store.rb +18 -6
  206. data/lib/woods/tasks.rb +3 -2
  207. data/lib/woods/temporal/json_snapshot_store.rb +58 -9
  208. data/lib/woods/unblocked/exporter.rb +59 -70
  209. data/lib/woods/version.rb +1 -1
  210. data/lib/woods/watch/boot_snapshot.rb +52 -0
  211. data/lib/woods/watch/daemon.rb +154 -32
  212. data/lib/woods/watch/listen_watcher.rb +4 -0
  213. data/lib/woods/watch/polling_watcher.rb +5 -1
  214. data/lib/woods/watch/status.rb +20 -15
  215. data/lib/woods/watch/tree_scan.rb +21 -13
  216. data/lib/woods/watch/watcher.rb +4 -1
  217. data/lib/woods.rb +50 -11
  218. data/plugin/.claude-plugin/plugin.json +1 -1
  219. data/plugin/hooks/adapters/normalize.jq +15 -0
  220. data/plugin/hooks/adapters/normalize.rb +63 -0
  221. data/plugin/hooks/hooks.json +20 -0
  222. data/plugin/hooks/woods-context.sh +50 -0
  223. data/plugin/hooks/woods-input-rules.sh +159 -0
  224. data/plugin/hooks/woods-opencode.mjs +65 -0
  225. data/plugin/hooks/woods-post-edit.sh +2 -225
  226. data/plugin/hooks/woods-refresh.sh +260 -0
  227. data/plugin/hooks/woods-session-start.sh +47 -55
  228. data/plugin/skills/woods-agent-enable/SKILL.md +19 -0
  229. data/plugin/skills/woods-diagnose/SKILL.md +319 -1
  230. data/plugin/skills/woods-investigate/SKILL.md +145 -0
  231. data/plugin/skills/woods-mcp-config/SKILL.md +90 -2
  232. data/plugin/skills/woods-setup/SKILL.md +110 -6
  233. metadata +87 -5
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+ require 'json'
5
+ require_relative '../ast/parser'
6
+ require_relative 'lexical_index'
7
+
8
+ module Woods
9
+ module Retrieval
10
+ # Selects complete spans from published unit bytes, never from host files.
11
+ # Published coordinates deliberately do not claim physical source locations:
12
+ # extractors may synthesize headers or inline commented concern bodies.
13
+ class SourceEvidence
14
+ MODES = %w[full compact outline].freeze
15
+ Result = Struct.new(:text, :provenance, keyword_init: true)
16
+ Span = Struct.new(:kind, :name, :lexical_owner, :receiver, :start_byte, :end_byte, :start_line, :end_line,
17
+ keyword_init: true)
18
+ private_constant :Span
19
+
20
+ def self.validate_mode!(mode)
21
+ return mode if MODES.include?(mode)
22
+
23
+ raise ArgumentError, 'evidence must be full, compact, or outline'
24
+ end
25
+
26
+ def initialize(unit:, query: nil, generation: nil, parser: Ast::Parser.new)
27
+ @unit = unit
28
+ @source = field(:source_code).to_s
29
+ @query = terms(query)
30
+ @generation = generation
31
+ @parser = parser
32
+ end
33
+
34
+ # The caller's counter includes the same tokenizer/estimate used by its
35
+ # response budget. No selected definition or metadata value is cut in half.
36
+ def render(mode:, budget:, counter:)
37
+ self.class.validate_mode!(mode)
38
+ raise ArgumentError, 'budget must be a positive Integer' unless budget.is_a?(Integer) && budget.positive?
39
+ raise ArgumentError, 'compact evidence requires compact or outline mode' if mode == 'full'
40
+
41
+ spans = source_spans
42
+ selected = []
43
+ metadata = []
44
+ ordered = spans.sort_by { |span| [-relevance(span), span.start_byte] }
45
+ relevant, other = ordered.partition { |span| @query.empty? || relevance(span).positive? }
46
+ relevant.each do |span|
47
+ trial = selected + [span]
48
+ selected = trial if counter.call(format_evidence(mode, trial, metadata, spans.size)) <= budget
49
+ end
50
+ runtime_records.each do |record|
51
+ trial = metadata + [record]
52
+ metadata = trial if counter.call(format_evidence(mode, selected, trial, spans.size)) <= budget
53
+ end
54
+ other.each do |span|
55
+ trial = selected + [span]
56
+ selected = trial if counter.call(format_evidence(mode, trial, metadata, spans.size)) <= budget
57
+ end
58
+ text = format_evidence(mode, selected, metadata, spans.size)
59
+ return Result.new(text: '', provenance: provenance(mode, [], [], spans)) if counter.call(text) > budget
60
+
61
+ Result.new(text: text, provenance: provenance(mode, selected, metadata, spans))
62
+ end
63
+
64
+ private
65
+
66
+ def field(key)
67
+ @unit[key] || @unit[key.to_s]
68
+ end
69
+
70
+ def source_spans
71
+ return [] if @source.empty?
72
+
73
+ spans = []
74
+ walk(@parser.parse(@source), nil, spans)
75
+ return whole_source_span('whole_source_fallback') unless spans.all? { |span| complete_definition?(span) }
76
+
77
+ spans.concat(concern_spans)
78
+ return spans unless spans.empty?
79
+
80
+ whole_source_span('published_source')
81
+ rescue Woods::ExtractionError
82
+ # Non-Ruby or legacy synthesized source remains available as one whole
83
+ # published span. An unavailable boundary is never guessed with regex.
84
+ whole_source_span('unparsed_source')
85
+ end
86
+
87
+ def whole_source_span(kind)
88
+ [Span.new(kind: kind, name: field(:identifier), lexical_owner: nil,
89
+ start_byte: 0, end_byte: @source.bytesize, start_line: 1, end_line: @source.lines.size)]
90
+ end
91
+
92
+ # A heredoc body may live outside a def node's syntactic range (notably
93
+ # endless defs and same-line `def ...; <<~DOC; end`). Refuse a broken
94
+ # isolated definition and fall back to the complete published source.
95
+ def complete_definition?(span)
96
+ @parser.valid_fragment?(span_text(span))
97
+ end
98
+
99
+ def walk(node, owner, spans, singleton_scope: false)
100
+ return unless node.is_a?(Ast::Node)
101
+
102
+ if %i[class module].include?(node.type)
103
+ singleton_scope = false
104
+ name = node.method_name.to_s
105
+ owner = name.start_with?('::') ? name.delete_prefix('::') : [owner, name].compact.join('::')
106
+ elsif %i[def defs].include?(node.type)
107
+ if node.start_byte && node.end_byte
108
+ kind = if singleton_scope
109
+ 'singleton_body_method'
110
+ else
111
+ (node.type == :def ? 'instance_method' : 'singleton_method')
112
+ end
113
+ spans << Span.new(kind: kind,
114
+ name: node.method_name, lexical_owner: owner, receiver: node.receiver,
115
+ start_byte: node.start_byte,
116
+ end_byte: node.end_byte, start_line: node.line, end_line: node.end_line)
117
+ end
118
+ return # Nested definitions/blocks are already covered by the complete outer method.
119
+ end
120
+ node.children&.each do |child|
121
+ walk(child, owner, spans, singleton_scope: singleton_scope || node.type == :sclass)
122
+ end
123
+ end
124
+
125
+ # These are Woods' published display blocks, not physical concern source.
126
+ # Require matching published metadata and delimiters; preserve every byte,
127
+ # including comment prefixes, instead of reconstructing executable Ruby.
128
+ def concern_spans
129
+ metadata = field(:metadata) || {}
130
+ Array(metadata['inlined_concerns'] || metadata[:inlined_concerns]).filter_map do |name|
131
+ escaped = Regexp.escape(name.to_s)
132
+ pattern = /^# │ Included from: #{escaped}[^\S\r\n]*│\r?\n.*?^# ─+ End #{escaped} ─+\r?$/m
133
+ match = pattern.match(@source)
134
+ next unless match
135
+
136
+ start_byte = @source[0...match.begin(0)].bytesize
137
+ end_byte = @source[0...match.end(0)].bytesize
138
+ Span.new(kind: 'inlined_concern_display', name: name, lexical_owner: name,
139
+ start_byte: start_byte, end_byte: end_byte,
140
+ start_line: @source.byteslice(0, start_byte).count("\n") + 1,
141
+ end_line: @source.byteslice(0, end_byte).count("\n") + 1)
142
+ end
143
+ end
144
+
145
+ def terms(value)
146
+ value.to_s.gsub(/(\p{Ll}|\d)(\p{Lu})/u, '\1 \2').downcase.scan(/[\p{L}\p{N}]+/u)
147
+ .reject { |term| QueryClassifier::STOP_WORDS.include?(term) }.uniq
148
+ end
149
+
150
+ def relevance(span)
151
+ ((terms(span.name) & @query).size * 4) + (terms(span_text(span)) & @query).size
152
+ end
153
+
154
+ def span_text(span)
155
+ @source.byteslice(span.start_byte...span.end_byte)
156
+ end
157
+
158
+ def runtime_records
159
+ metadata = field(:metadata)
160
+ return [] unless metadata.is_a?(Hash)
161
+
162
+ LexicalIndex::RUNTIME_FIELDS.filter_map do |key|
163
+ value = metadata[key] || metadata[key.to_sym]
164
+ next if value.nil?
165
+
166
+ record = [key, value]
167
+ record if @query.empty? || (terms(JSON.generate(record)) & @query).any?
168
+ end
169
+ end
170
+
171
+ def format_evidence(mode, selected, metadata, total)
172
+ parts = ["Evidence: #{mode}; published-unit coordinates (physical location unavailable).",
173
+ "Unit: #{field(:type)}:#{field(:identifier)}; path: #{field(:file_path)}",
174
+ "Source SHA256: #{Digest::SHA256.hexdigest(@source)}",
175
+ "Generation: #{@generation || 'unavailable (not recorded by this metadata store)'}"]
176
+ selected.each do |span|
177
+ label = "#{span.kind} #{span.lexical_owner} #{span.name}; published lines #{span.start_line}-#{span.end_line}"
178
+ parts << (mode == 'outline' ? label : "#{label}\n```ruby\n#{span_text(span)}\n```")
179
+ end
180
+ parts << "Published runtime metadata:\n#{JSON.generate(metadata.to_h)}" unless metadata.empty?
181
+ parts << "Omitted: #{total - selected.size} source spans; remaining source/metadata not reproduced. " \
182
+ 'Use lookup with evidence: full for the complete published unit.'
183
+ parts.join("\n\n")
184
+ end
185
+
186
+ def provenance(mode, selected, metadata, all)
187
+ { mode: mode, owner: { identifier: field(:identifier), type: field(:type) },
188
+ coordinate_system: 'published_unit', source_sha256: Digest::SHA256.hexdigest(@source),
189
+ source_path: field(:file_path), physical_location: nil,
190
+ physical_location_reason: 'published unit may contain synthesized or inlined source',
191
+ generation: @generation, generation_status: @generation ? 'recorded' : 'unavailable',
192
+ spans: selected.map { |span| span.to_h.merge(sha256: Digest::SHA256.hexdigest(span_text(span))) },
193
+ omitted_spans: all.size - selected.size, runtime_fields: metadata.map(&:first),
194
+ source_complete: false, full_evidence: { tool: 'lookup', evidence: 'full',
195
+ identifier: field(:identifier), type: field(:type),
196
+ source_sha256: Digest::SHA256.hexdigest(@source) } }
197
+ end
198
+ end
199
+ end
200
+ end
@@ -8,6 +8,11 @@ require_relative 'retrieval/query_classifier'
8
8
  require_relative 'retrieval/search_executor'
9
9
  require_relative 'retrieval/ranker'
10
10
  require_relative 'retrieval/context_assembler'
11
+ require_relative 'retrieval/lexical_index'
12
+ require_relative 'retrieval/lexical_assembler'
13
+ require_relative 'retrieval/scope'
14
+ require_relative 'retrieval/scoped_vector_store'
15
+ require_relative 'retrieval/scoped_graph_store'
11
16
  require_relative 'embedding/token_counter'
12
17
  require_relative 'token_utils'
13
18
 
@@ -59,6 +64,7 @@ module Woods
59
64
  ).freeze
60
65
 
61
66
  # Diagnostic trace for retrieval quality analysis.
67
+ # +tokens_used+ counts the final delivered context, including postprocessing.
62
68
  #
63
69
  # +skipped_missing_metadata+ carries {Retrieval::ContextAssembler}'s count
64
70
  # of candidates dropped because the metadata store had no record for
@@ -98,7 +104,7 @@ module Woods
98
104
  #
99
105
  # Nil for unfiltered queries.
100
106
  RetrievalResult = Struct.new(:context, :sources, :classification, :strategy, :tokens_used, :budget, :trace,
101
- :type_rank_context, keyword_init: true)
107
+ :type_rank_context, :applied_scope, keyword_init: true)
102
108
 
103
109
  # Raised when a metadata-store access fails during retrieval (M8). One
104
110
  # shared typed error for every store call site: the retriever used to
@@ -191,7 +197,7 @@ module Woods
191
197
  # The reload transaction swaps the whole struct via {#swap_stores!}.
192
198
  #
193
199
  # @return [Pipeline]
194
- attr_reader :pipeline
200
+ attr_reader :pipeline, :mode, :default_budget
195
201
 
196
202
  # Optional callback invoked with the pipeline struct the moment
197
203
  # {#retrieve} resolves it, before any pipeline work runs. Nil in
@@ -207,7 +213,13 @@ module Woods
207
213
  # @param graph_store [Storage::GraphStore::Interface] Graph store adapter
208
214
  # @param embedding_provider [Embedding::Provider::Interface] Embedding provider
209
215
  # @param formatter [#call, nil] Optional callable to post-process the context string
210
- def initialize(vector_store:, metadata_store:, graph_store:, embedding_provider:, formatter: nil)
216
+ # @param default_budget [Integer] Token budget when retrieve omits budget
217
+ def initialize(vector_store:, metadata_store:, graph_store:, embedding_provider:, formatter: nil, mode: :semantic,
218
+ default_budget: 8000)
219
+ raise ArgumentError, 'unknown retrieval mode' unless %i[semantic lexical].include?(mode)
220
+
221
+ @mode = mode
222
+ @default_budget = default_budget
211
223
  @embedding_provider = embedding_provider
212
224
  @formatter = formatter
213
225
  @classifier = Retrieval::QueryClassifier.new
@@ -246,7 +258,9 @@ module Woods
246
258
  # they make raises the typed {StoreError} instead of a raw adapter error
247
259
  # (MCP-6); the struct keeps the raw adapters for identity and capability
248
260
  # checks.
249
- def build_pipeline(vector_store:, metadata_store:, graph_store:)
261
+ def build_pipeline(vector_store:, metadata_store:, graph_store:) # rubocop:disable Metrics/MethodLength
262
+ return build_lexical_pipeline(metadata_store, graph_store) if @mode == :lexical
263
+
250
264
  translated_vector = translate_store(vector_store, :vector)
251
265
  translated_metadata = translate_store(metadata_store, :metadata)
252
266
  translated_graph = translate_store(graph_store, :graph)
@@ -269,6 +283,14 @@ module Woods
269
283
  graph_store: graph_store
270
284
  )
271
285
  end
286
+
287
+ def build_lexical_pipeline(metadata_store, graph_store)
288
+ executor = Retrieval::LexicalIndex.new(metadata_store: translate_store(metadata_store, :metadata))
289
+ Pipeline.new(executor: executor, assembler: Retrieval::LexicalAssembler.new, metadata_store: metadata_store,
290
+ vector_store: nil, graph_store: graph_store)
291
+ end
292
+ private :build_lexical_pipeline
293
+
272
294
  private :build_pipeline
273
295
 
274
296
  # Wrap one store adapter for the pipeline components. Nil stays nil — a
@@ -357,16 +379,29 @@ module Woods
357
379
  # unit types (overrides DEFAULT_EXCLUDE_TYPES).
358
380
  # @param exclude_types [Array<String, Symbol>, nil] Additional types to
359
381
  # exclude. Applied on top of DEFAULT_EXCLUDE_TYPES unless +types:+ is set.
360
- # @return [RetrievalResult] Complete retrieval result
361
- def retrieve(query, budget: 8000, types: nil, exclude_types: nil)
382
+ # @param packages [Array<String>, nil] Exact published nearest package owners
383
+ # @param source_paths [Array<String>, nil] Application-relative directory prefixes
384
+ # @return [RetrievalResult] Complete retrieval result; scoped calls carry applied_scope
385
+ def retrieve(query, budget: @default_budget, types: nil, exclude_types: nil, packages: nil, source_paths: nil,
386
+ evidence: 'full', evidence_generation: nil) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
362
387
  validate_query!(query)
388
+ Retrieval::SourceEvidence.validate_mode!(evidence)
389
+ evidence_options = evidence == 'full' ? {} : { evidence: evidence, query: query, generation: evidence_generation }
363
390
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
364
391
  # One atomic read of the bundle reference: everything this query does
365
392
  # from here on — execution, ranking, filtering, assembly — stays on the
366
393
  # SAME store set even if a reload swaps the pipeline mid-flight (M7).
367
394
  pipeline = @pipeline
368
395
  @pipeline_observer&.call(pipeline)
396
+ scope = resolve_scope(pipeline, packages, source_paths, types, exclude_types)
397
+ pipeline = scoped_pipeline(pipeline, scope) if scope
369
398
  classification = @classifier.classify(query)
399
+ if @mode == :lexical
400
+ result = retrieve_lexical(pipeline, query, classification, budget, types, exclude_types, start_time,
401
+ **evidence_options)
402
+ return attach_scope(result, scope)
403
+ end
404
+
370
405
  execution_result = pipeline.executor.execute(query: query, classification: classification)
371
406
  ranked = pipeline.ranker.rank(execution_result.candidates, classification: classification)
372
407
 
@@ -374,20 +409,61 @@ module Woods
374
409
  filtered, fallback_ran = apply_type_filter(pipeline, ranked, query, classification,
375
410
  types: types, type_list: type_list,
376
411
  exclude_types: exclude_types)
377
- type_rank_context = build_type_rank_context_for(ranked, pipeline, type_list, filtered,
378
- fallback_ran: fallback_ran)
379
-
380
- assembled = assemble_context(pipeline, filtered, classification, budget)
381
- trace = build_trace(classification, execution_result, filtered, assembled, start_time)
412
+ type_rank_context = unless scope
413
+ build_type_rank_context_for(ranked, pipeline, type_list, filtered,
414
+ fallback_ran: fallback_ran)
415
+ end
382
416
 
417
+ assembled = assemble_context(pipeline, filtered, classification, budget, **evidence_options)
383
418
  build_result(
384
- assembled: assembled, classification: classification, strategy: execution_result.strategy,
385
- budget: budget, trace: trace, type_rank_context: type_rank_context
386
- )
419
+ assembled: assembled, assembler: pipeline.assembler, classification: classification,
420
+ strategy: execution_result.strategy, budget: budget, type_rank_context: type_rank_context
421
+ ).tap do |result|
422
+ result.trace = build_trace(result, execution_result, filtered, assembled, start_time)
423
+ attach_scope(result, scope)
424
+ end
387
425
  end
388
426
 
389
427
  private
390
428
 
429
+ def resolve_scope(pipeline, packages, source_paths, types, excluded)
430
+ return unless Retrieval::Scope.requested?(packages: packages, source_paths: source_paths)
431
+
432
+ Retrieval::Scope.new(metadata_store: translate_store(pipeline.metadata_store, :metadata),
433
+ packages: packages, source_paths: source_paths, types: types,
434
+ exclude_types: DEFAULT_EXCLUDE_TYPES + Array(excluded).map(&:to_s))
435
+ end
436
+
437
+ def scoped_pipeline(pipeline, scope)
438
+ vector = Retrieval::ScopedVectorStore.new(store: pipeline.vector_store, scope: scope)
439
+ graph = Retrieval::ScopedGraphStore.new(store: pipeline.graph_store, scope: scope)
440
+ build_pipeline(vector_store: vector, metadata_store: scope.metadata_store, graph_store: graph)
441
+ end
442
+
443
+ def attach_scope(result, scope)
444
+ return result unless scope
445
+
446
+ result.applied_scope = scope.summary.merge(
447
+ outcome: if scope.keys.empty?
448
+ :empty_scope
449
+ else
450
+ (result.trace.ranked_count.zero? ? :no_match : :matched)
451
+ end,
452
+ candidate_count: result.trace.candidate_count, returned_units: result.sources.size
453
+ )
454
+ result
455
+ end
456
+
457
+ def retrieve_lexical(pipeline, query, classification, budget, types, exclude_types, start_time, **evidence_options)
458
+ excluded = DEFAULT_EXCLUDE_TYPES + Array(exclude_types).map(&:to_s)
459
+ execution = pipeline.executor.execute(query: query, type_filter: types, exclude_types: excluded)
460
+ assembled = pipeline.assembler.assemble(candidates: execution.candidates, budget: budget, **evidence_options)
461
+ result = build_result(assembled: assembled, assembler: pipeline.assembler, classification: classification,
462
+ strategy: :lexical, budget: budget)
463
+ result.trace = build_trace(result, execution, execution.candidates, assembled, start_time)
464
+ result
465
+ end
466
+
391
467
  # Validate +query+ before any classify/execute/rank work happens.
392
468
  #
393
469
  # @param query [Object] The caller-supplied query
@@ -464,23 +540,24 @@ module Woods
464
540
  # @param ranked [Array<Candidate>] Ranked search candidates
465
541
  # @param classification [QueryClassifier::Classification] Query classification
466
542
  # @return [AssembledContext]
467
- def assemble_context(pipeline, ranked, classification, budget)
543
+ def assemble_context(pipeline, ranked, classification, budget, **evidence_options)
468
544
  pipeline.assembler.assemble(
469
545
  candidates: ranked,
470
546
  classification: classification,
471
547
  structural_context: build_structural_context(pipeline.metadata_store),
472
- budget: budget
548
+ budget: budget, **evidence_options
473
549
  )
474
550
  end
475
551
 
476
552
  # Build a RetrievalResult from assembled context and pipeline metadata.
477
553
  #
478
554
  # @param assembled [AssembledContext] Assembled context
555
+ # @param assembler [Retrieval::ContextAssembler] Counter configuration used for assembly
479
556
  # @param classification [QueryClassifier::Classification] Query classification
480
557
  # @param strategy [Symbol] Search strategy used
481
558
  # @param budget [Integer] Token budget
482
559
  # @return [RetrievalResult]
483
- def build_result(assembled:, classification:, strategy:, budget:, trace: nil, type_rank_context: nil)
560
+ def build_result(assembled:, assembler:, classification:, strategy:, budget:, type_rank_context: nil)
484
561
  context = @formatter ? @formatter.call(assembled.context) : assembled.context
485
562
  context = append_type_rank_context(context, type_rank_context) if type_rank_context
486
563
 
@@ -489,9 +566,8 @@ module Woods
489
566
  sources: assembled.sources,
490
567
  classification: classification,
491
568
  strategy: strategy,
492
- tokens_used: assembled.tokens_used,
569
+ tokens_used: assembler.estimate_tokens(context),
493
570
  budget: budget,
494
- trace: trace,
495
571
  type_rank_context: type_rank_context
496
572
  )
497
573
  end
@@ -534,14 +610,14 @@ module Woods
534
610
  filter_by_type(pipeline, ranked, types: type_array, exclude_types: exclude_types)
535
611
  end
536
612
 
537
- def build_trace(classification, execution_result, filtered, assembled, start_time)
613
+ def build_trace(result, execution_result, filtered, assembled, start_time)
538
614
  elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(1)
539
615
  RetrievalTrace.new(
540
- classification: classification,
616
+ classification: result.classification,
541
617
  strategy: execution_result.strategy,
542
618
  candidate_count: execution_result.candidates.size,
543
619
  ranked_count: filtered.size,
544
- tokens_used: assembled.tokens_used,
620
+ tokens_used: result.tokens_used,
545
621
  elapsed_ms: elapsed_ms,
546
622
  skipped_missing_metadata: assembled.skipped_missing_metadata.to_i
547
623
  )
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'fiber'
4
+
3
5
  require_relative '../extracted_unit'
4
6
 
5
7
  module Woods
@@ -19,6 +21,10 @@ module Woods
19
21
  class TraceEnricher
20
22
  # Record method calls during block execution using TracePoint.
21
23
  #
24
+ # Records the current thread only, with independent stacks for its fibers.
25
+ # Caller fields identify the nearest observed Ruby method, not native or
26
+ # block frames. Calls entered before recording have an unknown caller.
27
+ #
22
28
  # @yield Block to trace
23
29
  # @return [Array<Hash>] Collected trace events
24
30
  # @raise [ArgumentError] if no block is given
@@ -26,21 +32,15 @@ module Woods
26
32
  raise ArgumentError, 'block required' unless block
27
33
 
28
34
  traces = []
29
-
35
+ stacks = Hash.new { |frames, fiber| frames[fiber] = [] }
30
36
  trace = TracePoint.new(:call, :return) do |tp|
31
- traces << {
32
- class_name: tp.defined_class&.name || tp.defined_class.to_s,
33
- method_name: tp.method_id.to_s,
34
- event: tp.event.to_s,
35
- path: tp.path,
36
- line: tp.lineno,
37
- caller_class: extract_caller_class(tp),
38
- caller_method: extract_caller_method(tp),
39
- return_class: tp.event == :return ? safe_return_class(tp) : nil
40
- }
37
+ fiber = Fiber.current
38
+ stack = stacks[fiber]
39
+ traces << record_event(tp, stack)
40
+ stacks.delete(fiber) if stack.empty?
41
41
  end
42
42
 
43
- trace.enable(&block)
43
+ trace.enable(target_thread: Thread.current, &block)
44
44
  traces
45
45
  end
46
46
 
@@ -55,14 +55,11 @@ module Woods
55
55
  def self.merge(units:, trace_data:)
56
56
  return units if trace_data.nil? || trace_data.empty?
57
57
 
58
- # Index traces by class_name + method_name
58
+ # Index by defining owner, method name, and instance/singleton kind.
59
59
  grouped = group_traces(trace_data)
60
60
 
61
61
  units.each do |unit|
62
- class_name, method_name = parse_identifier(unit.identifier)
63
- next unless class_name && method_name
64
-
65
- key = "#{class_name}##{method_name}"
62
+ key = parse_identifier(unit.identifier)
66
63
  next unless grouped.key?(key)
67
64
 
68
65
  traces = grouped[key]
@@ -75,7 +72,10 @@ module Woods
75
72
  caller_method = fetch_key(t, :caller_method)
76
73
  next unless caller_class
77
74
 
78
- { 'caller_class' => caller_class, 'caller_method' => caller_method }
75
+ caller = { 'caller_class' => caller_class, 'caller_method' => caller_method }
76
+ kind = fetch_key(t, :caller_method_kind)
77
+ caller['caller_method_kind'] = kind.to_s if kind
78
+ caller
79
79
  end
80
80
 
81
81
  return_types = returns.filter_map do |t|
@@ -104,35 +104,74 @@ module Woods
104
104
  method_name = fetch_key(trace, :method_name)
105
105
  next unless class_name && method_name
106
106
 
107
- key = "#{class_name}##{method_name}"
108
- grouped[key] << trace
107
+ # Legacy recorder output with a named owner describes instance
108
+ # methods. Never infer its kind from the units supplied to merge.
109
+ kind = (fetch_key(trace, :method_kind) || 'instance').to_s
110
+ next unless %w[instance singleton].include?(kind)
111
+ next if class_name.start_with?('#<')
112
+
113
+ grouped[[class_name, method_name, kind]] << trace
109
114
  end
110
115
  grouped
111
116
  end
112
117
 
113
118
  def parse_identifier(identifier)
114
- # Handle both "Class#method" and "Class.method" formats
115
- if identifier.include?('#')
116
- identifier.split('#', 2)
117
- elsif identifier.include?('.')
118
- identifier.split('.', 2)
119
- end
119
+ match = /\A(.+?)([#.])(.+)\z/.match(identifier)
120
+ return unless match
121
+
122
+ [match[1], match[3], match[2] == '#' ? 'instance' : 'singleton']
120
123
  end
121
124
 
122
- def extract_caller_class(tp)
123
- binding_obj = tp.binding
124
- receiver = binding_obj.receiver
125
- receiver.is_a?(Class) || receiver.is_a?(Module) ? receiver.name : receiver.class.name
126
- rescue StandardError
127
- nil
125
+ def event_identity(tp)
126
+ owner = tp.defined_class
127
+ singleton = owner&.singleton_class?
128
+ name = singleton ? singleton_owner_name(owner, tp.self) : owner&.name
129
+ { class_name: name, method_name: tp.method_id.to_s,
130
+ method_kind: singleton ? 'singleton' : 'instance' }
128
131
  end
129
132
 
130
- def extract_caller_method(_tp)
131
- # TracePoint doesn't directly expose caller method,
132
- # but we can get it from the call stack
133
- caller_locations(3, 1)&.first&.label
134
- rescue StandardError
135
- nil
133
+ # Ruby 3.0 has no Class#attached_object. Find the defining owner,
134
+ # not just the receiver: Child.run may be defined on Parent's singleton
135
+ # class. Singleton methods on individual objects have no named unit.
136
+ def singleton_owner_name(owner, receiver)
137
+ return unless receiver.is_a?(Module)
138
+
139
+ receiver.ancestors.find { |ancestor| ancestor.singleton_class.equal?(owner) }&.name
140
+ end
141
+
142
+ def record_event(tp, stack)
143
+ identity = event_identity(tp)
144
+ caller = if tp.event == :call
145
+ caller_fields(stack.last)
146
+ else
147
+ returning_caller(identity, stack)
148
+ end
149
+ event = identity.merge(
150
+ event: tp.event.to_s, path: tp.path, line: tp.lineno,
151
+ **caller, return_class: tp.event == :return ? safe_return_class(tp) : nil
152
+ )
153
+ stack << event if tp.event == :call
154
+ event
155
+ end
156
+
157
+ def caller_fields(frame)
158
+ frame = nil unless frame && frame[:class_name]
159
+ { caller_class: frame && frame[:class_name], caller_method: frame && frame[:method_name],
160
+ caller_method_kind: frame && frame[:method_kind] }
161
+ end
162
+
163
+ # Ruby emits :return during exceptional and nonlocal unwinds too. A
164
+ # return whose call predates recording has no known caller; discard an
165
+ # inconsistent stack instead of inventing an edge from unrelated frames.
166
+ def returning_caller(identity, stack)
167
+ frame = stack.pop
168
+ if frame && identity.all? { |key, value| frame[key] == value }
169
+ { caller_class: frame[:caller_class], caller_method: frame[:caller_method],
170
+ caller_method_kind: frame[:caller_method_kind] }
171
+ else
172
+ stack.clear
173
+ caller_fields(nil)
174
+ end
136
175
  end
137
176
 
138
177
  def safe_return_class(tp)
@@ -116,7 +116,8 @@ module Woods
116
116
  def clear(session_id)
117
117
  with_store_lock do
118
118
  FileUtils.rm_f(session_path(session_id))
119
- FileUtils.rm_f(legacy_session_path(session_id))
119
+ legacy = legacy_session_path(session_id)
120
+ FileUtils.rm_f(legacy) if legacy
120
121
  end
121
122
  end
122
123
 
@@ -158,6 +159,10 @@ module Woods
158
159
  def migrate_legacy_session!(session_id)
159
160
  target = session_path(session_id)
160
161
  legacy = legacy_session_path(session_id)
162
+ # Expire each half before a merge or append can refresh its mtime.
163
+ [target, legacy].compact.each do |path|
164
+ FileUtils.rm_f(path) if File.exist?(path) && expired?(path)
165
+ end
161
166
  return target unless legacy && File.exist?(legacy)
162
167
 
163
168
  if File.exist?(target)