woods 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +108 -0
  3. data/CONTRIBUTING.md +135 -10
  4. data/README.md +1 -1
  5. data/docs/AGENT_GUIDE.md +19 -0
  6. data/docs/AGENT_SETUP.md +22 -2
  7. data/docs/BACKEND_MATRIX.md +7 -0
  8. data/docs/CLIENT_HOOKS.md +6 -0
  9. data/docs/CONFIGURATION_REFERENCE.md +133 -18
  10. data/docs/CONSOLE_MCP_SETUP.md +141 -12
  11. data/docs/EMBEDDING_MODELS.md +16 -19
  12. data/docs/EXTRACTOR_REFERENCE.md +219 -21
  13. data/docs/FAQ.md +11 -25
  14. data/docs/GETTING_STARTED.md +7 -1
  15. data/docs/INCREMENTAL_EXTRACTION.md +261 -19
  16. data/docs/INDEX_LAYOUT.md +5 -0
  17. data/docs/INTERNALS.md +9 -0
  18. data/docs/MCP_HTTP_TRANSPORT.md +59 -2
  19. data/docs/MCP_SERVERS.md +87 -8
  20. data/docs/MCP_TOOL_COOKBOOK.md +13 -55
  21. data/docs/NOTION_INTEGRATION.md +7 -1
  22. data/docs/PUBLISHED_INDEX.md +6 -0
  23. data/docs/README.md +6 -1
  24. data/docs/RETRIEVAL_GUIDE.md +17 -0
  25. data/docs/SOURCE_FRESHNESS.md +157 -5
  26. data/docs/TOKEN_BENCHMARK.md +10 -18
  27. data/docs/TROUBLESHOOTING.md +70 -14
  28. data/docs/UNBLOCKED_INTEGRATION.md +60 -8
  29. data/docs/UPGRADING_TO_2.md +184 -9
  30. data/docs/WATCH_DAEMON.md +97 -14
  31. data/exe/woods-console-mcp +2 -2
  32. data/exe/woods-mcp-http +16 -9
  33. data/lib/generators/woods/templates/woods.rb.tt +2 -1
  34. data/lib/tasks/woods.rake +23 -7
  35. data/lib/tasks/woods_checks.rake +2 -2
  36. data/lib/woods/agent_configuration/cli.rb +1 -1
  37. data/lib/woods/agent_configuration/layout.rb +16 -2
  38. data/lib/woods/agent_configuration/plan.rb +13 -3
  39. data/lib/woods/agent_configuration/planner_validation.rb +4 -2
  40. data/lib/woods/agent_configuration/preflight.rb +5 -3
  41. data/lib/woods/builder.rb +17 -57
  42. data/lib/woods/cache/cache_middleware.rb +68 -41
  43. data/lib/woods/chunking/contributor_chunks.rb +119 -0
  44. data/lib/woods/chunking/semantic_chunker.rb +44 -21
  45. data/lib/woods/console/adapter_family.rb +39 -0
  46. data/lib/woods/console/connection_manager.rb +56 -3
  47. data/lib/woods/console/credential_index.rb +33 -3
  48. data/lib/woods/console/embedded_executor.rb +431 -48
  49. data/lib/woods/console/model_validator.rb +8 -0
  50. data/lib/woods/console/rack_middleware.rb +68 -11
  51. data/lib/woods/console/redactor.rb +24 -10
  52. data/lib/woods/console/safe_context.rb +44 -7
  53. data/lib/woods/console/sql_noise_stripper.rb +41 -12
  54. data/lib/woods/console/sql_table_scanner.rb +45 -34
  55. data/lib/woods/console/sql_validator.rb +37 -2
  56. data/lib/woods/dependency_graph.rb +34 -10
  57. data/lib/woods/embedding/fake.rb +12 -0
  58. data/lib/woods/embedding/indexer.rb +195 -98
  59. data/lib/woods/embedding/input_budget.rb +67 -0
  60. data/lib/woods/embedding/openai.rb +70 -20
  61. data/lib/woods/embedding/provider.rb +37 -25
  62. data/lib/woods/embedding/text_preparer.rb +76 -32
  63. data/lib/woods/embedding/token_counter.rb +18 -81
  64. data/lib/woods/embedding/vector_configuration.rb +48 -0
  65. data/lib/woods/extraction_identities.rb +175 -0
  66. data/lib/woods/extractor.rb +304 -107
  67. data/lib/woods/extractors/action_cable_extractor.rb +8 -3
  68. data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
  69. data/lib/woods/extractors/class_declarations.rb +121 -0
  70. data/lib/woods/extractors/configuration_extractor.rb +11 -3
  71. data/lib/woods/extractors/declaration_ancestry.rb +92 -0
  72. data/lib/woods/extractors/event_extractor.rb +8 -0
  73. data/lib/woods/extractors/graphql_extractor.rb +134 -77
  74. data/lib/woods/extractors/job_extractor.rb +5 -1
  75. data/lib/woods/extractors/lib_extractor.rb +132 -15
  76. data/lib/woods/extractors/mailer_extractor.rb +3 -5
  77. data/lib/woods/extractors/manager_extractor.rb +7 -21
  78. data/lib/woods/extractors/migration_declaration.rb +87 -0
  79. data/lib/woods/extractors/migration_extractor.rb +5 -39
  80. data/lib/woods/extractors/phlex_extractor.rb +6 -2
  81. data/lib/woods/extractors/policy_extractor.rb +9 -5
  82. data/lib/woods/extractors/poro_extractor.rb +112 -53
  83. data/lib/woods/extractors/pundit_extractor.rb +11 -6
  84. data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
  85. data/lib/woods/extractors/serializer_extractor.rb +34 -22
  86. data/lib/woods/extractors/shared_utility_methods.rb +18 -1
  87. data/lib/woods/extractors/source_nesting.rb +142 -106
  88. data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
  89. data/lib/woods/extractors/state_machine_extractor.rb +46 -40
  90. data/lib/woods/extractors/view_component_extractor.rb +9 -7
  91. data/lib/woods/flow_assembler.rb +4 -1
  92. data/lib/woods/generation.rb +25 -0
  93. data/lib/woods/hooks/context_hint.rb +7 -2
  94. data/lib/woods/mcp/bearer_auth.rb +1 -1
  95. data/lib/woods/mcp/bootstrapper.rb +33 -7
  96. data/lib/woods/mcp/config_resolver.rb +26 -7
  97. data/lib/woods/mcp/index_reader.rb +125 -24
  98. data/lib/woods/mcp/index_reader_pinning.rb +16 -0
  99. data/lib/woods/mcp/origin_guard.rb +24 -77
  100. data/lib/woods/mcp/origin_policy.rb +124 -0
  101. data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
  102. data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
  103. data/lib/woods/mcp/search_results.rb +7 -1
  104. data/lib/woods/mcp/server.rb +24 -4
  105. data/lib/woods/module_reconciliation.rb +151 -0
  106. data/lib/woods/path_dispatcher.rb +7 -2
  107. data/lib/woods/railtie_support.rb +8 -0
  108. data/lib/woods/rake_helpers.rb +43 -11
  109. data/lib/woods/release.rb +1 -1
  110. data/lib/woods/resilience/index_validator.rb +8 -3
  111. data/lib/woods/resilience/retryable_provider.rb +18 -1
  112. data/lib/woods/resolved_config.rb +68 -8
  113. data/lib/woods/retrieval/context_assembler.rb +3 -3
  114. data/lib/woods/retrieval/lexical_assembler.rb +3 -2
  115. data/lib/woods/retrieval/scope.rb +18 -2
  116. data/lib/woods/retrieval/source_evidence.rb +14 -2
  117. data/lib/woods/source_contributor_validation.rb +78 -0
  118. data/lib/woods/source_contributors.rb +116 -0
  119. data/lib/woods/source_inputs/handoff.rb +37 -0
  120. data/lib/woods/source_inputs/launcher.rb +53 -13
  121. data/lib/woods/source_inputs/manifest.rb +84 -3
  122. data/lib/woods/source_inputs/private_key.rb +44 -12
  123. data/lib/woods/source_inputs/scanner.rb +98 -27
  124. data/lib/woods/source_inputs/scopes.rb +1 -1
  125. data/lib/woods/source_inputs/session.rb +147 -15
  126. data/lib/woods/source_inputs/stable_reader.rb +127 -0
  127. data/lib/woods/source_inputs/status.rb +40 -8
  128. data/lib/woods/source_inputs/verifier.rb +28 -5
  129. data/lib/woods/source_path_encoding.rb +33 -0
  130. data/lib/woods/source_references/cache.rb +284 -0
  131. data/lib/woods/source_references/collector.rb +120 -0
  132. data/lib/woods/source_references/extraction.rb +185 -0
  133. data/lib/woods/source_references/inputs.rb +134 -0
  134. data/lib/woods/source_references/parser_adapter.rb +134 -0
  135. data/lib/woods/source_references/pass.rb +152 -0
  136. data/lib/woods/source_references/prism_adapter.rb +116 -0
  137. data/lib/woods/source_references/registry.rb +178 -0
  138. data/lib/woods/source_references/runtime_lookup.rb +127 -0
  139. data/lib/woods/source_references/value_class.rb +82 -0
  140. data/lib/woods/storage/metadata_store.rb +4 -1
  141. data/lib/woods/storage/qdrant.rb +2 -2
  142. data/lib/woods/unblocked/client.rb +12 -7
  143. data/lib/woods/unblocked/document_builder.rb +4 -1
  144. data/lib/woods/unblocked/exporter.rb +127 -37
  145. data/lib/woods/unblocked/sync_manifest.rb +137 -21
  146. data/lib/woods/unblocked/uri_migration.rb +105 -0
  147. data/lib/woods/util/host_guard.rb +3 -2
  148. data/lib/woods/version.rb +1 -1
  149. data/lib/woods/watch/catch_up.rb +138 -0
  150. data/lib/woods/watch/claim_lease.rb +150 -0
  151. data/lib/woods/watch/cli.rb +26 -2
  152. data/lib/woods/watch/daemon.rb +80 -59
  153. data/lib/woods/watch/installation/options.rb +1 -1
  154. data/lib/woods/watch/installation/receipt.rb +6 -1
  155. data/lib/woods/watch/managed_child.rb +1 -1
  156. data/lib/woods/watch/supervisor.rb +1 -1
  157. data/lib/woods/watch/tree_scan.rb +14 -2
  158. data/plugin/.claude-plugin/plugin.json +1 -1
  159. data/plugin/hooks/adapters/normalize.rb +3 -2
  160. data/plugin/hooks/woods-input-rules.sh +4 -0
  161. data/plugin/hooks/woods-refresh.sh +15 -7
  162. data/plugin/hooks/woods-session-start.sh +60 -3
  163. data/plugin/skills/woods-diagnose/SKILL.md +336 -2
  164. data/plugin/skills/woods-investigate/SKILL.md +11 -0
  165. data/plugin/skills/woods-mcp-config/SKILL.md +86 -0
  166. data/plugin/skills/woods-setup/SKILL.md +56 -1
  167. metadata +34 -5
@@ -23,7 +23,7 @@ module Woods
23
23
  sources = []
24
24
  candidates.each do |candidate|
25
25
  unit = candidate.metadata
26
- header = "\n\n## #{unit['identifier']} (#{unit['type']})\nFile: #{unit['file_path']}\n" \
26
+ header = "\n\n## #{unit['identifier']} (#{unit['type']})\n#{SourceContributors.label(unit)}\n" \
27
27
  "Matched: #{candidate.matched_fields.join(', ')}\n\n"
28
28
  remaining = (budget * 4) - context.length - header.length
29
29
  next unless remaining.positive?
@@ -37,7 +37,7 @@ module Woods
37
37
  context += header + selected.text
38
38
  sources << { identifier: unit['identifier'], type: unit['type'], file_path: unit['file_path'],
39
39
  score: candidate.score, matched_fields: candidate.matched_fields,
40
- evidence: selected.provenance }
40
+ evidence: selected.provenance }.merge(SourceContributors.attribution(unit))
41
41
  next
42
42
  end
43
43
 
@@ -49,6 +49,7 @@ module Woods
49
49
  context += header + (truncated ? source[0, remaining - marker.length] + marker : source)
50
50
  sources << { identifier: unit['identifier'], type: unit['type'], file_path: unit['file_path'],
51
51
  score: candidate.score, matched_fields: candidate.matched_fields, truncated: truncated }
52
+ .merge(SourceContributors.attribution(unit))
52
53
  break if truncated
53
54
  end
54
55
  body = context[notice.length..].to_s
@@ -3,6 +3,7 @@
3
3
  require 'json'
4
4
  require 'set'
5
5
  require_relative '../storage_identity'
6
+ require_relative '../source_contributors'
6
7
  require_relative '../storage/metadata_store'
7
8
 
8
9
  module Woods
@@ -83,13 +84,28 @@ module Woods
83
84
  def validate_packages!(records)
84
85
  names = records.filter_map { |unit| unit['identifier'] if unit['type'] == 'package' }
85
86
  names.concat(records.filter_map { |unit| unit.dig('metadata', 'package') })
87
+ names.concat(records.flat_map do |unit|
88
+ SourceContributors.records(unit).filter_map do |record|
89
+ record['package']
90
+ end
91
+ end)
86
92
  unknown = packages - names
87
93
  raise InvalidScopeError, "unknown package scope: #{unknown.join(', ')}" unless unknown.empty?
88
94
  end
89
95
 
90
96
  def eligible?(unit, types, excluded)
91
- return false unless packages.empty? || packages.include?(unit.dig('metadata', 'package'))
92
- return false unless source_paths.empty? || path_match?(unit['file_path'])
97
+ contributors = SourceContributors.records(unit)
98
+ owners = if contributors.empty?
99
+ [unit.dig('metadata', 'package')]
100
+ else
101
+ contributors.map do |record|
102
+ record['package']
103
+ end
104
+ end
105
+ return false unless packages.empty? || owners.all? { |owner| packages.include?(owner) }
106
+
107
+ paths = SourceContributors.paths(unit)
108
+ return false unless source_paths.empty? || (paths.any? && paths.all? { |path| path_match?(path) })
93
109
  return Array(types).map(&:to_s).include?(unit['type']) if types && !types.empty?
94
110
 
95
111
  !Array(excluded).map(&:to_s).include?(unit['type'])
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../source_contributors'
4
+
3
5
  require 'digest'
4
6
  require 'json'
5
7
  require_relative '../ast/parser'
@@ -170,7 +172,7 @@ module Woods
170
172
 
171
173
  def format_evidence(mode, selected, metadata, total)
172
174
  parts = ["Evidence: #{mode}; published-unit coordinates (physical location unavailable).",
173
- "Unit: #{field(:type)}:#{field(:identifier)}; path: #{field(:file_path)}",
175
+ "Unit: #{field(:type)}:#{field(:identifier)}; #{SourceContributors.label(@unit)}",
174
176
  "Source SHA256: #{Digest::SHA256.hexdigest(@source)}",
175
177
  "Generation: #{@generation || 'unavailable (not recorded by this metadata store)'}"]
176
178
  selected.each do |span|
@@ -183,17 +185,27 @@ module Woods
183
185
  parts.join("\n\n")
184
186
  end
185
187
 
188
+ def span_provenance(span)
189
+ result = span.to_h.merge(sha256: Digest::SHA256.hexdigest(span_text(span)))
190
+ if SourceContributors.multiple?(@unit)
191
+ result[:physical_location] =
192
+ SourceContributors.physical_span(@unit, start_byte: span.start_byte, end_byte: span.end_byte)
193
+ end
194
+ result
195
+ end
196
+
186
197
  def provenance(mode, selected, metadata, all)
187
198
  { mode: mode, owner: { identifier: field(:identifier), type: field(:type) },
188
199
  coordinate_system: 'published_unit', source_sha256: Digest::SHA256.hexdigest(@source),
189
200
  source_path: field(:file_path), physical_location: nil,
190
201
  physical_location_reason: 'published unit may contain synthesized or inlined source',
191
202
  generation: @generation, generation_status: @generation ? 'recorded' : 'unavailable',
192
- spans: selected.map { |span| span.to_h.merge(sha256: Digest::SHA256.hexdigest(span_text(span))) },
203
+ spans: selected.map { |span| span_provenance(span) },
193
204
  omitted_spans: all.size - selected.size, runtime_fields: metadata.map(&:first),
194
205
  source_complete: false, full_evidence: { tool: 'lookup', evidence: 'full',
195
206
  identifier: field(:identifier), type: field(:type),
196
207
  source_sha256: Digest::SHA256.hexdigest(@source) } }
208
+ .merge(SourceContributors.attribution(@unit))
197
209
  end
198
210
  end
199
211
  end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+
5
+ module Woods
6
+ # Checks the versioned contributor boundary before consumers use its paths or coordinates.
7
+ module SourceContributorValidation
8
+ class Invalid < ArgumentError; end
9
+
10
+ RANGE_KEYS = %w[source_start_line source_end_line published_start_byte published_end_byte
11
+ published_start_line published_end_line].freeze
12
+
13
+ module_function
14
+
15
+ def validate!(records, type:, version:, source:)
16
+ unless type.to_s == 'lib' && version == 1 && records.is_a?(Array) && records.size.between?(2, 100_000)
17
+ raise Invalid, 'Invalid library contributor version or records'
18
+ end
19
+
20
+ validate_records!(records)
21
+ records.each { |record| validate_source!(source, record) } if source.is_a?(String)
22
+ records
23
+ end
24
+
25
+ def validate_records!(records)
26
+ previous = 0
27
+ paths = records.map do |record|
28
+ unless valid_record?(record) && valid_ranges?(record, previous)
29
+ raise Invalid, 'Invalid library contributor path or range'
30
+ end
31
+
32
+ previous = record.fetch('published_end_byte')
33
+ record.fetch('file_path')
34
+ end
35
+ raise Invalid, 'Unordered or duplicate library contributor path' unless paths.uniq.sort == paths
36
+ end
37
+
38
+ def valid_record?(record)
39
+ record.is_a?(Hash) && record.keys.all?(String) && valid_path?(record['file_path']) &&
40
+ valid_hash?(record['source_sha256']) &&
41
+ record['facts'].is_a?(Hash) && RANGE_KEYS.all? { |key| record[key].is_a?(Integer) }
42
+ end
43
+
44
+ def valid_hash?(hash)
45
+ hash.is_a?(String) && hash.match?(/\A[0-9a-f]{64}\z/)
46
+ end
47
+
48
+ def valid_ranges?(record, previous)
49
+ record['source_start_line'] == 1 && record['source_end_line'].positive? &&
50
+ record['published_start_byte'] >= previous && record['published_end_byte'] > record['published_start_byte'] &&
51
+ record['published_start_line'].positive? && record['published_end_line'] >= record['published_start_line']
52
+ end
53
+
54
+ def valid_path?(path)
55
+ path.is_a?(String) && path.start_with?('lib/') && path.end_with?('.rb') && !path.include?("\0") &&
56
+ !path.include?('\\') && path.split('/', -1).none? { |part| ['', '.', '..'].include?(part) }
57
+ end
58
+
59
+ def validate_source!(source, record)
60
+ raise Invalid, 'Contributor range exceeds published source' if record['published_end_byte'] > source.bytesize
61
+
62
+ fragment = source.byteslice(record['published_start_byte']...record['published_end_byte'])
63
+ first = source.byteslice(0...record['published_start_byte']).count("\n") + 1
64
+ return if valid_fragment?(fragment, first, record)
65
+
66
+ raise Invalid, 'Library contributor bytes do not match published provenance'
67
+ end
68
+
69
+ def valid_fragment?(fragment, first, record)
70
+ Digest::SHA256.hexdigest(fragment) == record['source_sha256'] &&
71
+ fragment.lines.size == record['source_end_line'] && first == record['published_start_line'] &&
72
+ first + fragment.lines.size - 1 == record['published_end_line']
73
+ end
74
+ private_class_method :validate_records!, :valid_record?, :valid_hash?, :valid_ranges?, :valid_path?,
75
+ :validate_source!,
76
+ :valid_fragment?
77
+ end
78
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'source_contributor_validation'
4
+
5
+ module Woods
6
+ # Provenance for a published unit assembled from several original files.
7
+ # Byte ranges are half-open and exclude every generated header/separator.
8
+ module SourceContributors
9
+ Invalid = SourceContributorValidation::Invalid
10
+
11
+ VERSION = 1
12
+
13
+ module_function
14
+
15
+ # @param unit [ExtractedUnit, Hash]
16
+ # @return [Array<Hash>] validated, string-keyed contributor records
17
+ def records(unit)
18
+ metadata = field(unit, :metadata) || {}
19
+ raw = field(metadata, :source_contributors)
20
+ return [] if raw.nil?
21
+
22
+ SourceContributorValidation.validate!(raw, type: field(unit, :type),
23
+ version: field(metadata, :source_contributors_version),
24
+ source: field(unit, :source_code))
25
+ end
26
+
27
+ # @param unit [ExtractedUnit, Hash]
28
+ # @return [Array<String>] every physical source, or the legacy primary path
29
+ def paths(unit)
30
+ contributors = records(unit)
31
+ contributors.empty? ? Array(field(unit, :file_path)) : contributors.map { |record| record.fetch('file_path') }
32
+ end
33
+
34
+ # @param unit [ExtractedUnit, Hash]
35
+ # @return [Boolean]
36
+ def multiple?(unit)
37
+ !records(unit).empty?
38
+ end
39
+
40
+ # @param unit [ExtractedUnit, Hash]
41
+ # @param start_byte [Integer] inclusive published byte offset
42
+ # @param end_byte [Integer] exclusive published byte offset
43
+ # @return [Hash, nil] physical coordinates only for an exact original fragment
44
+ def physical_span(unit, start_byte:, end_byte:)
45
+ return unless valid_span?(start_byte, end_byte)
46
+
47
+ record = records(unit).find do |fragment|
48
+ start_byte >= fragment['published_start_byte'] && end_byte <= fragment['published_end_byte']
49
+ end
50
+ return unless record
51
+
52
+ source = field(unit, :source_code)
53
+ return unless source.is_a?(String)
54
+
55
+ first = record['source_start_line'] + source.byteslice(record['published_start_byte']...start_byte).count("\n")
56
+ { file_path: record['file_path'], source_sha256: record['source_sha256'], start_line: first,
57
+ end_line: first + source.byteslice(start_byte...end_byte).delete_suffix("\n").count("\n") }
58
+ end
59
+
60
+ def valid_span?(first, last)
61
+ first.is_a?(Integer) && last.is_a?(Integer) && last > first
62
+ end
63
+ private_class_method :valid_span?
64
+
65
+ # @param unit [ExtractedUnit, Hash]
66
+ # @return [String] explicit primary/contributor label, or legacy file label
67
+ def label(unit)
68
+ return "File: #{field(unit, :file_path)}" unless multiple?(unit)
69
+
70
+ "Primary file: #{field(unit, :file_path)}\nContributing files: #{paths(unit).join(', ')}"
71
+ end
72
+
73
+ # @param unit [ExtractedUnit, Hash]
74
+ # @return [Hash] additive contributor evidence for response renderers
75
+ def attribution(unit)
76
+ multiple?(unit) ? { source_contributors: records(unit) } : {}
77
+ end
78
+
79
+ # @param unit [ExtractedUnit, Hash]
80
+ # @param resolver [Object] package_for(path) collaborator
81
+ # @return [String, nil] unanimous package ownership, or nil
82
+ def annotate_package(unit, resolver)
83
+ contributors = records(unit)
84
+ return resolver.package_for(field(unit, :file_path)) if contributors.empty?
85
+
86
+ packages = contributors.map do |record|
87
+ package = resolver.package_for(record['file_path'])
88
+ package ? record['package'] = package : record.delete('package')
89
+ package
90
+ end
91
+ packages.first if packages.uniq.one?
92
+ end
93
+
94
+ # @param unit [ExtractedUnit, Hash]
95
+ # @param data [Hash] Git facts keyed by application-relative source path
96
+ # @return [void]
97
+ def annotate_git(unit, data)
98
+ records(unit).each do |record|
99
+ record.delete('git')
100
+ record['git'] = data[record['file_path']] if data[record['file_path']]
101
+ end
102
+ metadata = field(unit, :metadata)
103
+ metadata.delete(:git)
104
+ metadata.delete('git')
105
+ end
106
+
107
+ # @param value [Object, Hash]
108
+ # @param key [Symbol]
109
+ # @return [Object]
110
+ def field(value, key)
111
+ return value[key] || value[key.to_s] if value.is_a?(Hash)
112
+
113
+ value.public_send(key) if value.respond_to?(key)
114
+ end
115
+ end
116
+ end
@@ -9,9 +9,35 @@ module Woods
9
9
  # existing index manifest nor a user-supplied proof of runtime freshness.
10
10
  module Handoff
11
11
  ENV_KEY = 'WOODS_SOURCE_CAPTURE'
12
+ class OutputMismatch < StandardError; end
12
13
 
13
14
  module_function
14
15
 
16
+ # An implicit preboot default cannot override finalized application config.
17
+ # Explicit launcher outputs remain intentional overrides. Called before
18
+ # the writer lock/extraction, while the one-use handoff is still present.
19
+ # @param root [String, Pathname] finalized application root
20
+ # @param output_dir [String, Pathname] selected writer output
21
+ # @return [void]
22
+ def validate_output!(root:, output_dir:)
23
+ token = ENV.fetch(ENV_KEY, nil)
24
+ return if token.to_s.empty?
25
+
26
+ descriptor = JSON.parse(token)
27
+ return unless descriptor.is_a?(Hash) && descriptor['implicit_output'] == true
28
+
29
+ expected = File.expand_path('tmp/woods', root.to_s)
30
+ configured = File.expand_path(Woods.configuration.output_dir.to_s, root.to_s)
31
+ return if configured == expected && File.expand_path(output_dir.to_s, root.to_s) == expected
32
+
33
+ raise OutputMismatch,
34
+ "woods-extract configured output #{configured.inspect} differs from " \
35
+ "its preboot default #{expected.inspect}; " \
36
+ 'rerun with matching --output PATH or WOODS_OUTPUT so capture and publication use the same index'
37
+ rescue JSON::ParserError, TypeError
38
+ nil # Ordinary handoff validation will refuse unverifiable capture.
39
+ end
40
+
15
41
  # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
16
42
  # Every independent handoff binding must match before a token is consumed.
17
43
  def read(root:, output_dir:, operation:, rules:, key_id:)
@@ -85,6 +111,17 @@ module Woods
85
111
 
86
112
  # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
87
113
 
114
+ # An oversized launcher capture conveys only a downgrade, never proof.
115
+ def unavailable_evidence
116
+ descriptor = JSON.parse(ENV.fetch(ENV_KEY, '{}'))
117
+ value = descriptor.is_a?(Hash) && descriptor['unavailable']
118
+ return unless Manifest.valid_unavailable_evidence?(value)
119
+
120
+ value.slice('reason', 'size_bytes', 'limit_bytes')
121
+ rescue JSON::ParserError
122
+ nil
123
+ end
124
+
88
125
  def extra_roots
89
126
  token = ENV.fetch(ENV_KEY, nil)
90
127
  return [] if token.nil? || token.empty?
@@ -17,22 +17,27 @@ module Woods
17
17
 
18
18
  def self.run(argv, **options)
19
19
  new(argv, **options).run
20
- rescue OptionParser::ParseError, ArgumentError, PrivateKey::Unavailable, SystemCallError => e
20
+ rescue PrivateKey::Unavailable => e
21
+ warn "woods-extract: #{e.diagnostic}"
22
+ 1
23
+ rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
21
24
  warn "woods-extract: #{e.message}"
22
25
  1
23
26
  end
24
27
 
25
- def initialize(argv, command: %w[bundle exec rake])
28
+ def initialize(argv, command: nil)
26
29
  @arguments = argv.dup
27
30
  @command = command
28
31
  @original_directory = Dir.pwd
29
32
  @root = @original_directory
33
+ @implicit_output = !ENV.key?('WOODS_OUTPUT')
30
34
  @output = ENV.fetch('WOODS_OUTPUT', 'tmp/woods')
31
35
  @extra_roots = []
32
36
  parse!
37
+ @command ||= default_command
33
38
  end
34
39
 
35
- def run # rubocop:disable Metrics/MethodLength -- handoff lifetime encloses the fresh child
40
+ def run
36
41
  return 0 if @help
37
42
 
38
43
  operation, task = task_for_arguments
@@ -40,25 +45,57 @@ module Woods
40
45
  rules = Scopes.new(extra_roots: @extra_roots)
41
46
  snapshot = Scanner.new(root: @root, output_dir: @output, key: key, scopes: rules).call
42
47
  nonce = SecureRandom.hex(32)
48
+ bytes = capture_bytes(snapshot, nonce, operation, rules)
49
+ return run_without_capture(task) unless bytes
50
+
43
51
  Tempfile.create(['woods-source-capture-', '.json']) do |file|
44
52
  file.binmode
45
53
  file.chmod(0o600)
46
- file.write(JSON.generate(version: 1, nonce: nonce, root: @root, output: @output,
47
- operation: operation, rules: rules.fingerprint,
48
- launcher_pid: Process.pid, snapshot: snapshot))
54
+ file.write(bytes)
49
55
  file.flush
50
56
  file.fsync
51
- descriptor = JSON.generate(path: file.path, nonce: nonce, extra_roots: @extra_roots)
52
- environment = { Handoff::ENV_KEY => descriptor, 'WOODS_OUTPUT' => @output }
53
- if ENV['BUNDLE_GEMFILE']
54
- environment['BUNDLE_GEMFILE'] = File.expand_path(ENV.fetch('BUNDLE_GEMFILE'), @original_directory)
55
- end
56
- run_child(environment, task)
57
+ descriptor = JSON.generate(path: file.path, nonce: nonce, extra_roots: @extra_roots,
58
+ implicit_output: @implicit_output)
59
+ run_child(child_environment(descriptor), task)
57
60
  end
58
61
  end
59
62
 
60
63
  private
61
64
 
65
+ def default_command
66
+ binstub = File.join(@root, 'bin/rake')
67
+ File.file?(binstub) && File.executable?(binstub) ? [binstub] : %w[bundle exec rake]
68
+ end
69
+
70
+ def run_without_capture(task)
71
+ descriptor = JSON.generate(extra_roots: @extra_roots, implicit_output: @implicit_output,
72
+ unavailable: @capture_unavailable)
73
+ run_child(child_environment(descriptor), task)
74
+ end
75
+
76
+ def capture_bytes(snapshot, nonce, operation, rules)
77
+ bytes = JSON.generate(version: 1, nonce: nonce, root: @root, output: @output,
78
+ operation: operation, rules: rules.fingerprint,
79
+ launcher_pid: Process.pid, snapshot: snapshot)
80
+ return bytes if bytes.bytesize <= Manifest::MAX_BYTES
81
+
82
+ @capture_unavailable = { 'reason' => 'source_manifest_too_large', 'size_bytes' => bytes.bytesize,
83
+ 'limit_bytes' => Manifest::MAX_BYTES }
84
+ warn 'woods-extract: source freshness unavailable (source_manifest_too_large): ' \
85
+ "capture #{bytes.bytesize} bytes exceeds limit #{Manifest::MAX_BYTES}; " \
86
+ 'continuing without verified preboot evidence.'
87
+ nil
88
+ end
89
+
90
+ def child_environment(descriptor)
91
+ environment = { Handoff::ENV_KEY => descriptor }
92
+ environment['WOODS_OUTPUT'] = @output unless @implicit_output
93
+ if ENV['BUNDLE_GEMFILE']
94
+ environment['BUNDLE_GEMFILE'] = File.expand_path(ENV.fetch('BUNDLE_GEMFILE'), @original_directory)
95
+ end
96
+ environment
97
+ end
98
+
62
99
  def parse!
63
100
  parser.permute!(@arguments)
64
101
  validate_arguments! unless @help
@@ -68,7 +105,10 @@ module Woods
68
105
  OptionParser.new do |options|
69
106
  options.banner = 'Usage: woods-extract [options] full | incremental PATH... | refresh TYPE...'
70
107
  options.on('--root PATH', 'Application root (default: current directory)') { |path| @root = path }
71
- options.on('--output PATH', 'Index output (default: WOODS_OUTPUT or tmp/woods)') { |path| @output = path }
108
+ options.on('--output PATH', 'Custom output (default: WOODS_OUTPUT or tmp/woods)') do |path|
109
+ @output = path
110
+ @implicit_output = false
111
+ end
72
112
  options.on('--source-root PATH', 'Additional application-relative runtime source root') do |path|
73
113
  @extra_roots << path
74
114
  end
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'json'
4
+ require 'digest'
4
5
 
5
6
  module Woods
6
7
  module SourceInputs
7
8
  # Compact wire format: consumers refer to a shared identity table. Internal
8
9
  # mutation uses expanded digests so table indices never leak across runs.
9
- class Manifest
10
+ class Manifest # rubocop:disable Metrics/ClassLength -- compact wire validation and bounded serialization
10
11
  FILE_NAME = 'source_inputs.json'
11
12
  VERSION = 1
12
13
  MAX_BYTES = 16 * 1024 * 1024
14
+ UNKNOWN_BASELINES = %w[invalid_source_baseline missing_or_incompatible_source_baseline
15
+ incomplete_source_baseline].freeze
13
16
  class Invalid < StandardError; end
14
17
 
15
18
  attr_reader :data
@@ -34,24 +37,73 @@ module Woods
34
37
  end
35
38
  end
36
39
 
40
+ # Older version-1 writers encoded missing comparison evidence only as
41
+ # errors. Never interpret their empty ledger as proof of an empty tree.
42
+ def comparison_complete?
43
+ @data['complete'] && @data.fetch('comparison_complete', true) &&
44
+ @data.fetch('errors').none? { |error| UNKNOWN_BASELINES.include?(error['reason']) }
45
+ end
46
+
37
47
  # Explicit wire fields keep incomplete coverage separate from boot provenance.
38
48
  # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
39
- def self.build(snapshot:, scopes:, boot_verified:, generation:, errors: [], unverified_scopes: [])
49
+ def self.build(snapshot:, scopes:, boot_verified:, generation:, errors: [], unverified_scopes: [],
50
+ comparison_complete: true, unavailable: nil)
40
51
  identities = scopes.values.flat_map(&:values).uniq.sort
41
52
  indices = identities.each_with_index.to_h
42
53
  new('version' => VERSION, 'generation' => generation, 'root' => snapshot.fetch('root'),
54
+ 'captured_at' => snapshot['captured_at'],
43
55
  'key_id' => snapshot.fetch('key_id'), 'rules' => snapshot.fetch('rules'),
44
56
  'extra_roots' => snapshot.fetch('extra_roots'), 'boot_verified' => boot_verified,
45
57
  'complete' => snapshot.fetch('complete'), 'errors' => (snapshot.fetch('errors') + errors).uniq,
58
+ 'comparison_complete' => comparison_complete,
46
59
  'unverified_scopes' => unverified_scopes.uniq.sort,
47
60
  'identities' => identities, 'metrics' => snapshot.fetch('metrics'),
48
61
  'scopes' => scopes.sort.to_h.transform_values do |paths|
49
62
  paths.sort.to_h.transform_values { |digest| indices.fetch(digest) }
50
- end)
63
+ end).tap { |manifest| manifest.bound_publication!(snapshot: snapshot, unavailable: unavailable) }
51
64
  end
52
65
 
53
66
  # rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
54
67
 
68
+ # Compact watch boundary only: this never certifies runtime freshness.
69
+ def self.capture_digest(files)
70
+ Digest::SHA256.hexdigest(JSON.generate(files.sort.to_h))
71
+ end
72
+
73
+ def self.valid_unavailable_evidence?(value)
74
+ value.is_a?(Hash) && value['reason'] == 'source_manifest_too_large' &&
75
+ value.values_at('size_bytes', 'limit_bytes').all?(Integer) &&
76
+ value['limit_bytes'].positive? && value['size_bytes'] > value['limit_bytes']
77
+ end
78
+
79
+ def unavailable?
80
+ @data['state'] == 'unavailable'
81
+ end
82
+
83
+ # Bind only an already validated, generation-local reference cache. This
84
+ # preserves its consumed-source proof without publishing the large ledger.
85
+ def bind_reference_cache!(cache)
86
+ @data['reference_cache_sha256'] = Digest::SHA256.hexdigest(JSON.generate(cache)) if unavailable? && cache
87
+ end
88
+
89
+ # Source evidence is advisory. Its size limit must not withhold a valid
90
+ # code index. Keep errors so final publication still refuses unstable input.
91
+ def bound_publication!(snapshot:, unavailable: nil)
92
+ size = JSON.pretty_generate(@data).bytesize
93
+ return unless unavailable || size > MAX_BYTES
94
+
95
+ diagnostic = unavailable || { 'reason' => 'source_manifest_too_large',
96
+ 'size_bytes' => size, 'limit_bytes' => MAX_BYTES }
97
+ @data['capture_sha256'] = self.class.capture_digest(snapshot.fetch('files')) if snapshot['complete']
98
+ @data.merge!('state' => 'unavailable', 'unavailable' => diagnostic,
99
+ 'complete' => false, 'comparison_complete' => false,
100
+ 'identities' => [], 'scopes' => {}, 'metrics' => {})
101
+ warn "[woods] source freshness unavailable: #{diagnostic['reason']}; " \
102
+ "#{diagnostic['size_bytes']} bytes exceeds limit #{diagnostic['limit_bytes']}. " \
103
+ 'Publishing the code index without freshness comparison evidence; ' \
104
+ 'another full extraction will not reduce this size.'
105
+ end
106
+
55
107
  private
56
108
 
57
109
  def validate!
@@ -59,11 +111,28 @@ module Woods
59
111
  raise Invalid, 'invalid_source_manifest_header' unless valid_header?
60
112
 
61
113
  validate_tables!
114
+ validate_unavailable!
62
115
  return if @data['extra_roots'].all? { |path| valid_path?(path) }
63
116
 
64
117
  raise Invalid, 'invalid_source_roots'
65
118
  end
66
119
 
120
+ def validate_unavailable!
121
+ return if (%w[state unavailable reference_cache_sha256 capture_sha256] & @data.keys).empty?
122
+
123
+ valid = unavailable? && self.class.valid_unavailable_evidence?(@data['unavailable']) && empty_comparison?
124
+ valid &&= valid_optional_digests?
125
+ raise Invalid, 'invalid_unavailable_source_manifest' unless valid
126
+ end
127
+
128
+ def valid_optional_digests?
129
+ %w[reference_cache_sha256 capture_sha256].all? { |key| !@data.key?(key) || digest?(@data[key]) }
130
+ end
131
+
132
+ def empty_comparison?
133
+ !@data['complete'] && !@data['comparison_complete'] && @data['scopes'].empty? && @data['identities'].empty?
134
+ end
135
+
67
136
  def validate_tables!
68
137
  identities = @data['identities']
69
138
  scopes = @data['scopes']
@@ -89,9 +158,21 @@ module Woods
89
158
 
90
159
  def valid_header?
91
160
  valid_root? && %w[key_id rules].all? { |key| digest?(@data[key]) } &&
161
+ valid_capture_time? && valid_comparison_coverage? &&
92
162
  %w[boot_verified complete].all? { |key| [true, false].include?(@data[key]) }
93
163
  end
94
164
 
165
+ def valid_comparison_coverage?
166
+ !@data.key?('comparison_complete') || [true, false].include?(@data['comparison_complete'])
167
+ end
168
+
169
+ # Optional so existing version-1 manifests remain readable. Consumers
170
+ # needing a catch-up boundary must reconcile when it is absent.
171
+ def valid_capture_time?
172
+ value = @data['captured_at']
173
+ value.nil? || (value.is_a?(Numeric) && value.finite? && value.positive?)
174
+ end
175
+
95
176
  def valid_root?
96
177
  root = @data['root']
97
178
  root.is_a?(String) && root.start_with?('/') && !root.include?("\0")