woods 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +108 -0
- data/CONTRIBUTING.md +135 -10
- data/README.md +1 -1
- data/docs/AGENT_GUIDE.md +19 -0
- data/docs/AGENT_SETUP.md +22 -2
- data/docs/BACKEND_MATRIX.md +7 -0
- data/docs/CLIENT_HOOKS.md +6 -0
- data/docs/CONFIGURATION_REFERENCE.md +133 -18
- data/docs/CONSOLE_MCP_SETUP.md +141 -12
- data/docs/EMBEDDING_MODELS.md +16 -19
- data/docs/EXTRACTOR_REFERENCE.md +219 -21
- data/docs/FAQ.md +11 -25
- data/docs/GETTING_STARTED.md +7 -1
- data/docs/INCREMENTAL_EXTRACTION.md +261 -19
- data/docs/INDEX_LAYOUT.md +5 -0
- data/docs/INTERNALS.md +9 -0
- data/docs/MCP_HTTP_TRANSPORT.md +59 -2
- data/docs/MCP_SERVERS.md +87 -8
- data/docs/MCP_TOOL_COOKBOOK.md +13 -55
- data/docs/NOTION_INTEGRATION.md +7 -1
- data/docs/PUBLISHED_INDEX.md +6 -0
- data/docs/README.md +6 -1
- data/docs/RETRIEVAL_GUIDE.md +17 -0
- data/docs/SOURCE_FRESHNESS.md +157 -5
- data/docs/TOKEN_BENCHMARK.md +10 -18
- data/docs/TROUBLESHOOTING.md +70 -14
- data/docs/UNBLOCKED_INTEGRATION.md +60 -8
- data/docs/UPGRADING_TO_2.md +184 -9
- data/docs/WATCH_DAEMON.md +97 -14
- data/exe/woods-console-mcp +2 -2
- data/exe/woods-mcp-http +16 -9
- data/lib/generators/woods/templates/woods.rb.tt +2 -1
- data/lib/tasks/woods.rake +23 -7
- data/lib/tasks/woods_checks.rake +2 -2
- data/lib/woods/agent_configuration/cli.rb +1 -1
- data/lib/woods/agent_configuration/layout.rb +16 -2
- data/lib/woods/agent_configuration/plan.rb +13 -3
- data/lib/woods/agent_configuration/planner_validation.rb +4 -2
- data/lib/woods/agent_configuration/preflight.rb +5 -3
- data/lib/woods/builder.rb +17 -57
- data/lib/woods/cache/cache_middleware.rb +68 -41
- data/lib/woods/chunking/contributor_chunks.rb +119 -0
- data/lib/woods/chunking/semantic_chunker.rb +44 -21
- data/lib/woods/console/adapter_family.rb +39 -0
- data/lib/woods/console/connection_manager.rb +56 -3
- data/lib/woods/console/credential_index.rb +33 -3
- data/lib/woods/console/embedded_executor.rb +431 -48
- data/lib/woods/console/model_validator.rb +8 -0
- data/lib/woods/console/rack_middleware.rb +68 -11
- data/lib/woods/console/redactor.rb +24 -10
- data/lib/woods/console/safe_context.rb +44 -7
- data/lib/woods/console/sql_noise_stripper.rb +41 -12
- data/lib/woods/console/sql_table_scanner.rb +45 -34
- data/lib/woods/console/sql_validator.rb +37 -2
- data/lib/woods/dependency_graph.rb +34 -10
- data/lib/woods/embedding/fake.rb +12 -0
- data/lib/woods/embedding/indexer.rb +195 -98
- data/lib/woods/embedding/input_budget.rb +67 -0
- data/lib/woods/embedding/openai.rb +70 -20
- data/lib/woods/embedding/provider.rb +37 -25
- data/lib/woods/embedding/text_preparer.rb +76 -32
- data/lib/woods/embedding/token_counter.rb +18 -81
- data/lib/woods/embedding/vector_configuration.rb +48 -0
- data/lib/woods/extraction_identities.rb +175 -0
- data/lib/woods/extractor.rb +304 -107
- data/lib/woods/extractors/action_cable_extractor.rb +8 -3
- data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
- data/lib/woods/extractors/class_declarations.rb +121 -0
- data/lib/woods/extractors/configuration_extractor.rb +11 -3
- data/lib/woods/extractors/declaration_ancestry.rb +92 -0
- data/lib/woods/extractors/event_extractor.rb +8 -0
- data/lib/woods/extractors/graphql_extractor.rb +134 -77
- data/lib/woods/extractors/job_extractor.rb +5 -1
- data/lib/woods/extractors/lib_extractor.rb +132 -15
- data/lib/woods/extractors/mailer_extractor.rb +3 -5
- data/lib/woods/extractors/manager_extractor.rb +7 -21
- data/lib/woods/extractors/migration_declaration.rb +87 -0
- data/lib/woods/extractors/migration_extractor.rb +5 -39
- data/lib/woods/extractors/phlex_extractor.rb +6 -2
- data/lib/woods/extractors/policy_extractor.rb +9 -5
- data/lib/woods/extractors/poro_extractor.rb +112 -53
- data/lib/woods/extractors/pundit_extractor.rb +11 -6
- data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
- data/lib/woods/extractors/serializer_extractor.rb +34 -22
- data/lib/woods/extractors/shared_utility_methods.rb +18 -1
- data/lib/woods/extractors/source_nesting.rb +142 -106
- data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
- data/lib/woods/extractors/state_machine_extractor.rb +46 -40
- data/lib/woods/extractors/view_component_extractor.rb +9 -7
- data/lib/woods/flow_assembler.rb +4 -1
- data/lib/woods/generation.rb +25 -0
- data/lib/woods/hooks/context_hint.rb +7 -2
- data/lib/woods/mcp/bearer_auth.rb +1 -1
- data/lib/woods/mcp/bootstrapper.rb +33 -7
- data/lib/woods/mcp/config_resolver.rb +26 -7
- data/lib/woods/mcp/index_reader.rb +125 -24
- data/lib/woods/mcp/index_reader_pinning.rb +16 -0
- data/lib/woods/mcp/origin_guard.rb +24 -77
- data/lib/woods/mcp/origin_policy.rb +124 -0
- data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
- data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
- data/lib/woods/mcp/search_results.rb +7 -1
- data/lib/woods/mcp/server.rb +24 -4
- data/lib/woods/module_reconciliation.rb +151 -0
- data/lib/woods/path_dispatcher.rb +7 -2
- data/lib/woods/railtie_support.rb +8 -0
- data/lib/woods/rake_helpers.rb +43 -11
- data/lib/woods/release.rb +1 -1
- data/lib/woods/resilience/index_validator.rb +8 -3
- data/lib/woods/resilience/retryable_provider.rb +18 -1
- data/lib/woods/resolved_config.rb +68 -8
- data/lib/woods/retrieval/context_assembler.rb +3 -3
- data/lib/woods/retrieval/lexical_assembler.rb +3 -2
- data/lib/woods/retrieval/scope.rb +18 -2
- data/lib/woods/retrieval/source_evidence.rb +14 -2
- data/lib/woods/source_contributor_validation.rb +78 -0
- data/lib/woods/source_contributors.rb +116 -0
- data/lib/woods/source_inputs/handoff.rb +37 -0
- data/lib/woods/source_inputs/launcher.rb +53 -13
- data/lib/woods/source_inputs/manifest.rb +84 -3
- data/lib/woods/source_inputs/private_key.rb +44 -12
- data/lib/woods/source_inputs/scanner.rb +98 -27
- data/lib/woods/source_inputs/scopes.rb +1 -1
- data/lib/woods/source_inputs/session.rb +147 -15
- data/lib/woods/source_inputs/stable_reader.rb +127 -0
- data/lib/woods/source_inputs/status.rb +40 -8
- data/lib/woods/source_inputs/verifier.rb +28 -5
- data/lib/woods/source_path_encoding.rb +33 -0
- data/lib/woods/source_references/cache.rb +284 -0
- data/lib/woods/source_references/collector.rb +120 -0
- data/lib/woods/source_references/extraction.rb +185 -0
- data/lib/woods/source_references/inputs.rb +134 -0
- data/lib/woods/source_references/parser_adapter.rb +134 -0
- data/lib/woods/source_references/pass.rb +152 -0
- data/lib/woods/source_references/prism_adapter.rb +116 -0
- data/lib/woods/source_references/registry.rb +178 -0
- data/lib/woods/source_references/runtime_lookup.rb +127 -0
- data/lib/woods/source_references/value_class.rb +82 -0
- data/lib/woods/storage/metadata_store.rb +4 -1
- data/lib/woods/storage/qdrant.rb +2 -2
- data/lib/woods/unblocked/client.rb +12 -7
- data/lib/woods/unblocked/document_builder.rb +4 -1
- data/lib/woods/unblocked/exporter.rb +127 -37
- data/lib/woods/unblocked/sync_manifest.rb +137 -21
- data/lib/woods/unblocked/uri_migration.rb +105 -0
- data/lib/woods/util/host_guard.rb +3 -2
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/catch_up.rb +138 -0
- data/lib/woods/watch/claim_lease.rb +150 -0
- data/lib/woods/watch/cli.rb +26 -2
- data/lib/woods/watch/daemon.rb +80 -59
- data/lib/woods/watch/installation/options.rb +1 -1
- data/lib/woods/watch/installation/receipt.rb +6 -1
- data/lib/woods/watch/managed_child.rb +1 -1
- data/lib/woods/watch/supervisor.rb +1 -1
- data/lib/woods/watch/tree_scan.rb +14 -2
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/hooks/adapters/normalize.rb +3 -2
- data/plugin/hooks/woods-input-rules.sh +4 -0
- data/plugin/hooks/woods-refresh.sh +15 -7
- data/plugin/hooks/woods-session-start.sh +60 -3
- data/plugin/skills/woods-diagnose/SKILL.md +336 -2
- data/plugin/skills/woods-investigate/SKILL.md +11 -0
- data/plugin/skills/woods-mcp-config/SKILL.md +86 -0
- data/plugin/skills/woods-setup/SKILL.md +56 -1
- metadata +34 -5
|
@@ -23,7 +23,7 @@ module Woods
|
|
|
23
23
|
sources = []
|
|
24
24
|
candidates.each do |candidate|
|
|
25
25
|
unit = candidate.metadata
|
|
26
|
-
header = "\n\n## #{unit['identifier']} (#{unit['type']})\
|
|
26
|
+
header = "\n\n## #{unit['identifier']} (#{unit['type']})\n#{SourceContributors.label(unit)}\n" \
|
|
27
27
|
"Matched: #{candidate.matched_fields.join(', ')}\n\n"
|
|
28
28
|
remaining = (budget * 4) - context.length - header.length
|
|
29
29
|
next unless remaining.positive?
|
|
@@ -37,7 +37,7 @@ module Woods
|
|
|
37
37
|
context += header + selected.text
|
|
38
38
|
sources << { identifier: unit['identifier'], type: unit['type'], file_path: unit['file_path'],
|
|
39
39
|
score: candidate.score, matched_fields: candidate.matched_fields,
|
|
40
|
-
evidence: selected.provenance }
|
|
40
|
+
evidence: selected.provenance }.merge(SourceContributors.attribution(unit))
|
|
41
41
|
next
|
|
42
42
|
end
|
|
43
43
|
|
|
@@ -49,6 +49,7 @@ module Woods
|
|
|
49
49
|
context += header + (truncated ? source[0, remaining - marker.length] + marker : source)
|
|
50
50
|
sources << { identifier: unit['identifier'], type: unit['type'], file_path: unit['file_path'],
|
|
51
51
|
score: candidate.score, matched_fields: candidate.matched_fields, truncated: truncated }
|
|
52
|
+
.merge(SourceContributors.attribution(unit))
|
|
52
53
|
break if truncated
|
|
53
54
|
end
|
|
54
55
|
body = context[notice.length..].to_s
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'set'
|
|
5
5
|
require_relative '../storage_identity'
|
|
6
|
+
require_relative '../source_contributors'
|
|
6
7
|
require_relative '../storage/metadata_store'
|
|
7
8
|
|
|
8
9
|
module Woods
|
|
@@ -83,13 +84,28 @@ module Woods
|
|
|
83
84
|
def validate_packages!(records)
|
|
84
85
|
names = records.filter_map { |unit| unit['identifier'] if unit['type'] == 'package' }
|
|
85
86
|
names.concat(records.filter_map { |unit| unit.dig('metadata', 'package') })
|
|
87
|
+
names.concat(records.flat_map do |unit|
|
|
88
|
+
SourceContributors.records(unit).filter_map do |record|
|
|
89
|
+
record['package']
|
|
90
|
+
end
|
|
91
|
+
end)
|
|
86
92
|
unknown = packages - names
|
|
87
93
|
raise InvalidScopeError, "unknown package scope: #{unknown.join(', ')}" unless unknown.empty?
|
|
88
94
|
end
|
|
89
95
|
|
|
90
96
|
def eligible?(unit, types, excluded)
|
|
91
|
-
|
|
92
|
-
|
|
97
|
+
contributors = SourceContributors.records(unit)
|
|
98
|
+
owners = if contributors.empty?
|
|
99
|
+
[unit.dig('metadata', 'package')]
|
|
100
|
+
else
|
|
101
|
+
contributors.map do |record|
|
|
102
|
+
record['package']
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
return false unless packages.empty? || owners.all? { |owner| packages.include?(owner) }
|
|
106
|
+
|
|
107
|
+
paths = SourceContributors.paths(unit)
|
|
108
|
+
return false unless source_paths.empty? || (paths.any? && paths.all? { |path| path_match?(path) })
|
|
93
109
|
return Array(types).map(&:to_s).include?(unit['type']) if types && !types.empty?
|
|
94
110
|
|
|
95
111
|
!Array(excluded).map(&:to_s).include?(unit['type'])
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../source_contributors'
|
|
4
|
+
|
|
3
5
|
require 'digest'
|
|
4
6
|
require 'json'
|
|
5
7
|
require_relative '../ast/parser'
|
|
@@ -170,7 +172,7 @@ module Woods
|
|
|
170
172
|
|
|
171
173
|
def format_evidence(mode, selected, metadata, total)
|
|
172
174
|
parts = ["Evidence: #{mode}; published-unit coordinates (physical location unavailable).",
|
|
173
|
-
"Unit: #{field(:type)}:#{field(:identifier)};
|
|
175
|
+
"Unit: #{field(:type)}:#{field(:identifier)}; #{SourceContributors.label(@unit)}",
|
|
174
176
|
"Source SHA256: #{Digest::SHA256.hexdigest(@source)}",
|
|
175
177
|
"Generation: #{@generation || 'unavailable (not recorded by this metadata store)'}"]
|
|
176
178
|
selected.each do |span|
|
|
@@ -183,17 +185,27 @@ module Woods
|
|
|
183
185
|
parts.join("\n\n")
|
|
184
186
|
end
|
|
185
187
|
|
|
188
|
+
def span_provenance(span)
|
|
189
|
+
result = span.to_h.merge(sha256: Digest::SHA256.hexdigest(span_text(span)))
|
|
190
|
+
if SourceContributors.multiple?(@unit)
|
|
191
|
+
result[:physical_location] =
|
|
192
|
+
SourceContributors.physical_span(@unit, start_byte: span.start_byte, end_byte: span.end_byte)
|
|
193
|
+
end
|
|
194
|
+
result
|
|
195
|
+
end
|
|
196
|
+
|
|
186
197
|
def provenance(mode, selected, metadata, all)
|
|
187
198
|
{ mode: mode, owner: { identifier: field(:identifier), type: field(:type) },
|
|
188
199
|
coordinate_system: 'published_unit', source_sha256: Digest::SHA256.hexdigest(@source),
|
|
189
200
|
source_path: field(:file_path), physical_location: nil,
|
|
190
201
|
physical_location_reason: 'published unit may contain synthesized or inlined source',
|
|
191
202
|
generation: @generation, generation_status: @generation ? 'recorded' : 'unavailable',
|
|
192
|
-
spans: selected.map { |span|
|
|
203
|
+
spans: selected.map { |span| span_provenance(span) },
|
|
193
204
|
omitted_spans: all.size - selected.size, runtime_fields: metadata.map(&:first),
|
|
194
205
|
source_complete: false, full_evidence: { tool: 'lookup', evidence: 'full',
|
|
195
206
|
identifier: field(:identifier), type: field(:type),
|
|
196
207
|
source_sha256: Digest::SHA256.hexdigest(@source) } }
|
|
208
|
+
.merge(SourceContributors.attribution(@unit))
|
|
197
209
|
end
|
|
198
210
|
end
|
|
199
211
|
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Woods
|
|
6
|
+
# Checks the versioned contributor boundary before consumers use its paths or coordinates.
|
|
7
|
+
module SourceContributorValidation
|
|
8
|
+
class Invalid < ArgumentError; end
|
|
9
|
+
|
|
10
|
+
RANGE_KEYS = %w[source_start_line source_end_line published_start_byte published_end_byte
|
|
11
|
+
published_start_line published_end_line].freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def validate!(records, type:, version:, source:)
|
|
16
|
+
unless type.to_s == 'lib' && version == 1 && records.is_a?(Array) && records.size.between?(2, 100_000)
|
|
17
|
+
raise Invalid, 'Invalid library contributor version or records'
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
validate_records!(records)
|
|
21
|
+
records.each { |record| validate_source!(source, record) } if source.is_a?(String)
|
|
22
|
+
records
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def validate_records!(records)
|
|
26
|
+
previous = 0
|
|
27
|
+
paths = records.map do |record|
|
|
28
|
+
unless valid_record?(record) && valid_ranges?(record, previous)
|
|
29
|
+
raise Invalid, 'Invalid library contributor path or range'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
previous = record.fetch('published_end_byte')
|
|
33
|
+
record.fetch('file_path')
|
|
34
|
+
end
|
|
35
|
+
raise Invalid, 'Unordered or duplicate library contributor path' unless paths.uniq.sort == paths
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def valid_record?(record)
|
|
39
|
+
record.is_a?(Hash) && record.keys.all?(String) && valid_path?(record['file_path']) &&
|
|
40
|
+
valid_hash?(record['source_sha256']) &&
|
|
41
|
+
record['facts'].is_a?(Hash) && RANGE_KEYS.all? { |key| record[key].is_a?(Integer) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def valid_hash?(hash)
|
|
45
|
+
hash.is_a?(String) && hash.match?(/\A[0-9a-f]{64}\z/)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def valid_ranges?(record, previous)
|
|
49
|
+
record['source_start_line'] == 1 && record['source_end_line'].positive? &&
|
|
50
|
+
record['published_start_byte'] >= previous && record['published_end_byte'] > record['published_start_byte'] &&
|
|
51
|
+
record['published_start_line'].positive? && record['published_end_line'] >= record['published_start_line']
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def valid_path?(path)
|
|
55
|
+
path.is_a?(String) && path.start_with?('lib/') && path.end_with?('.rb') && !path.include?("\0") &&
|
|
56
|
+
!path.include?('\\') && path.split('/', -1).none? { |part| ['', '.', '..'].include?(part) }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def validate_source!(source, record)
|
|
60
|
+
raise Invalid, 'Contributor range exceeds published source' if record['published_end_byte'] > source.bytesize
|
|
61
|
+
|
|
62
|
+
fragment = source.byteslice(record['published_start_byte']...record['published_end_byte'])
|
|
63
|
+
first = source.byteslice(0...record['published_start_byte']).count("\n") + 1
|
|
64
|
+
return if valid_fragment?(fragment, first, record)
|
|
65
|
+
|
|
66
|
+
raise Invalid, 'Library contributor bytes do not match published provenance'
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def valid_fragment?(fragment, first, record)
|
|
70
|
+
Digest::SHA256.hexdigest(fragment) == record['source_sha256'] &&
|
|
71
|
+
fragment.lines.size == record['source_end_line'] && first == record['published_start_line'] &&
|
|
72
|
+
first + fragment.lines.size - 1 == record['published_end_line']
|
|
73
|
+
end
|
|
74
|
+
private_class_method :validate_records!, :valid_record?, :valid_hash?, :valid_ranges?, :valid_path?,
|
|
75
|
+
:validate_source!,
|
|
76
|
+
:valid_fragment?
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'source_contributor_validation'
|
|
4
|
+
|
|
5
|
+
module Woods
|
|
6
|
+
# Provenance for a published unit assembled from several original files.
|
|
7
|
+
# Byte ranges are half-open and exclude every generated header/separator.
|
|
8
|
+
module SourceContributors
|
|
9
|
+
Invalid = SourceContributorValidation::Invalid
|
|
10
|
+
|
|
11
|
+
VERSION = 1
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# @param unit [ExtractedUnit, Hash]
|
|
16
|
+
# @return [Array<Hash>] validated, string-keyed contributor records
|
|
17
|
+
def records(unit)
|
|
18
|
+
metadata = field(unit, :metadata) || {}
|
|
19
|
+
raw = field(metadata, :source_contributors)
|
|
20
|
+
return [] if raw.nil?
|
|
21
|
+
|
|
22
|
+
SourceContributorValidation.validate!(raw, type: field(unit, :type),
|
|
23
|
+
version: field(metadata, :source_contributors_version),
|
|
24
|
+
source: field(unit, :source_code))
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param unit [ExtractedUnit, Hash]
|
|
28
|
+
# @return [Array<String>] every physical source, or the legacy primary path
|
|
29
|
+
def paths(unit)
|
|
30
|
+
contributors = records(unit)
|
|
31
|
+
contributors.empty? ? Array(field(unit, :file_path)) : contributors.map { |record| record.fetch('file_path') }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# @param unit [ExtractedUnit, Hash]
|
|
35
|
+
# @return [Boolean]
|
|
36
|
+
def multiple?(unit)
|
|
37
|
+
!records(unit).empty?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @param unit [ExtractedUnit, Hash]
|
|
41
|
+
# @param start_byte [Integer] inclusive published byte offset
|
|
42
|
+
# @param end_byte [Integer] exclusive published byte offset
|
|
43
|
+
# @return [Hash, nil] physical coordinates only for an exact original fragment
|
|
44
|
+
def physical_span(unit, start_byte:, end_byte:)
|
|
45
|
+
return unless valid_span?(start_byte, end_byte)
|
|
46
|
+
|
|
47
|
+
record = records(unit).find do |fragment|
|
|
48
|
+
start_byte >= fragment['published_start_byte'] && end_byte <= fragment['published_end_byte']
|
|
49
|
+
end
|
|
50
|
+
return unless record
|
|
51
|
+
|
|
52
|
+
source = field(unit, :source_code)
|
|
53
|
+
return unless source.is_a?(String)
|
|
54
|
+
|
|
55
|
+
first = record['source_start_line'] + source.byteslice(record['published_start_byte']...start_byte).count("\n")
|
|
56
|
+
{ file_path: record['file_path'], source_sha256: record['source_sha256'], start_line: first,
|
|
57
|
+
end_line: first + source.byteslice(start_byte...end_byte).delete_suffix("\n").count("\n") }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def valid_span?(first, last)
|
|
61
|
+
first.is_a?(Integer) && last.is_a?(Integer) && last > first
|
|
62
|
+
end
|
|
63
|
+
private_class_method :valid_span?
|
|
64
|
+
|
|
65
|
+
# @param unit [ExtractedUnit, Hash]
|
|
66
|
+
# @return [String] explicit primary/contributor label, or legacy file label
|
|
67
|
+
def label(unit)
|
|
68
|
+
return "File: #{field(unit, :file_path)}" unless multiple?(unit)
|
|
69
|
+
|
|
70
|
+
"Primary file: #{field(unit, :file_path)}\nContributing files: #{paths(unit).join(', ')}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @param unit [ExtractedUnit, Hash]
|
|
74
|
+
# @return [Hash] additive contributor evidence for response renderers
|
|
75
|
+
def attribution(unit)
|
|
76
|
+
multiple?(unit) ? { source_contributors: records(unit) } : {}
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @param unit [ExtractedUnit, Hash]
|
|
80
|
+
# @param resolver [Object] package_for(path) collaborator
|
|
81
|
+
# @return [String, nil] unanimous package ownership, or nil
|
|
82
|
+
def annotate_package(unit, resolver)
|
|
83
|
+
contributors = records(unit)
|
|
84
|
+
return resolver.package_for(field(unit, :file_path)) if contributors.empty?
|
|
85
|
+
|
|
86
|
+
packages = contributors.map do |record|
|
|
87
|
+
package = resolver.package_for(record['file_path'])
|
|
88
|
+
package ? record['package'] = package : record.delete('package')
|
|
89
|
+
package
|
|
90
|
+
end
|
|
91
|
+
packages.first if packages.uniq.one?
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @param unit [ExtractedUnit, Hash]
|
|
95
|
+
# @param data [Hash] Git facts keyed by application-relative source path
|
|
96
|
+
# @return [void]
|
|
97
|
+
def annotate_git(unit, data)
|
|
98
|
+
records(unit).each do |record|
|
|
99
|
+
record.delete('git')
|
|
100
|
+
record['git'] = data[record['file_path']] if data[record['file_path']]
|
|
101
|
+
end
|
|
102
|
+
metadata = field(unit, :metadata)
|
|
103
|
+
metadata.delete(:git)
|
|
104
|
+
metadata.delete('git')
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @param value [Object, Hash]
|
|
108
|
+
# @param key [Symbol]
|
|
109
|
+
# @return [Object]
|
|
110
|
+
def field(value, key)
|
|
111
|
+
return value[key] || value[key.to_s] if value.is_a?(Hash)
|
|
112
|
+
|
|
113
|
+
value.public_send(key) if value.respond_to?(key)
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -9,9 +9,35 @@ module Woods
|
|
|
9
9
|
# existing index manifest nor a user-supplied proof of runtime freshness.
|
|
10
10
|
module Handoff
|
|
11
11
|
ENV_KEY = 'WOODS_SOURCE_CAPTURE'
|
|
12
|
+
class OutputMismatch < StandardError; end
|
|
12
13
|
|
|
13
14
|
module_function
|
|
14
15
|
|
|
16
|
+
# An implicit preboot default cannot override finalized application config.
|
|
17
|
+
# Explicit launcher outputs remain intentional overrides. Called before
|
|
18
|
+
# the writer lock/extraction, while the one-use handoff is still present.
|
|
19
|
+
# @param root [String, Pathname] finalized application root
|
|
20
|
+
# @param output_dir [String, Pathname] selected writer output
|
|
21
|
+
# @return [void]
|
|
22
|
+
def validate_output!(root:, output_dir:)
|
|
23
|
+
token = ENV.fetch(ENV_KEY, nil)
|
|
24
|
+
return if token.to_s.empty?
|
|
25
|
+
|
|
26
|
+
descriptor = JSON.parse(token)
|
|
27
|
+
return unless descriptor.is_a?(Hash) && descriptor['implicit_output'] == true
|
|
28
|
+
|
|
29
|
+
expected = File.expand_path('tmp/woods', root.to_s)
|
|
30
|
+
configured = File.expand_path(Woods.configuration.output_dir.to_s, root.to_s)
|
|
31
|
+
return if configured == expected && File.expand_path(output_dir.to_s, root.to_s) == expected
|
|
32
|
+
|
|
33
|
+
raise OutputMismatch,
|
|
34
|
+
"woods-extract configured output #{configured.inspect} differs from " \
|
|
35
|
+
"its preboot default #{expected.inspect}; " \
|
|
36
|
+
'rerun with matching --output PATH or WOODS_OUTPUT so capture and publication use the same index'
|
|
37
|
+
rescue JSON::ParserError, TypeError
|
|
38
|
+
nil # Ordinary handoff validation will refuse unverifiable capture.
|
|
39
|
+
end
|
|
40
|
+
|
|
15
41
|
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
16
42
|
# Every independent handoff binding must match before a token is consumed.
|
|
17
43
|
def read(root:, output_dir:, operation:, rules:, key_id:)
|
|
@@ -85,6 +111,17 @@ module Woods
|
|
|
85
111
|
|
|
86
112
|
# rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
87
113
|
|
|
114
|
+
# An oversized launcher capture conveys only a downgrade, never proof.
|
|
115
|
+
def unavailable_evidence
|
|
116
|
+
descriptor = JSON.parse(ENV.fetch(ENV_KEY, '{}'))
|
|
117
|
+
value = descriptor.is_a?(Hash) && descriptor['unavailable']
|
|
118
|
+
return unless Manifest.valid_unavailable_evidence?(value)
|
|
119
|
+
|
|
120
|
+
value.slice('reason', 'size_bytes', 'limit_bytes')
|
|
121
|
+
rescue JSON::ParserError
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
|
|
88
125
|
def extra_roots
|
|
89
126
|
token = ENV.fetch(ENV_KEY, nil)
|
|
90
127
|
return [] if token.nil? || token.empty?
|
|
@@ -17,22 +17,27 @@ module Woods
|
|
|
17
17
|
|
|
18
18
|
def self.run(argv, **options)
|
|
19
19
|
new(argv, **options).run
|
|
20
|
-
rescue
|
|
20
|
+
rescue PrivateKey::Unavailable => e
|
|
21
|
+
warn "woods-extract: #{e.diagnostic}"
|
|
22
|
+
1
|
|
23
|
+
rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
|
|
21
24
|
warn "woods-extract: #{e.message}"
|
|
22
25
|
1
|
|
23
26
|
end
|
|
24
27
|
|
|
25
|
-
def initialize(argv, command:
|
|
28
|
+
def initialize(argv, command: nil)
|
|
26
29
|
@arguments = argv.dup
|
|
27
30
|
@command = command
|
|
28
31
|
@original_directory = Dir.pwd
|
|
29
32
|
@root = @original_directory
|
|
33
|
+
@implicit_output = !ENV.key?('WOODS_OUTPUT')
|
|
30
34
|
@output = ENV.fetch('WOODS_OUTPUT', 'tmp/woods')
|
|
31
35
|
@extra_roots = []
|
|
32
36
|
parse!
|
|
37
|
+
@command ||= default_command
|
|
33
38
|
end
|
|
34
39
|
|
|
35
|
-
def run
|
|
40
|
+
def run
|
|
36
41
|
return 0 if @help
|
|
37
42
|
|
|
38
43
|
operation, task = task_for_arguments
|
|
@@ -40,25 +45,57 @@ module Woods
|
|
|
40
45
|
rules = Scopes.new(extra_roots: @extra_roots)
|
|
41
46
|
snapshot = Scanner.new(root: @root, output_dir: @output, key: key, scopes: rules).call
|
|
42
47
|
nonce = SecureRandom.hex(32)
|
|
48
|
+
bytes = capture_bytes(snapshot, nonce, operation, rules)
|
|
49
|
+
return run_without_capture(task) unless bytes
|
|
50
|
+
|
|
43
51
|
Tempfile.create(['woods-source-capture-', '.json']) do |file|
|
|
44
52
|
file.binmode
|
|
45
53
|
file.chmod(0o600)
|
|
46
|
-
file.write(
|
|
47
|
-
operation: operation, rules: rules.fingerprint,
|
|
48
|
-
launcher_pid: Process.pid, snapshot: snapshot))
|
|
54
|
+
file.write(bytes)
|
|
49
55
|
file.flush
|
|
50
56
|
file.fsync
|
|
51
|
-
descriptor = JSON.generate(path: file.path, nonce: nonce, extra_roots: @extra_roots
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
environment['BUNDLE_GEMFILE'] = File.expand_path(ENV.fetch('BUNDLE_GEMFILE'), @original_directory)
|
|
55
|
-
end
|
|
56
|
-
run_child(environment, task)
|
|
57
|
+
descriptor = JSON.generate(path: file.path, nonce: nonce, extra_roots: @extra_roots,
|
|
58
|
+
implicit_output: @implicit_output)
|
|
59
|
+
run_child(child_environment(descriptor), task)
|
|
57
60
|
end
|
|
58
61
|
end
|
|
59
62
|
|
|
60
63
|
private
|
|
61
64
|
|
|
65
|
+
def default_command
|
|
66
|
+
binstub = File.join(@root, 'bin/rake')
|
|
67
|
+
File.file?(binstub) && File.executable?(binstub) ? [binstub] : %w[bundle exec rake]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def run_without_capture(task)
|
|
71
|
+
descriptor = JSON.generate(extra_roots: @extra_roots, implicit_output: @implicit_output,
|
|
72
|
+
unavailable: @capture_unavailable)
|
|
73
|
+
run_child(child_environment(descriptor), task)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def capture_bytes(snapshot, nonce, operation, rules)
|
|
77
|
+
bytes = JSON.generate(version: 1, nonce: nonce, root: @root, output: @output,
|
|
78
|
+
operation: operation, rules: rules.fingerprint,
|
|
79
|
+
launcher_pid: Process.pid, snapshot: snapshot)
|
|
80
|
+
return bytes if bytes.bytesize <= Manifest::MAX_BYTES
|
|
81
|
+
|
|
82
|
+
@capture_unavailable = { 'reason' => 'source_manifest_too_large', 'size_bytes' => bytes.bytesize,
|
|
83
|
+
'limit_bytes' => Manifest::MAX_BYTES }
|
|
84
|
+
warn 'woods-extract: source freshness unavailable (source_manifest_too_large): ' \
|
|
85
|
+
"capture #{bytes.bytesize} bytes exceeds limit #{Manifest::MAX_BYTES}; " \
|
|
86
|
+
'continuing without verified preboot evidence.'
|
|
87
|
+
nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def child_environment(descriptor)
|
|
91
|
+
environment = { Handoff::ENV_KEY => descriptor }
|
|
92
|
+
environment['WOODS_OUTPUT'] = @output unless @implicit_output
|
|
93
|
+
if ENV['BUNDLE_GEMFILE']
|
|
94
|
+
environment['BUNDLE_GEMFILE'] = File.expand_path(ENV.fetch('BUNDLE_GEMFILE'), @original_directory)
|
|
95
|
+
end
|
|
96
|
+
environment
|
|
97
|
+
end
|
|
98
|
+
|
|
62
99
|
def parse!
|
|
63
100
|
parser.permute!(@arguments)
|
|
64
101
|
validate_arguments! unless @help
|
|
@@ -68,7 +105,10 @@ module Woods
|
|
|
68
105
|
OptionParser.new do |options|
|
|
69
106
|
options.banner = 'Usage: woods-extract [options] full | incremental PATH... | refresh TYPE...'
|
|
70
107
|
options.on('--root PATH', 'Application root (default: current directory)') { |path| @root = path }
|
|
71
|
-
options.on('--output PATH', '
|
|
108
|
+
options.on('--output PATH', 'Custom output (default: WOODS_OUTPUT or tmp/woods)') do |path|
|
|
109
|
+
@output = path
|
|
110
|
+
@implicit_output = false
|
|
111
|
+
end
|
|
72
112
|
options.on('--source-root PATH', 'Additional application-relative runtime source root') do |path|
|
|
73
113
|
@extra_roots << path
|
|
74
114
|
end
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
|
+
require 'digest'
|
|
4
5
|
|
|
5
6
|
module Woods
|
|
6
7
|
module SourceInputs
|
|
7
8
|
# Compact wire format: consumers refer to a shared identity table. Internal
|
|
8
9
|
# mutation uses expanded digests so table indices never leak across runs.
|
|
9
|
-
class Manifest
|
|
10
|
+
class Manifest # rubocop:disable Metrics/ClassLength -- compact wire validation and bounded serialization
|
|
10
11
|
FILE_NAME = 'source_inputs.json'
|
|
11
12
|
VERSION = 1
|
|
12
13
|
MAX_BYTES = 16 * 1024 * 1024
|
|
14
|
+
UNKNOWN_BASELINES = %w[invalid_source_baseline missing_or_incompatible_source_baseline
|
|
15
|
+
incomplete_source_baseline].freeze
|
|
13
16
|
class Invalid < StandardError; end
|
|
14
17
|
|
|
15
18
|
attr_reader :data
|
|
@@ -34,24 +37,73 @@ module Woods
|
|
|
34
37
|
end
|
|
35
38
|
end
|
|
36
39
|
|
|
40
|
+
# Older version-1 writers encoded missing comparison evidence only as
|
|
41
|
+
# errors. Never interpret their empty ledger as proof of an empty tree.
|
|
42
|
+
def comparison_complete?
|
|
43
|
+
@data['complete'] && @data.fetch('comparison_complete', true) &&
|
|
44
|
+
@data.fetch('errors').none? { |error| UNKNOWN_BASELINES.include?(error['reason']) }
|
|
45
|
+
end
|
|
46
|
+
|
|
37
47
|
# Explicit wire fields keep incomplete coverage separate from boot provenance.
|
|
38
48
|
# rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
39
|
-
def self.build(snapshot:, scopes:, boot_verified:, generation:, errors: [], unverified_scopes: []
|
|
49
|
+
def self.build(snapshot:, scopes:, boot_verified:, generation:, errors: [], unverified_scopes: [],
|
|
50
|
+
comparison_complete: true, unavailable: nil)
|
|
40
51
|
identities = scopes.values.flat_map(&:values).uniq.sort
|
|
41
52
|
indices = identities.each_with_index.to_h
|
|
42
53
|
new('version' => VERSION, 'generation' => generation, 'root' => snapshot.fetch('root'),
|
|
54
|
+
'captured_at' => snapshot['captured_at'],
|
|
43
55
|
'key_id' => snapshot.fetch('key_id'), 'rules' => snapshot.fetch('rules'),
|
|
44
56
|
'extra_roots' => snapshot.fetch('extra_roots'), 'boot_verified' => boot_verified,
|
|
45
57
|
'complete' => snapshot.fetch('complete'), 'errors' => (snapshot.fetch('errors') + errors).uniq,
|
|
58
|
+
'comparison_complete' => comparison_complete,
|
|
46
59
|
'unverified_scopes' => unverified_scopes.uniq.sort,
|
|
47
60
|
'identities' => identities, 'metrics' => snapshot.fetch('metrics'),
|
|
48
61
|
'scopes' => scopes.sort.to_h.transform_values do |paths|
|
|
49
62
|
paths.sort.to_h.transform_values { |digest| indices.fetch(digest) }
|
|
50
|
-
end)
|
|
63
|
+
end).tap { |manifest| manifest.bound_publication!(snapshot: snapshot, unavailable: unavailable) }
|
|
51
64
|
end
|
|
52
65
|
|
|
53
66
|
# rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
|
|
54
67
|
|
|
68
|
+
# Compact watch boundary only: this never certifies runtime freshness.
|
|
69
|
+
def self.capture_digest(files)
|
|
70
|
+
Digest::SHA256.hexdigest(JSON.generate(files.sort.to_h))
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def self.valid_unavailable_evidence?(value)
|
|
74
|
+
value.is_a?(Hash) && value['reason'] == 'source_manifest_too_large' &&
|
|
75
|
+
value.values_at('size_bytes', 'limit_bytes').all?(Integer) &&
|
|
76
|
+
value['limit_bytes'].positive? && value['size_bytes'] > value['limit_bytes']
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def unavailable?
|
|
80
|
+
@data['state'] == 'unavailable'
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Bind only an already validated, generation-local reference cache. This
|
|
84
|
+
# preserves its consumed-source proof without publishing the large ledger.
|
|
85
|
+
def bind_reference_cache!(cache)
|
|
86
|
+
@data['reference_cache_sha256'] = Digest::SHA256.hexdigest(JSON.generate(cache)) if unavailable? && cache
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Source evidence is advisory. Its size limit must not withhold a valid
|
|
90
|
+
# code index. Keep errors so final publication still refuses unstable input.
|
|
91
|
+
def bound_publication!(snapshot:, unavailable: nil)
|
|
92
|
+
size = JSON.pretty_generate(@data).bytesize
|
|
93
|
+
return unless unavailable || size > MAX_BYTES
|
|
94
|
+
|
|
95
|
+
diagnostic = unavailable || { 'reason' => 'source_manifest_too_large',
|
|
96
|
+
'size_bytes' => size, 'limit_bytes' => MAX_BYTES }
|
|
97
|
+
@data['capture_sha256'] = self.class.capture_digest(snapshot.fetch('files')) if snapshot['complete']
|
|
98
|
+
@data.merge!('state' => 'unavailable', 'unavailable' => diagnostic,
|
|
99
|
+
'complete' => false, 'comparison_complete' => false,
|
|
100
|
+
'identities' => [], 'scopes' => {}, 'metrics' => {})
|
|
101
|
+
warn "[woods] source freshness unavailable: #{diagnostic['reason']}; " \
|
|
102
|
+
"#{diagnostic['size_bytes']} bytes exceeds limit #{diagnostic['limit_bytes']}. " \
|
|
103
|
+
'Publishing the code index without freshness comparison evidence; ' \
|
|
104
|
+
'another full extraction will not reduce this size.'
|
|
105
|
+
end
|
|
106
|
+
|
|
55
107
|
private
|
|
56
108
|
|
|
57
109
|
def validate!
|
|
@@ -59,11 +111,28 @@ module Woods
|
|
|
59
111
|
raise Invalid, 'invalid_source_manifest_header' unless valid_header?
|
|
60
112
|
|
|
61
113
|
validate_tables!
|
|
114
|
+
validate_unavailable!
|
|
62
115
|
return if @data['extra_roots'].all? { |path| valid_path?(path) }
|
|
63
116
|
|
|
64
117
|
raise Invalid, 'invalid_source_roots'
|
|
65
118
|
end
|
|
66
119
|
|
|
120
|
+
def validate_unavailable!
|
|
121
|
+
return if (%w[state unavailable reference_cache_sha256 capture_sha256] & @data.keys).empty?
|
|
122
|
+
|
|
123
|
+
valid = unavailable? && self.class.valid_unavailable_evidence?(@data['unavailable']) && empty_comparison?
|
|
124
|
+
valid &&= valid_optional_digests?
|
|
125
|
+
raise Invalid, 'invalid_unavailable_source_manifest' unless valid
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def valid_optional_digests?
|
|
129
|
+
%w[reference_cache_sha256 capture_sha256].all? { |key| !@data.key?(key) || digest?(@data[key]) }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def empty_comparison?
|
|
133
|
+
!@data['complete'] && !@data['comparison_complete'] && @data['scopes'].empty? && @data['identities'].empty?
|
|
134
|
+
end
|
|
135
|
+
|
|
67
136
|
def validate_tables!
|
|
68
137
|
identities = @data['identities']
|
|
69
138
|
scopes = @data['scopes']
|
|
@@ -89,9 +158,21 @@ module Woods
|
|
|
89
158
|
|
|
90
159
|
def valid_header?
|
|
91
160
|
valid_root? && %w[key_id rules].all? { |key| digest?(@data[key]) } &&
|
|
161
|
+
valid_capture_time? && valid_comparison_coverage? &&
|
|
92
162
|
%w[boot_verified complete].all? { |key| [true, false].include?(@data[key]) }
|
|
93
163
|
end
|
|
94
164
|
|
|
165
|
+
def valid_comparison_coverage?
|
|
166
|
+
!@data.key?('comparison_complete') || [true, false].include?(@data['comparison_complete'])
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Optional so existing version-1 manifests remain readable. Consumers
|
|
170
|
+
# needing a catch-up boundary must reconcile when it is absent.
|
|
171
|
+
def valid_capture_time?
|
|
172
|
+
value = @data['captured_at']
|
|
173
|
+
value.nil? || (value.is_a?(Numeric) && value.finite? && value.positive?)
|
|
174
|
+
end
|
|
175
|
+
|
|
95
176
|
def valid_root?
|
|
96
177
|
root = @data['root']
|
|
97
178
|
root.is_a?(String) && root.start_with?('/') && !root.include?("\0")
|