woods 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +94 -7
- data/CONTRIBUTING.md +134 -19
- data/README.md +1 -1
- data/docs/AGENT_GUIDE.md +19 -0
- data/docs/AGENT_SETUP.md +22 -2
- data/docs/BACKEND_MATRIX.md +7 -0
- data/docs/CLIENT_HOOKS.md +6 -0
- data/docs/CONFIGURATION_REFERENCE.md +133 -25
- data/docs/CONSOLE_MCP_SETUP.md +82 -30
- data/docs/EMBEDDING_MODELS.md +16 -19
- data/docs/EXTRACTOR_REFERENCE.md +219 -21
- data/docs/FAQ.md +11 -25
- data/docs/GETTING_STARTED.md +7 -1
- data/docs/INCREMENTAL_EXTRACTION.md +261 -19
- data/docs/INDEX_LAYOUT.md +5 -0
- data/docs/INTERNALS.md +9 -0
- data/docs/MCP_HTTP_TRANSPORT.md +20 -15
- data/docs/MCP_SERVERS.md +87 -8
- data/docs/MCP_TOOL_COOKBOOK.md +13 -55
- data/docs/NOTION_INTEGRATION.md +7 -1
- data/docs/PUBLISHED_INDEX.md +6 -0
- data/docs/README.md +6 -1
- data/docs/RETRIEVAL_GUIDE.md +17 -0
- data/docs/SOURCE_FRESHNESS.md +157 -5
- data/docs/TOKEN_BENCHMARK.md +10 -18
- data/docs/TROUBLESHOOTING.md +70 -14
- data/docs/UNBLOCKED_INTEGRATION.md +60 -8
- data/docs/UPGRADING_TO_2.md +153 -38
- data/docs/WATCH_DAEMON.md +97 -14
- data/exe/woods-console-mcp +2 -2
- data/lib/generators/woods/templates/woods.rb.tt +2 -1
- data/lib/tasks/woods.rake +23 -7
- data/lib/tasks/woods_checks.rake +2 -2
- data/lib/woods/agent_configuration/cli.rb +1 -1
- data/lib/woods/agent_configuration/layout.rb +16 -2
- data/lib/woods/agent_configuration/plan.rb +13 -3
- data/lib/woods/agent_configuration/planner_validation.rb +4 -2
- data/lib/woods/agent_configuration/preflight.rb +5 -3
- data/lib/woods/builder.rb +17 -57
- data/lib/woods/cache/cache_middleware.rb +56 -30
- data/lib/woods/chunking/contributor_chunks.rb +119 -0
- data/lib/woods/chunking/semantic_chunker.rb +44 -21
- data/lib/woods/console/connection_manager.rb +56 -3
- data/lib/woods/console/embedded_executor.rb +30 -5
- data/lib/woods/console/rack_middleware.rb +29 -1
- data/lib/woods/dependency_graph.rb +34 -10
- data/lib/woods/embedding/fake.rb +12 -0
- data/lib/woods/embedding/indexer.rb +195 -98
- data/lib/woods/embedding/input_budget.rb +67 -0
- data/lib/woods/embedding/openai.rb +70 -20
- data/lib/woods/embedding/provider.rb +37 -25
- data/lib/woods/embedding/text_preparer.rb +76 -32
- data/lib/woods/embedding/token_counter.rb +18 -81
- data/lib/woods/embedding/vector_configuration.rb +48 -0
- data/lib/woods/extraction_identities.rb +175 -0
- data/lib/woods/extractor.rb +304 -107
- data/lib/woods/extractors/action_cable_extractor.rb +8 -3
- data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
- data/lib/woods/extractors/class_declarations.rb +121 -0
- data/lib/woods/extractors/configuration_extractor.rb +11 -3
- data/lib/woods/extractors/declaration_ancestry.rb +92 -0
- data/lib/woods/extractors/event_extractor.rb +8 -0
- data/lib/woods/extractors/graphql_extractor.rb +134 -77
- data/lib/woods/extractors/job_extractor.rb +5 -1
- data/lib/woods/extractors/lib_extractor.rb +132 -15
- data/lib/woods/extractors/mailer_extractor.rb +3 -5
- data/lib/woods/extractors/manager_extractor.rb +7 -21
- data/lib/woods/extractors/migration_declaration.rb +87 -0
- data/lib/woods/extractors/migration_extractor.rb +5 -39
- data/lib/woods/extractors/phlex_extractor.rb +6 -2
- data/lib/woods/extractors/policy_extractor.rb +9 -5
- data/lib/woods/extractors/poro_extractor.rb +112 -53
- data/lib/woods/extractors/pundit_extractor.rb +11 -6
- data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
- data/lib/woods/extractors/serializer_extractor.rb +34 -22
- data/lib/woods/extractors/shared_utility_methods.rb +18 -1
- data/lib/woods/extractors/source_nesting.rb +142 -106
- data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
- data/lib/woods/extractors/state_machine_extractor.rb +46 -40
- data/lib/woods/extractors/view_component_extractor.rb +9 -7
- data/lib/woods/flow_assembler.rb +4 -1
- data/lib/woods/generation.rb +25 -0
- data/lib/woods/hooks/context_hint.rb +7 -2
- data/lib/woods/mcp/bootstrapper.rb +33 -7
- data/lib/woods/mcp/config_resolver.rb +26 -7
- data/lib/woods/mcp/index_reader.rb +125 -24
- data/lib/woods/mcp/index_reader_pinning.rb +16 -0
- data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
- data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
- data/lib/woods/mcp/search_results.rb +7 -1
- data/lib/woods/mcp/server.rb +24 -4
- data/lib/woods/module_reconciliation.rb +151 -0
- data/lib/woods/path_dispatcher.rb +7 -2
- data/lib/woods/rake_helpers.rb +43 -11
- data/lib/woods/release.rb +1 -1
- data/lib/woods/resilience/index_validator.rb +8 -3
- data/lib/woods/resilience/retryable_provider.rb +18 -1
- data/lib/woods/resolved_config.rb +68 -8
- data/lib/woods/retrieval/context_assembler.rb +3 -3
- data/lib/woods/retrieval/lexical_assembler.rb +3 -2
- data/lib/woods/retrieval/scope.rb +18 -2
- data/lib/woods/retrieval/source_evidence.rb +14 -2
- data/lib/woods/source_contributor_validation.rb +78 -0
- data/lib/woods/source_contributors.rb +116 -0
- data/lib/woods/source_inputs/handoff.rb +37 -0
- data/lib/woods/source_inputs/launcher.rb +53 -13
- data/lib/woods/source_inputs/manifest.rb +84 -3
- data/lib/woods/source_inputs/private_key.rb +44 -12
- data/lib/woods/source_inputs/scanner.rb +98 -27
- data/lib/woods/source_inputs/scopes.rb +1 -1
- data/lib/woods/source_inputs/session.rb +147 -15
- data/lib/woods/source_inputs/stable_reader.rb +127 -0
- data/lib/woods/source_inputs/status.rb +40 -8
- data/lib/woods/source_inputs/verifier.rb +28 -5
- data/lib/woods/source_path_encoding.rb +33 -0
- data/lib/woods/source_references/cache.rb +284 -0
- data/lib/woods/source_references/collector.rb +120 -0
- data/lib/woods/source_references/extraction.rb +185 -0
- data/lib/woods/source_references/inputs.rb +134 -0
- data/lib/woods/source_references/parser_adapter.rb +134 -0
- data/lib/woods/source_references/pass.rb +152 -0
- data/lib/woods/source_references/prism_adapter.rb +116 -0
- data/lib/woods/source_references/registry.rb +178 -0
- data/lib/woods/source_references/runtime_lookup.rb +127 -0
- data/lib/woods/source_references/value_class.rb +82 -0
- data/lib/woods/storage/metadata_store.rb +4 -1
- data/lib/woods/storage/qdrant.rb +2 -2
- data/lib/woods/unblocked/client.rb +12 -7
- data/lib/woods/unblocked/document_builder.rb +4 -1
- data/lib/woods/unblocked/exporter.rb +127 -37
- data/lib/woods/unblocked/sync_manifest.rb +137 -21
- data/lib/woods/unblocked/uri_migration.rb +105 -0
- data/lib/woods/util/host_guard.rb +3 -2
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/catch_up.rb +138 -0
- data/lib/woods/watch/claim_lease.rb +150 -0
- data/lib/woods/watch/cli.rb +26 -2
- data/lib/woods/watch/daemon.rb +80 -59
- data/lib/woods/watch/installation/options.rb +1 -1
- data/lib/woods/watch/installation/receipt.rb +6 -1
- data/lib/woods/watch/managed_child.rb +1 -1
- data/lib/woods/watch/supervisor.rb +1 -1
- data/lib/woods/watch/tree_scan.rb +14 -2
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/hooks/adapters/normalize.rb +3 -2
- data/plugin/hooks/woods-input-rules.sh +4 -0
- data/plugin/hooks/woods-refresh.sh +15 -7
- data/plugin/hooks/woods-session-start.sh +60 -3
- data/plugin/skills/woods-diagnose/SKILL.md +334 -11
- data/plugin/skills/woods-investigate/SKILL.md +11 -0
- data/plugin/skills/woods-mcp-config/SKILL.md +79 -8
- data/plugin/skills/woods-setup/SKILL.md +53 -6
- metadata +32 -5
|
@@ -12,6 +12,13 @@ module Woods
|
|
|
12
12
|
BUDGETS = { 'quick' => 0.25, 'deep' => 5.0 }.freeze
|
|
13
13
|
|
|
14
14
|
def self.from_transport(encoded)
|
|
15
|
+
options = transport_options(encoded)
|
|
16
|
+
new(output_dir: options.fetch('output', ENV.fetch('WOODS_OUTPUT', 'tmp/woods')),
|
|
17
|
+
root: options.fetch('root', Dir.pwd), mode: options.fetch('mode', 'quick')).call
|
|
18
|
+
.merge('root_source' => options.key?('root') ? 'explicit' : 'working_directory')
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.transport_options(encoded)
|
|
15
22
|
raise ArgumentError, 'source status options are too large' if encoded.to_s.bytesize > 16_384
|
|
16
23
|
|
|
17
24
|
options = encoded.nil? ? {} : JSON.parse(Base64.strict_decode64(encoded))
|
|
@@ -19,20 +26,21 @@ module Woods
|
|
|
19
26
|
raise ArgumentError, 'source status options must contain only output, root and mode strings'
|
|
20
27
|
end
|
|
21
28
|
|
|
22
|
-
|
|
23
|
-
root: options['root'], mode: options.fetch('mode', 'quick')).call
|
|
29
|
+
options
|
|
24
30
|
rescue JSON::ParserError
|
|
25
31
|
raise ArgumentError, 'invalid source status options'
|
|
26
32
|
end
|
|
27
33
|
|
|
34
|
+
private_class_method :transport_options
|
|
35
|
+
|
|
28
36
|
def initialize(output_dir:, payload_dir: nil, generation: nil, root: nil, mode: 'quick')
|
|
29
37
|
raise ArgumentError, 'source status mode must be quick or deep' unless BUDGETS.key?(mode)
|
|
30
38
|
|
|
31
|
-
@output =
|
|
32
|
-
@root = root
|
|
39
|
+
@output = SourcePathEncoding.expand(output_dir)
|
|
40
|
+
@root = SourcePathEncoding.expand(root) if root
|
|
33
41
|
@mode = mode
|
|
34
42
|
@generation = generation
|
|
35
|
-
@payload = payload_dir
|
|
43
|
+
@payload = SourcePathEncoding.utf8!(payload_dir) if payload_dir
|
|
36
44
|
end
|
|
37
45
|
|
|
38
46
|
def call
|
|
@@ -46,16 +54,38 @@ module Woods
|
|
|
46
54
|
|
|
47
55
|
Manifest.parse(file.read(Manifest::MAX_BYTES + 1))
|
|
48
56
|
end
|
|
49
|
-
Verifier.new(manifest: manifest, output_dir: @output, root: @root,
|
|
50
|
-
|
|
57
|
+
result = Verifier.new(manifest: manifest, output_dir: @output, root: @root,
|
|
58
|
+
generation: @generation, max_seconds: BUDGETS.fetch(@mode)).call
|
|
59
|
+
result.merge('check' => @mode, 'recommendations' => recommendations(result, manifest))
|
|
51
60
|
rescue Errno::ENOENT
|
|
52
61
|
unknown('source_manifest_unavailable')
|
|
53
62
|
rescue Manifest::Invalid, SystemCallError, IOError
|
|
54
63
|
unknown('invalid_source_manifest')
|
|
64
|
+
rescue EncodingError, SourcePathEncoding::Invalid
|
|
65
|
+
unknown('undecodable_source_path')
|
|
55
66
|
end
|
|
56
67
|
|
|
57
68
|
private
|
|
58
69
|
|
|
70
|
+
def recommendations(result, manifest)
|
|
71
|
+
return ['inspect_source_limits'] if result['state'] == 'unavailable'
|
|
72
|
+
|
|
73
|
+
advice = []
|
|
74
|
+
reader_reasons = result.fetch('verification_reasons', result.fetch('reasons'))
|
|
75
|
+
if @mode == 'quick' && reader_reasons.include?('scan_time_budget')
|
|
76
|
+
advice << 'deep_check'
|
|
77
|
+
reader_reasons -= ['scan_time_budget']
|
|
78
|
+
end
|
|
79
|
+
advice << 'inspect_source_scan' unless reader_reasons.empty?
|
|
80
|
+
advice << 'fresh_capture' if incomplete_capture?(manifest)
|
|
81
|
+
advice
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def incomplete_capture?(manifest)
|
|
85
|
+
!manifest.comparison_complete? || !manifest.data['boot_verified'] ||
|
|
86
|
+
!manifest.data.fetch('errors').empty? || !manifest.data.fetch('unverified_scopes').empty?
|
|
87
|
+
end
|
|
88
|
+
|
|
59
89
|
def resolve_payload
|
|
60
90
|
generation = Generation.new(output_dir: @output)
|
|
61
91
|
marker = if @payload
|
|
@@ -77,7 +107,9 @@ module Woods
|
|
|
77
107
|
|
|
78
108
|
def unknown(reason)
|
|
79
109
|
{ 'state' => 'unknown', 'mode' => 'content', 'check' => @mode, 'generation' => @generation,
|
|
80
|
-
'checked_at' => Time.now.utc.iso8601, 'complete' => false, 'reasons' => [reason]
|
|
110
|
+
'checked_at' => Time.now.utc.iso8601, 'complete' => false, 'reasons' => [reason],
|
|
111
|
+
'recorded_root' => nil, 'checked_root' => @root, 'root_source' => @root ? 'explicit' : 'recorded',
|
|
112
|
+
'recommendations' => ['fresh_capture'] }
|
|
81
113
|
end
|
|
82
114
|
end
|
|
83
115
|
end
|
|
@@ -16,14 +16,27 @@ module Woods
|
|
|
16
16
|
def initialize(manifest:, output_dir:, root: nil, generation: nil, max_seconds: DEFAULT_SECONDS, **limits)
|
|
17
17
|
@manifest = manifest.is_a?(Manifest) ? manifest : Manifest.new(manifest)
|
|
18
18
|
@output_dir = output_dir
|
|
19
|
-
@root = root || @manifest.data.fetch('root')
|
|
19
|
+
@root = SourcePathEncoding.expand(root || @manifest.data.fetch('root'))
|
|
20
|
+
@root_source = root ? 'explicit' : 'recorded'
|
|
20
21
|
@generation = generation
|
|
21
22
|
@limits = limits.merge(max_seconds: max_seconds)
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
# rubocop:enable Metrics/ParameterLists
|
|
25
26
|
|
|
26
|
-
def call
|
|
27
|
+
def call
|
|
28
|
+
result = unavailable_generation? ? unavailable : verify
|
|
29
|
+
result.merge('recorded_root' => @manifest.data.fetch('root'), 'checked_root' => @root,
|
|
30
|
+
'root_source' => @root_source)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def unavailable_generation?
|
|
36
|
+
@manifest.unavailable? && (!@generation || @generation == @manifest.data['generation'])
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def verify # rubocop:disable Metrics/AbcSize -- cheap refusal checks precede the only source scan
|
|
27
40
|
return unknown('generation_mismatch') if @generation && @generation != @manifest.data['generation']
|
|
28
41
|
return unknown('source_root_unavailable') unless File.directory?(@root)
|
|
29
42
|
|
|
@@ -39,10 +52,10 @@ module Woods
|
|
|
39
52
|
unknown(e.message)
|
|
40
53
|
rescue SystemCallError, IOError
|
|
41
54
|
unknown('source_root_unavailable')
|
|
55
|
+
rescue EncodingError, SourcePathEncoding::Invalid
|
|
56
|
+
unknown('undecodable_source_path')
|
|
42
57
|
end
|
|
43
58
|
|
|
44
|
-
private
|
|
45
|
-
|
|
46
59
|
def compare(current)
|
|
47
60
|
previous = @manifest.expanded
|
|
48
61
|
changes = { 'added' => [], 'changed' => [], 'removed' => [] }
|
|
@@ -53,7 +66,7 @@ module Woods
|
|
|
53
66
|
end
|
|
54
67
|
|
|
55
68
|
def complete?(current)
|
|
56
|
-
@manifest.
|
|
69
|
+
@manifest.comparison_complete? && current['complete']
|
|
57
70
|
end
|
|
58
71
|
|
|
59
72
|
def compare_scope(scope, paths, changes, current)
|
|
@@ -76,6 +89,7 @@ module Woods
|
|
|
76
89
|
reasons = (@manifest.data.fetch('errors') + current.fetch('errors')).map { |error| error['reason'] }.compact
|
|
77
90
|
reasons << 'incomplete_capture' unless @manifest.data['complete']
|
|
78
91
|
reasons << 'incomplete_verification' unless current['complete']
|
|
92
|
+
reasons << 'incomplete_comparison_baseline' unless @manifest.comparison_complete?
|
|
79
93
|
reasons << 'unverified_boot_boundary' unless @manifest.data['boot_verified']
|
|
80
94
|
reasons << 'unverified_consumption_scopes' unless @manifest.data.fetch('unverified_scopes').empty?
|
|
81
95
|
reasons.uniq
|
|
@@ -86,7 +100,9 @@ module Woods
|
|
|
86
100
|
counts = changes.transform_values(&:size)
|
|
87
101
|
{ 'state' => state_for(counts, reasons), 'mode' => 'content', 'generation' => @manifest.data['generation'],
|
|
88
102
|
'checked_at' => Time.now.utc.iso8601, 'reasons' => reasons,
|
|
103
|
+
'verification_reasons' => current.fetch('errors').map { |error| error['reason'] }.uniq,
|
|
89
104
|
'complete' => current['complete'] && @manifest.data['complete'],
|
|
105
|
+
'comparison_complete' => @manifest.comparison_complete?,
|
|
90
106
|
'counts' => counts, 'truncated' => counts.values.any? { |count| count > SUMMARY_LIMIT },
|
|
91
107
|
'changes' => changes.transform_values { |paths| paths.first(SUMMARY_LIMIT) },
|
|
92
108
|
'metrics' => current.fetch('metrics') }
|
|
@@ -98,6 +114,13 @@ module Woods
|
|
|
98
114
|
reasons.empty? ? 'current' : 'unknown'
|
|
99
115
|
end
|
|
100
116
|
|
|
117
|
+
def unavailable
|
|
118
|
+
diagnostic = @manifest.data.fetch('unavailable')
|
|
119
|
+
{ 'state' => 'unavailable', 'mode' => 'content', 'generation' => @manifest.data['generation'],
|
|
120
|
+
'checked_at' => Time.now.utc.iso8601, 'reasons' => [diagnostic.fetch('reason')],
|
|
121
|
+
'complete' => false, 'comparison_complete' => false, 'unavailable' => diagnostic }
|
|
122
|
+
end
|
|
123
|
+
|
|
101
124
|
def unknown(reason)
|
|
102
125
|
{ 'state' => 'unknown', 'mode' => 'content', 'generation' => @manifest.data['generation'],
|
|
103
126
|
'checked_at' => Time.now.utc.iso8601, 'reasons' => [reason], 'complete' => false }
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
# Filesystem encoding tags follow the locale; published source paths use
|
|
5
|
+
# UTF-8. Validate the original bytes without transcoding or replacing them.
|
|
6
|
+
module SourcePathEncoding
|
|
7
|
+
DIAGNOSTIC_BYTES = 256
|
|
8
|
+
class Invalid < ArgumentError; end
|
|
9
|
+
|
|
10
|
+
def self.utf8(path)
|
|
11
|
+
string = path.to_s
|
|
12
|
+
string = string.dup.force_encoding(Encoding::UTF_8) unless string.encoding == Encoding::UTF_8
|
|
13
|
+
string if string.valid_encoding?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.utf8!(path)
|
|
17
|
+
utf8(path) || raise(Invalid, 'source paths must contain valid UTF-8 bytes')
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.expand(path)
|
|
21
|
+
utf8!(File.expand_path(utf8!(path)))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# An escaped byte label, never a substituted filesystem path. Bound the
|
|
25
|
+
# original bytes before escaping, so even malformed deep paths stay small.
|
|
26
|
+
def self.diagnostic(path)
|
|
27
|
+
bytes = path.to_s.b
|
|
28
|
+
label = bytes.byteslice(0, DIAGNOSTIC_BYTES).inspect
|
|
29
|
+
label += '...' if bytes.bytesize > DIAGNOSTIC_BYTES
|
|
30
|
+
label
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require_relative '../atomic_file'
|
|
5
|
+
require_relative 'registry'
|
|
6
|
+
|
|
7
|
+
module Woods
|
|
8
|
+
module SourceReferences
|
|
9
|
+
# Generation-local candidate evidence and the edges this pass added. Source
|
|
10
|
+
# bytes never belong here. Replacing the file must not mutate a hardlinked
|
|
11
|
+
# seed belonging to the previously published generation.
|
|
12
|
+
# Keep the closed schema beside its I/O boundary: neither half may accept a
|
|
13
|
+
# cache that the other half cannot safely publish or replay.
|
|
14
|
+
class Cache # rubocop:disable Metrics/ClassLength
|
|
15
|
+
class Invalid < StandardError; end
|
|
16
|
+
|
|
17
|
+
# Older JSON parsers pass pairs through #[]=; newer ones must also use
|
|
18
|
+
# allow_duplicate_key: false before constructing custom objects. A plain
|
|
19
|
+
# builder avoids optimized Hash writes and returns ordinary consumer data.
|
|
20
|
+
class UniqueObject
|
|
21
|
+
attr_reader :data
|
|
22
|
+
|
|
23
|
+
def initialize
|
|
24
|
+
@data = {}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def []=(key, value)
|
|
28
|
+
raise Invalid, 'Source reference cache contains a duplicate JSON key' if @data.key?(key)
|
|
29
|
+
|
|
30
|
+
@data[key] = unwrap(value)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def unwrap(value)
|
|
34
|
+
case value
|
|
35
|
+
when UniqueObject then value.data
|
|
36
|
+
when Array then value.map { |item| unwrap(item) }
|
|
37
|
+
else value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
private_constant :UniqueObject
|
|
42
|
+
|
|
43
|
+
FILE_NAME = 'source_references.json'
|
|
44
|
+
VERSION = 3
|
|
45
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
46
|
+
MAX_FILES = 100_000
|
|
47
|
+
MAX_OWNERS = 200_000
|
|
48
|
+
MAX_RECORDS = 1_000_000
|
|
49
|
+
MAX_NESTING = 128
|
|
50
|
+
MAX_STRING_BYTES = 4096
|
|
51
|
+
MAX_LINE = 1_000_000_000
|
|
52
|
+
IDENTITY = /\A[0-9a-f]{64}\z/
|
|
53
|
+
TYPES = Registry::TYPES.map(&:to_s).freeze
|
|
54
|
+
SKIP_REASONS = %w[dynamic_declaration dynamic_superclass dynamic_constant_path unowned_reference
|
|
55
|
+
dynamic_singleton_scope dynamic_method_owner].freeze
|
|
56
|
+
|
|
57
|
+
class << self
|
|
58
|
+
# @param path [String, Pathname] cache in one generation payload
|
|
59
|
+
# @return [Hash, nil] validated evidence, or nil only when absent
|
|
60
|
+
# @raise [Invalid] unreadable, unsupported or invalid cache
|
|
61
|
+
def read(path)
|
|
62
|
+
return unless regular_file?(path)
|
|
63
|
+
|
|
64
|
+
data = JSON.parse(read_bytes(path), object_class: UniqueObject, allow_duplicate_key: false)
|
|
65
|
+
data = data.data if data.is_a?(UniqueObject)
|
|
66
|
+
validate!(data)
|
|
67
|
+
data
|
|
68
|
+
rescue JSON::ParserError, EncodingError, SystemCallError, IOError => e
|
|
69
|
+
raise Invalid, "Cannot read source reference cache: #{e.class}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @param path [String, Pathname] destination within the new payload
|
|
73
|
+
# @param data [Hash] versioned candidate and pass-owned edge evidence
|
|
74
|
+
# @return [void]
|
|
75
|
+
# @raise [Invalid] unsupported or invalid cache; previous bytes survive
|
|
76
|
+
def write(path, data)
|
|
77
|
+
validate!(data)
|
|
78
|
+
bytes = JSON.generate(data)
|
|
79
|
+
raise Invalid, 'Source reference cache exceeds size limit' if bytes.bytesize > MAX_BYTES
|
|
80
|
+
|
|
81
|
+
regular_file?(path)
|
|
82
|
+
AtomicFile.write(path, bytes)
|
|
83
|
+
rescue JSON::GeneratorError, EncodingError => e
|
|
84
|
+
raise Invalid, "Cannot encode source reference cache: #{e.class}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
def regular_file?(path)
|
|
90
|
+
stat = File.lstat(path)
|
|
91
|
+
raise Invalid, 'Source reference cache must be a regular file, not a symlink' unless stat.file?
|
|
92
|
+
|
|
93
|
+
true
|
|
94
|
+
rescue Errno::ENOENT
|
|
95
|
+
false
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def read_bytes(path)
|
|
99
|
+
flags = File::RDONLY | File::NONBLOCK
|
|
100
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
101
|
+
File.open(path, flags) do |file|
|
|
102
|
+
raise Invalid, 'Source reference cache must be a regular file' unless file.stat.file?
|
|
103
|
+
raise Invalid, 'Source reference cache exceeds size limit' if file.stat.size > MAX_BYTES
|
|
104
|
+
|
|
105
|
+
bytes = (file.read(MAX_BYTES + 1) || '').force_encoding(Encoding::UTF_8)
|
|
106
|
+
raise Invalid, 'Source reference cache exceeds size limit' if bytes.bytesize > MAX_BYTES
|
|
107
|
+
raise Invalid, 'Source reference cache is not UTF-8' unless bytes.valid_encoding?
|
|
108
|
+
|
|
109
|
+
bytes
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def validate!(data)
|
|
114
|
+
shape!(data, %w[version files owners])
|
|
115
|
+
require!(data['version'].instance_of?(Integer) && data['version'] == VERSION, 'unsupported version')
|
|
116
|
+
files = data['files']
|
|
117
|
+
require!(files.is_a?(Hash) && files.size <= MAX_FILES, 'invalid files or file limit')
|
|
118
|
+
records = 0
|
|
119
|
+
files.each { |path, record| records = bounded_count(records + file!(path, record)) }
|
|
120
|
+
owners!(data['owners'], files, records)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def file!(path, record)
|
|
124
|
+
path!(path)
|
|
125
|
+
shape!(record, %w[identity analysis])
|
|
126
|
+
require!(record['identity'].is_a?(String) && IDENTITY.match?(record['identity']), 'invalid identity')
|
|
127
|
+
analysis!(record['analysis'])
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def analysis!(analysis)
|
|
131
|
+
shape!(analysis, %w[declarations references skipped parse_error])
|
|
132
|
+
declarations = records!(analysis['declarations'])
|
|
133
|
+
references = records!(analysis['references'])
|
|
134
|
+
skipped = records!(analysis['skipped'])
|
|
135
|
+
total = declarations.size + references.size + skipped.size
|
|
136
|
+
require!(total <= MAX_RECORDS, 'candidate record limit exceeded')
|
|
137
|
+
declarations.each { |record| declaration!(record) }
|
|
138
|
+
references.each { |record| reference!(record) }
|
|
139
|
+
skipped.each { |record| skipped!(record) }
|
|
140
|
+
parse_error!(analysis['parse_error'], total)
|
|
141
|
+
total
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def declaration!(record)
|
|
145
|
+
shape!(record, %w[owner name kind nesting enclosing_nesting line end_line], %w[singleton_depth constructor])
|
|
146
|
+
lexical_record!(record)
|
|
147
|
+
if record.key?('constructor')
|
|
148
|
+
require!(record['kind'] == 'class' && %w[Struct ::Struct Data ::Data].include?(record['constructor']),
|
|
149
|
+
'invalid value-class constructor')
|
|
150
|
+
end
|
|
151
|
+
require!(%w[class module].include?(record['kind']), 'invalid declaration kind')
|
|
152
|
+
nesting!(record['enclosing_nesting'])
|
|
153
|
+
line!(record['end_line'])
|
|
154
|
+
require!(record['end_line'] >= record['line'], 'invalid declaration range')
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def reference!(record)
|
|
158
|
+
shape!(record, %w[owner name nesting line], %w[singleton_depth])
|
|
159
|
+
lexical_record!(record)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def lexical_record!(record)
|
|
163
|
+
constant!(record['owner'])
|
|
164
|
+
constant!(record['name'], rooted: true)
|
|
165
|
+
nesting!(record['nesting'])
|
|
166
|
+
require!(record['nesting'].first == record['owner'], 'owner disagrees with lexical nesting')
|
|
167
|
+
line!(record['line'])
|
|
168
|
+
return unless record.key?('singleton_depth')
|
|
169
|
+
|
|
170
|
+
depth = record['singleton_depth']
|
|
171
|
+
require!(depth.instance_of?(Integer) && depth.between?(0, MAX_NESTING), 'invalid singleton depth')
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def skipped!(record)
|
|
175
|
+
shape!(record, %w[reason owner line])
|
|
176
|
+
require!(SKIP_REASONS.include?(record['reason']), 'invalid skip reason')
|
|
177
|
+
constant!(record['owner']) unless record['owner'].nil?
|
|
178
|
+
line!(record['line'])
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def parse_error!(error, total)
|
|
182
|
+
return if error.nil?
|
|
183
|
+
|
|
184
|
+
shape!(error, %w[message line])
|
|
185
|
+
string!(error['message'])
|
|
186
|
+
line!(error['line'])
|
|
187
|
+
require!(total.zero?, 'parse failure contains partial records')
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def owners!(owners, files, count)
|
|
191
|
+
require!(owners.is_a?(Array) && owners.size <= MAX_OWNERS, 'invalid owners or owner limit')
|
|
192
|
+
seen = {}
|
|
193
|
+
owners.each do |owner|
|
|
194
|
+
count = bounded_count(count + owner!(owner, files))
|
|
195
|
+
identity = owner.values_at('type', 'identifier')
|
|
196
|
+
require!(!seen.key?(identity), 'duplicate owner identity')
|
|
197
|
+
seen[identity] = true
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def owner!(owner, files)
|
|
202
|
+
shape!(owner, %w[type identifier file_path added], %w[file_paths])
|
|
203
|
+
type!(owner['type'])
|
|
204
|
+
constant!(owner['identifier'])
|
|
205
|
+
path!(owner['file_path'])
|
|
206
|
+
require!(files.key?(owner['file_path']), 'owner source is absent from files')
|
|
207
|
+
contributor_paths!(owner, files) if owner.key?('file_paths')
|
|
208
|
+
edges = records!(owner['added'])
|
|
209
|
+
edges.each { |edge| edge!(edge) }
|
|
210
|
+
require!(edges.uniq.size == edges.size, 'duplicate pass-owned edge')
|
|
211
|
+
edges.size
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def contributor_paths!(owner, files)
|
|
215
|
+
paths = owner['file_paths']
|
|
216
|
+
require!(paths.is_a?(Array) && paths.size > 1 && paths.uniq == paths &&
|
|
217
|
+
paths.first == owner['file_path'], 'invalid owner contributors')
|
|
218
|
+
paths.each do |path|
|
|
219
|
+
path!(path)
|
|
220
|
+
require!(files.key?(path), 'owner contributor is absent from files')
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def edge!(edge)
|
|
225
|
+
shape!(edge, %w[type target via])
|
|
226
|
+
type!(edge['type'])
|
|
227
|
+
constant!(edge['target'])
|
|
228
|
+
require!(edge['via'] == 'code_reference', 'invalid pass-owned edge label')
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def type!(type)
|
|
232
|
+
require!(TYPES.include?(type), 'invalid constant-owned unit type')
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def shape!(value, required, optional = [])
|
|
236
|
+
require!(value.is_a?(Hash), 'invalid record shape')
|
|
237
|
+
require!((required - value.keys).empty? && (value.keys - required - optional).empty?, 'invalid record keys')
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def records!(value)
|
|
241
|
+
require!(value.is_a?(Array) && value.size <= MAX_RECORDS, 'invalid candidate array or record limit')
|
|
242
|
+
value
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def bounded_count(count)
|
|
246
|
+
require!(count <= MAX_RECORDS, 'candidate record limit exceeded')
|
|
247
|
+
count
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def nesting!(value)
|
|
251
|
+
require!(value.is_a?(Array) && value.size <= MAX_NESTING, 'invalid lexical nesting')
|
|
252
|
+
value.each { |name| constant!(name) }
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def constant!(value, rooted: false)
|
|
256
|
+
string!(value)
|
|
257
|
+
valid = RuntimeLookup::CONSTANT.match?(value) && (rooted || !value.start_with?('::'))
|
|
258
|
+
require!(valid, 'invalid constant name')
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def path!(value)
|
|
262
|
+
string!(value)
|
|
263
|
+
segments = value.split('/', -1)
|
|
264
|
+
valid = %w[app lib].include?(segments.first) && segments.size >= 2 && value.end_with?('.rb') &&
|
|
265
|
+
!value.include?('\\') && segments.none? { |part| part.empty? || %w[. ..].include?(part) }
|
|
266
|
+
require!(valid, 'unsafe source path')
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def string!(value)
|
|
270
|
+
require!(value.is_a?(String) && value.valid_encoding? && !value.empty? &&
|
|
271
|
+
value.bytesize <= MAX_STRING_BYTES && !value.match?(/[\x00-\x1f\x7f]/), 'invalid evidence string')
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def line!(value)
|
|
275
|
+
require!(value.instance_of?(Integer) && value.between?(1, MAX_LINE), 'invalid source line')
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def require!(condition, message)
|
|
279
|
+
raise Invalid, "Invalid source reference cache: #{message}" unless condition
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'prism_adapter'
|
|
4
|
+
require_relative 'parser_adapter'
|
|
5
|
+
|
|
6
|
+
module Woods
|
|
7
|
+
module SourceReferences
|
|
8
|
+
# Collects source constant reads without resolving or executing Ruby objects.
|
|
9
|
+
# Owners are declaration candidates, not proof of runtime ownership. In
|
|
10
|
+
# particular, a nested qualified declaration must be verified by the registry.
|
|
11
|
+
# Lexical nesting is innermost first and preserves compact namespace syntax.
|
|
12
|
+
# References/declarations inside class << self carry singleton_depth; resolving
|
|
13
|
+
# them requires the singleton constant table, not ordinary owner lookup.
|
|
14
|
+
class Collector
|
|
15
|
+
HANDLERS = { declaration: :declaration, value_class: :value_class, constant: :reference,
|
|
16
|
+
singleton: :singleton, method: :method_body }.freeze
|
|
17
|
+
# @param backend [Symbol] :prism (default) or :parser
|
|
18
|
+
# @raise [ArgumentError] for an unsupported backend
|
|
19
|
+
def initialize(backend: :prism)
|
|
20
|
+
@adapter = case backend
|
|
21
|
+
when :prism then PrismAdapter.new
|
|
22
|
+
when :parser then ParserAdapter.new
|
|
23
|
+
else raise ArgumentError, "Unsupported source reference backend: #{backend}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Parse original source and return JSON-compatible candidate evidence.
|
|
28
|
+
# Parse failures return no partial records. Unsupported ownership has an
|
|
29
|
+
# explicit skip reason; declaration ranges are inclusive source lines.
|
|
30
|
+
#
|
|
31
|
+
# @param source [String] original Ruby source
|
|
32
|
+
# @return [Hash] declarations, references, skipped records and parse_error
|
|
33
|
+
def call(source)
|
|
34
|
+
@result = { 'declarations' => [], 'references' => [], 'skipped' => [], 'parse_error' => nil }
|
|
35
|
+
root, error = @adapter.parse(source)
|
|
36
|
+
@result['parse_error'] = error
|
|
37
|
+
visit(root, []) unless error
|
|
38
|
+
@result
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def visit(node, nesting, singleton_depth = 0)
|
|
44
|
+
return unless node
|
|
45
|
+
|
|
46
|
+
kind = @adapter.kind(node)
|
|
47
|
+
return skip(node, 'dynamic_declaration', nesting) if kind == :dynamic_scope
|
|
48
|
+
|
|
49
|
+
handler = HANDLERS[kind]
|
|
50
|
+
handler ? send(handler, node, nesting, singleton_depth) : visit_children(node, nesting, singleton_depth)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def visit_children(node, nesting, singleton_depth)
|
|
54
|
+
@adapter.children(node).each { |child| visit(child, nesting, singleton_depth) }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def declaration(node, nesting, singleton_depth)
|
|
58
|
+
name = @adapter.declaration_name(node)
|
|
59
|
+
return skip(node, 'dynamic_declaration', nesting) unless name
|
|
60
|
+
|
|
61
|
+
owner = name.start_with?('::') ? name.delete_prefix('::') : [nesting.first, name].compact.join('::')
|
|
62
|
+
inner = [owner, *nesting]
|
|
63
|
+
record = declaration_record(node, name, inner, nesting)
|
|
64
|
+
record['singleton_depth'] = singleton_depth if singleton_depth.positive?
|
|
65
|
+
@result['declarations'] << record
|
|
66
|
+
parent = @adapter.superclass(node)
|
|
67
|
+
skip(parent, 'dynamic_superclass', inner) if parent && !@adapter.constant_name(parent)
|
|
68
|
+
visit(@adapter.body(node), inner, singleton_depth)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def value_class(node, nesting, singleton_depth)
|
|
72
|
+
name = @adapter.assignment_name(node)
|
|
73
|
+
return skip(node, 'dynamic_declaration', nesting) unless name
|
|
74
|
+
|
|
75
|
+
owner = name.start_with?('::') ? name.delete_prefix('::') : [nesting.first, name].compact.join('::')
|
|
76
|
+
record = { 'owner' => owner, 'name' => name, 'kind' => 'class',
|
|
77
|
+
'constructor' => @adapter.value_constructor(node), 'nesting' => [owner, *nesting],
|
|
78
|
+
'enclosing_nesting' => nesting,
|
|
79
|
+
'line' => @adapter.line(node), 'end_line' => @adapter.end_line(node) }
|
|
80
|
+
record['singleton_depth'] = singleton_depth if singleton_depth.positive?
|
|
81
|
+
@result['declarations'] << record
|
|
82
|
+
# Struct/Data blocks execute with a dynamic receiver; assignment does not
|
|
83
|
+
# establish lexical nesting. Keep their bodies explicitly unsupported.
|
|
84
|
+
visit_children(node, nesting, singleton_depth)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def declaration_record(node, name, nesting, enclosing)
|
|
88
|
+
{ 'owner' => nesting.first, 'name' => name, 'kind' => @adapter.declaration_kind(node),
|
|
89
|
+
'nesting' => nesting, 'enclosing_nesting' => enclosing,
|
|
90
|
+
'line' => @adapter.line(node), 'end_line' => @adapter.end_line(node) }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def reference(node, nesting, singleton_depth)
|
|
94
|
+
name = @adapter.constant_name(node)
|
|
95
|
+
return skip(node, 'dynamic_constant_path', nesting) unless name
|
|
96
|
+
return skip(node, 'unowned_reference', nesting) if nesting.empty?
|
|
97
|
+
|
|
98
|
+
record = { 'owner' => nesting.first, 'nesting' => nesting, 'name' => name, 'line' => @adapter.line(node) }
|
|
99
|
+
record['singleton_depth'] = singleton_depth if singleton_depth.positive?
|
|
100
|
+
@result['references'] << record
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def singleton(node, nesting, singleton_depth)
|
|
104
|
+
return skip(node, 'dynamic_singleton_scope', nesting) unless @adapter.self_receiver?(node) && nesting.any?
|
|
105
|
+
|
|
106
|
+
visit(@adapter.body(node), nesting, singleton_depth + 1)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def method_body(node, nesting, singleton_depth)
|
|
110
|
+
return skip(node, 'dynamic_method_owner', nesting) unless @adapter.local_method?(node)
|
|
111
|
+
|
|
112
|
+
visit_children(node, nesting, singleton_depth)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def skip(node, reason, nesting)
|
|
116
|
+
@result['skipped'] << { 'reason' => reason, 'owner' => nesting.first, 'line' => @adapter.line(node) }
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|