woods 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +94 -7
- data/CONTRIBUTING.md +134 -19
- data/README.md +1 -1
- data/docs/AGENT_GUIDE.md +19 -0
- data/docs/AGENT_SETUP.md +22 -2
- data/docs/BACKEND_MATRIX.md +7 -0
- data/docs/CLIENT_HOOKS.md +6 -0
- data/docs/CONFIGURATION_REFERENCE.md +133 -25
- data/docs/CONSOLE_MCP_SETUP.md +82 -30
- data/docs/EMBEDDING_MODELS.md +16 -19
- data/docs/EXTRACTOR_REFERENCE.md +219 -21
- data/docs/FAQ.md +11 -25
- data/docs/GETTING_STARTED.md +7 -1
- data/docs/INCREMENTAL_EXTRACTION.md +261 -19
- data/docs/INDEX_LAYOUT.md +5 -0
- data/docs/INTERNALS.md +9 -0
- data/docs/MCP_HTTP_TRANSPORT.md +20 -15
- data/docs/MCP_SERVERS.md +87 -8
- data/docs/MCP_TOOL_COOKBOOK.md +13 -55
- data/docs/NOTION_INTEGRATION.md +7 -1
- data/docs/PUBLISHED_INDEX.md +6 -0
- data/docs/README.md +6 -1
- data/docs/RETRIEVAL_GUIDE.md +17 -0
- data/docs/SOURCE_FRESHNESS.md +157 -5
- data/docs/TOKEN_BENCHMARK.md +10 -18
- data/docs/TROUBLESHOOTING.md +70 -14
- data/docs/UNBLOCKED_INTEGRATION.md +60 -8
- data/docs/UPGRADING_TO_2.md +153 -38
- data/docs/WATCH_DAEMON.md +97 -14
- data/exe/woods-console-mcp +2 -2
- data/lib/generators/woods/templates/woods.rb.tt +2 -1
- data/lib/tasks/woods.rake +23 -7
- data/lib/tasks/woods_checks.rake +2 -2
- data/lib/woods/agent_configuration/cli.rb +1 -1
- data/lib/woods/agent_configuration/layout.rb +16 -2
- data/lib/woods/agent_configuration/plan.rb +13 -3
- data/lib/woods/agent_configuration/planner_validation.rb +4 -2
- data/lib/woods/agent_configuration/preflight.rb +5 -3
- data/lib/woods/builder.rb +17 -57
- data/lib/woods/cache/cache_middleware.rb +56 -30
- data/lib/woods/chunking/contributor_chunks.rb +119 -0
- data/lib/woods/chunking/semantic_chunker.rb +44 -21
- data/lib/woods/console/connection_manager.rb +56 -3
- data/lib/woods/console/embedded_executor.rb +30 -5
- data/lib/woods/console/rack_middleware.rb +29 -1
- data/lib/woods/dependency_graph.rb +34 -10
- data/lib/woods/embedding/fake.rb +12 -0
- data/lib/woods/embedding/indexer.rb +195 -98
- data/lib/woods/embedding/input_budget.rb +67 -0
- data/lib/woods/embedding/openai.rb +70 -20
- data/lib/woods/embedding/provider.rb +37 -25
- data/lib/woods/embedding/text_preparer.rb +76 -32
- data/lib/woods/embedding/token_counter.rb +18 -81
- data/lib/woods/embedding/vector_configuration.rb +48 -0
- data/lib/woods/extraction_identities.rb +175 -0
- data/lib/woods/extractor.rb +304 -107
- data/lib/woods/extractors/action_cable_extractor.rb +8 -3
- data/lib/woods/extractors/assigned_value_discovery.rb +74 -0
- data/lib/woods/extractors/class_declarations.rb +121 -0
- data/lib/woods/extractors/configuration_extractor.rb +11 -3
- data/lib/woods/extractors/declaration_ancestry.rb +92 -0
- data/lib/woods/extractors/event_extractor.rb +8 -0
- data/lib/woods/extractors/graphql_extractor.rb +134 -77
- data/lib/woods/extractors/job_extractor.rb +5 -1
- data/lib/woods/extractors/lib_extractor.rb +132 -15
- data/lib/woods/extractors/mailer_extractor.rb +3 -5
- data/lib/woods/extractors/manager_extractor.rb +7 -21
- data/lib/woods/extractors/migration_declaration.rb +87 -0
- data/lib/woods/extractors/migration_extractor.rb +5 -39
- data/lib/woods/extractors/phlex_extractor.rb +6 -2
- data/lib/woods/extractors/policy_extractor.rb +9 -5
- data/lib/woods/extractors/poro_extractor.rb +112 -53
- data/lib/woods/extractors/pundit_extractor.rb +11 -6
- data/lib/woods/extractors/scheduled_job_extractor.rb +45 -4
- data/lib/woods/extractors/serializer_extractor.rb +34 -22
- data/lib/woods/extractors/shared_utility_methods.rb +18 -1
- data/lib/woods/extractors/source_nesting.rb +142 -106
- data/lib/woods/extractors/standalone_module_discovery.rb +123 -0
- data/lib/woods/extractors/state_machine_extractor.rb +46 -40
- data/lib/woods/extractors/view_component_extractor.rb +9 -7
- data/lib/woods/flow_assembler.rb +4 -1
- data/lib/woods/generation.rb +25 -0
- data/lib/woods/hooks/context_hint.rb +7 -2
- data/lib/woods/mcp/bootstrapper.rb +33 -7
- data/lib/woods/mcp/config_resolver.rb +26 -7
- data/lib/woods/mcp/index_reader.rb +125 -24
- data/lib/woods/mcp/index_reader_pinning.rb +16 -0
- data/lib/woods/mcp/renderers/markdown_renderer.rb +7 -1
- data/lib/woods/mcp/renderers/plain_renderer.rb +3 -1
- data/lib/woods/mcp/search_results.rb +7 -1
- data/lib/woods/mcp/server.rb +24 -4
- data/lib/woods/module_reconciliation.rb +151 -0
- data/lib/woods/path_dispatcher.rb +7 -2
- data/lib/woods/rake_helpers.rb +43 -11
- data/lib/woods/release.rb +1 -1
- data/lib/woods/resilience/index_validator.rb +8 -3
- data/lib/woods/resilience/retryable_provider.rb +18 -1
- data/lib/woods/resolved_config.rb +68 -8
- data/lib/woods/retrieval/context_assembler.rb +3 -3
- data/lib/woods/retrieval/lexical_assembler.rb +3 -2
- data/lib/woods/retrieval/scope.rb +18 -2
- data/lib/woods/retrieval/source_evidence.rb +14 -2
- data/lib/woods/source_contributor_validation.rb +78 -0
- data/lib/woods/source_contributors.rb +116 -0
- data/lib/woods/source_inputs/handoff.rb +37 -0
- data/lib/woods/source_inputs/launcher.rb +53 -13
- data/lib/woods/source_inputs/manifest.rb +84 -3
- data/lib/woods/source_inputs/private_key.rb +44 -12
- data/lib/woods/source_inputs/scanner.rb +98 -27
- data/lib/woods/source_inputs/scopes.rb +1 -1
- data/lib/woods/source_inputs/session.rb +147 -15
- data/lib/woods/source_inputs/stable_reader.rb +127 -0
- data/lib/woods/source_inputs/status.rb +40 -8
- data/lib/woods/source_inputs/verifier.rb +28 -5
- data/lib/woods/source_path_encoding.rb +33 -0
- data/lib/woods/source_references/cache.rb +284 -0
- data/lib/woods/source_references/collector.rb +120 -0
- data/lib/woods/source_references/extraction.rb +185 -0
- data/lib/woods/source_references/inputs.rb +134 -0
- data/lib/woods/source_references/parser_adapter.rb +134 -0
- data/lib/woods/source_references/pass.rb +152 -0
- data/lib/woods/source_references/prism_adapter.rb +116 -0
- data/lib/woods/source_references/registry.rb +178 -0
- data/lib/woods/source_references/runtime_lookup.rb +127 -0
- data/lib/woods/source_references/value_class.rb +82 -0
- data/lib/woods/storage/metadata_store.rb +4 -1
- data/lib/woods/storage/qdrant.rb +2 -2
- data/lib/woods/unblocked/client.rb +12 -7
- data/lib/woods/unblocked/document_builder.rb +4 -1
- data/lib/woods/unblocked/exporter.rb +127 -37
- data/lib/woods/unblocked/sync_manifest.rb +137 -21
- data/lib/woods/unblocked/uri_migration.rb +105 -0
- data/lib/woods/util/host_guard.rb +3 -2
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/catch_up.rb +138 -0
- data/lib/woods/watch/claim_lease.rb +150 -0
- data/lib/woods/watch/cli.rb +26 -2
- data/lib/woods/watch/daemon.rb +80 -59
- data/lib/woods/watch/installation/options.rb +1 -1
- data/lib/woods/watch/installation/receipt.rb +6 -1
- data/lib/woods/watch/managed_child.rb +1 -1
- data/lib/woods/watch/supervisor.rb +1 -1
- data/lib/woods/watch/tree_scan.rb +14 -2
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/hooks/adapters/normalize.rb +3 -2
- data/plugin/hooks/woods-input-rules.sh +4 -0
- data/plugin/hooks/woods-refresh.sh +15 -7
- data/plugin/hooks/woods-session-start.sh +60 -3
- data/plugin/skills/woods-diagnose/SKILL.md +334 -11
- data/plugin/skills/woods-investigate/SKILL.md +11 -0
- data/plugin/skills/woods-mcp-config/SKILL.md +79 -8
- data/plugin/skills/woods-setup/SKILL.md +53 -6
- metadata +32 -5
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'woods/generation'
|
|
4
|
+
require 'woods/source_inputs/manifest'
|
|
5
|
+
require 'woods/source_inputs/scanner'
|
|
6
|
+
require 'woods/watch/tree_scan'
|
|
7
|
+
|
|
8
|
+
module Woods
|
|
9
|
+
module Watch
|
|
10
|
+
# Generation-bound startup evidence. Publication time is too late: an edit
|
|
11
|
+
# can land after capture, or even after final verification, before the marker.
|
|
12
|
+
class CatchUp
|
|
13
|
+
DIRTY_REASONS = %w[source_changed_during_extraction source_changed_during_read].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(root:, output_dir:, ignored:)
|
|
16
|
+
@root = root
|
|
17
|
+
@output = output_dir
|
|
18
|
+
@ignored = ignored
|
|
19
|
+
@generation = Generation.new(output_dir: output_dir)
|
|
20
|
+
@marker = @generation.current
|
|
21
|
+
@manifest = read_manifest
|
|
22
|
+
report_unavailable if @manifest&.unavailable?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# A full run records the missing boundary even when incremental discovery
|
|
26
|
+
# would find no units to change (and therefore publish no new generation).
|
|
27
|
+
def rebuild?
|
|
28
|
+
@marker.number.positive? && @manifest.nil?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def paths
|
|
32
|
+
files = TreeScan.files(root: @root, ignored: @ignored)
|
|
33
|
+
return files unless @manifest
|
|
34
|
+
|
|
35
|
+
candidates = files.select { |path| recent?(path) } | dirty_paths
|
|
36
|
+
return [] if candidates.empty?
|
|
37
|
+
|
|
38
|
+
current = current_identities
|
|
39
|
+
return [] if captured_tree_matches?(current)
|
|
40
|
+
|
|
41
|
+
expected = recorded_identities
|
|
42
|
+
candidates.reject { |path| covered?(path, current, expected) }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def report_unavailable
|
|
48
|
+
evidence = @manifest.data.fetch('unavailable')
|
|
49
|
+
warn "[Woods] Watch source freshness unavailable: #{evidence.fetch('reason')}; " \
|
|
50
|
+
"#{evidence.fetch('size_bytes')} bytes; limit #{evidence.fetch('limit_bytes')}. " \
|
|
51
|
+
'Continuing source-change catch-up without a freshness comparison ledger.'
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def captured_tree_matches?(current)
|
|
55
|
+
@manifest.unavailable? && @current_complete &&
|
|
56
|
+
@manifest.data['capture_sha256'] == SourceInputs::Manifest.capture_digest(current)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def covered?(path, current, expected)
|
|
60
|
+
relative = path.delete_prefix("#{@root}/")
|
|
61
|
+
identities = expected[relative]
|
|
62
|
+
identities && current[relative] && identities.all? { |identity| identity == current[relative] }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def read_manifest
|
|
66
|
+
payload = @generation.payload_dir(@marker)
|
|
67
|
+
return nil if @marker.payload && payload == @generation.root
|
|
68
|
+
|
|
69
|
+
manifest = load_manifest(payload)
|
|
70
|
+
data = manifest.data
|
|
71
|
+
return nil unless matching_manifest?(data)
|
|
72
|
+
|
|
73
|
+
@key = SourceInputs::PrivateKey.new(output_dir: @output)
|
|
74
|
+
@scopes = SourceInputs::Scopes.new(extra_roots: data.fetch('extra_roots'))
|
|
75
|
+
return nil unless @key.identifier == data['key_id'] && @scopes.fingerprint == data['rules']
|
|
76
|
+
|
|
77
|
+
manifest
|
|
78
|
+
rescue SourceInputs::Manifest::Invalid, SourceInputs::PrivateKey::Unavailable, SystemCallError, IOError, TypeError
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def matching_manifest?(data)
|
|
83
|
+
data['root'] == @root && data['generation'] == @marker.number &&
|
|
84
|
+
data['captured_at'] && data['captured_at'] <= Time.now.to_f
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def load_manifest(payload)
|
|
88
|
+
flags = File::RDONLY | File::NONBLOCK
|
|
89
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
90
|
+
File.open(payload.join(SourceInputs::Manifest::FILE_NAME), flags) do |file|
|
|
91
|
+
unless file.stat.file? && file.stat.size <= SourceInputs::Manifest::MAX_BYTES
|
|
92
|
+
raise SourceInputs::Manifest::Invalid, 'invalid_source_manifest'
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
SourceInputs::Manifest.parse(file.read(SourceInputs::Manifest::MAX_BYTES + 1))
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def recent?(path)
|
|
100
|
+
# Filesystems may record only whole seconds. Include the entire capture
|
|
101
|
+
# second; content comparison below prevents unchanged files repeating.
|
|
102
|
+
File.mtime(path).to_f >= @manifest.data.fetch('captured_at').floor
|
|
103
|
+
rescue SystemCallError
|
|
104
|
+
false
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def dirty_paths
|
|
108
|
+
@manifest.data.fetch('errors').filter_map do |error|
|
|
109
|
+
next unless DIRTY_REASONS.include?(error['reason'])
|
|
110
|
+
|
|
111
|
+
path = error['path']
|
|
112
|
+
next unless relative_path?(path)
|
|
113
|
+
|
|
114
|
+
File.join(@root, path)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def relative_path?(path)
|
|
119
|
+
path.is_a?(String) && !path.empty? && !path.start_with?('/') && !path.include?("\0") &&
|
|
120
|
+
path.split('/', -1).none? { |part| ['', '.', '..'].include?(part) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def recorded_identities
|
|
124
|
+
@manifest.expanded.each_value.with_object({}) do |paths, result|
|
|
125
|
+
paths.each { |path, identity| (result[path] ||= []) << identity }
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def current_identities
|
|
130
|
+
snapshot = SourceInputs::Scanner.new(root: @root, output_dir: @output, key: @key, scopes: @scopes).call
|
|
131
|
+
@current_complete = snapshot['complete']
|
|
132
|
+
snapshot.fetch('files')
|
|
133
|
+
rescue SystemCallError, IOError
|
|
134
|
+
{}
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'securerandom'
|
|
5
|
+
|
|
6
|
+
module Woods
|
|
7
|
+
module Watch
|
|
8
|
+
# The daemon, not its launcher, holds this descriptor for its whole run.
|
|
9
|
+
# Sidecar inodes are permanent: unlinking a locked file defeats flock.
|
|
10
|
+
class ClaimLease
|
|
11
|
+
class Unavailable < StandardError; end
|
|
12
|
+
CLAIM = 'watch_claim.json'
|
|
13
|
+
MAX_BYTES = 4096
|
|
14
|
+
|
|
15
|
+
attr_reader :token
|
|
16
|
+
|
|
17
|
+
# @param index [String] existing index directory
|
|
18
|
+
def initialize(index)
|
|
19
|
+
@index = File.realpath(index)
|
|
20
|
+
@token = SecureRandom.hex(32)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @yield while serialized with daemon claim creation and retirement
|
|
24
|
+
def coordinate
|
|
25
|
+
lock = open_lock("#{CLAIM}.lock", create: true)
|
|
26
|
+
yield
|
|
27
|
+
ensure
|
|
28
|
+
lock&.close
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @param create [Boolean] only a new daemon can create the permanent lease
|
|
32
|
+
# @return [void]
|
|
33
|
+
def acquire(create: true)
|
|
34
|
+
@lease = open_lock("#{CLAIM}.lease", create: create)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @return [Hash] identity bound to this exact locked inode
|
|
38
|
+
def fields
|
|
39
|
+
stat = @lease.stat
|
|
40
|
+
{ lease_version: 1, token: token, lease_device: stat.dev, lease_inode: stat.ino }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def close
|
|
44
|
+
@lease&.close
|
|
45
|
+
@lease = nil
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @return [String] bounded regular claim bytes, without following links
|
|
49
|
+
def snapshot
|
|
50
|
+
read_claim
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Refuse legacy records and re-check the selected claim while holding both
|
|
54
|
+
# locks. Neither host names nor claim ages authorize this operation.
|
|
55
|
+
# @param token [String] exact claim token selected by the operator
|
|
56
|
+
# @return [void]
|
|
57
|
+
def recover(token:)
|
|
58
|
+
raise Unavailable, 'claim token must be 64 hexadecimal characters' unless valid_token?(token)
|
|
59
|
+
|
|
60
|
+
coordinate do
|
|
61
|
+
snapshot = read_claim
|
|
62
|
+
record = validate_claim(snapshot, token)
|
|
63
|
+
acquire(create: false)
|
|
64
|
+
verify_lease(record)
|
|
65
|
+
raise Unavailable, 'claim changed during recovery; inspect its current owner' unless read_claim == snapshot
|
|
66
|
+
|
|
67
|
+
File.unlink(File.join(@index, CLAIM))
|
|
68
|
+
end
|
|
69
|
+
ensure
|
|
70
|
+
close
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
# @param name [String] fixed sidecar basename
|
|
76
|
+
# @param create [Boolean] permit initial creation
|
|
77
|
+
# @return [File] locked descriptor; never blocks on a live owner
|
|
78
|
+
def open_lock(name, create:)
|
|
79
|
+
flags = File::RDWR | File::NONBLOCK
|
|
80
|
+
flags |= File::CREAT if create
|
|
81
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
82
|
+
path = File.join(@index, name)
|
|
83
|
+
file = File.open(path, flags, 0o600) # rubocop:disable Style/FileOpen -- descriptor lifetime is ownership
|
|
84
|
+
verify_regular_path(file, path)
|
|
85
|
+
unless file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
86
|
+
label = name.end_with?('.lease') ? 'owner lease is held' : 'claim coordination is busy'
|
|
87
|
+
raise Unavailable, "#{label}; stop the owner through its supervisor and retry"
|
|
88
|
+
end
|
|
89
|
+
verify_regular_path(file, path)
|
|
90
|
+
file
|
|
91
|
+
rescue StandardError
|
|
92
|
+
file&.close
|
|
93
|
+
raise
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @param file [File] opened sidecar
|
|
97
|
+
# @param path [String] unchanged regular path on disk
|
|
98
|
+
def verify_regular_path(file, path)
|
|
99
|
+
opened = file.stat
|
|
100
|
+
current = File.lstat(path)
|
|
101
|
+
return if opened.file? && current.file? && [opened.dev, opened.ino] == [current.dev, current.ino]
|
|
102
|
+
|
|
103
|
+
raise Unavailable, 'claim sidecar must be an unchanged regular file; do not replace lock files'
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def read_claim
|
|
107
|
+
path = File.join(@index, CLAIM)
|
|
108
|
+
flags = File::RDONLY | File::NONBLOCK
|
|
109
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
110
|
+
File.open(path, flags) do |file|
|
|
111
|
+
verify_regular_path(file, path)
|
|
112
|
+
bytes = file.read(MAX_BYTES + 1)
|
|
113
|
+
raise Unavailable, 'claim exceeds the recovery size limit' if bytes.bytesize > MAX_BYTES
|
|
114
|
+
|
|
115
|
+
bytes
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def validate_claim(snapshot, token)
|
|
120
|
+
record = JSON.parse(snapshot)
|
|
121
|
+
unless valid_record?(record)
|
|
122
|
+
raise Unavailable, 'legacy or unverifiable claim; use the documented legacy recovery procedure'
|
|
123
|
+
end
|
|
124
|
+
unless record['token'] == token
|
|
125
|
+
raise Unavailable, 'claim token changed; inspect the current owner before retrying'
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
record
|
|
129
|
+
rescue JSON::ParserError
|
|
130
|
+
raise Unavailable, 'legacy or unverifiable claim; use the documented legacy recovery procedure'
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def valid_record?(record)
|
|
134
|
+
record.is_a?(Hash) && record['lease_version'] == 1 && valid_token?(record['token']) &&
|
|
135
|
+
record['pid'].is_a?(Integer) && record['pid'].positive? && record['host'].is_a?(String)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def verify_lease(record)
|
|
139
|
+
stat = @lease.stat
|
|
140
|
+
return if record['lease_device'] == stat.dev && record['lease_inode'] == stat.ino
|
|
141
|
+
|
|
142
|
+
raise Unavailable, 'owner lease inode changed; ownership cannot be verified'
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def valid_token?(value)
|
|
146
|
+
value.is_a?(String) && value.match?(/\A[a-f0-9]{64}\z/)
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
data/lib/woods/watch/cli.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'optparse'
|
|
4
4
|
require_relative 'supervisor'
|
|
5
|
+
require_relative 'claim_lease'
|
|
5
6
|
|
|
6
7
|
module Woods
|
|
7
8
|
module Watch
|
|
@@ -17,12 +18,13 @@ module Woods
|
|
|
17
18
|
def run(argv)
|
|
18
19
|
options, command = parse(argv.dup)
|
|
19
20
|
return 0 if @help
|
|
21
|
+
return recover_claim(options, command) if @recovery_index || @claim_token
|
|
20
22
|
|
|
21
23
|
validate_platform!
|
|
22
24
|
validate_command!(command, options[:root])
|
|
23
25
|
supervisor = Supervisor.new(command: command, logger: @output, **options)
|
|
24
26
|
with_signals(supervisor) { supervisor.run }
|
|
25
|
-
rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
|
|
27
|
+
rescue OptionParser::ParseError, ArgumentError, SystemCallError, ClaimLease::Unavailable => e
|
|
26
28
|
@output.puts("[woods-watch] #{e.message}")
|
|
27
29
|
2
|
|
28
30
|
end
|
|
@@ -31,6 +33,7 @@ module Woods
|
|
|
31
33
|
|
|
32
34
|
def parse(argv)
|
|
33
35
|
@help = false
|
|
36
|
+
@recovery_index = @claim_token = nil
|
|
34
37
|
options = { root: Dir.pwd, boot_timeout: 300, shutdown_timeout: 10 }
|
|
35
38
|
parser = OptionParser.new do |flags|
|
|
36
39
|
flags.banner = 'Usage: woods-watch [options] -- [bin/rails woods:watch]'
|
|
@@ -43,11 +46,32 @@ module Woods
|
|
|
43
46
|
flags.on('--shutdown-timeout SECONDS', 'Graceful shutdown limit (default: 10)') do |value|
|
|
44
47
|
options[:shutdown_timeout] = positive_seconds(value)
|
|
45
48
|
end
|
|
49
|
+
recovery_options(flags)
|
|
46
50
|
flags.on('-h', '--help', 'Show help without booting Rails') { @help = true }
|
|
47
51
|
end
|
|
48
52
|
parser.order!(argv)
|
|
49
53
|
@output.puts(parser) if @help
|
|
50
|
-
[options, argv.empty? ? ['bin/rails', 'woods:watch'] : argv]
|
|
54
|
+
[options, argv.empty? && !@recovery_index && !@claim_token ? ['bin/rails', 'woods:watch'] : argv]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def recovery_options(flags)
|
|
58
|
+
flags.on('--recover-claim INDEX', 'Recover an abandoned managed claim; requires --claim-token') do |path|
|
|
59
|
+
@recovery_index = path
|
|
60
|
+
end
|
|
61
|
+
flags.on('--claim-token TOKEN', 'Exact claim token; never overrides a live owner lease') do |token|
|
|
62
|
+
@claim_token = token
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def recover_claim(options, command)
|
|
67
|
+
unless @recovery_index && @claim_token && command.empty?
|
|
68
|
+
raise ArgumentError, 'use --recover-claim INDEX --claim-token TOKEN without a child command'
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
index = File.expand_path(@recovery_index, options[:root])
|
|
72
|
+
ClaimLease.new(index).recover(token: @claim_token)
|
|
73
|
+
@output.puts("[woods-watch] recovered the selected abandoned claim at #{index}; restart its normal owner")
|
|
74
|
+
0
|
|
51
75
|
end
|
|
52
76
|
|
|
53
77
|
def positive_seconds(value)
|
data/lib/woods/watch/daemon.rb
CHANGED
|
@@ -10,6 +10,8 @@ require_relative 'status'
|
|
|
10
10
|
require_relative 'tree_scan'
|
|
11
11
|
require_relative 'watcher'
|
|
12
12
|
require_relative 'boot_snapshot'
|
|
13
|
+
require_relative 'catch_up'
|
|
14
|
+
require_relative 'claim_lease'
|
|
13
15
|
require 'json'
|
|
14
16
|
require 'set'
|
|
15
17
|
require 'securerandom'
|
|
@@ -110,7 +112,7 @@ module Woods
|
|
|
110
112
|
# daemons that both pass {#another_daemon_alive?} before either has
|
|
111
113
|
# published a status record would otherwise both proceed. See
|
|
112
114
|
# {#claim_startup?}.
|
|
113
|
-
CLAIM_FILENAME =
|
|
115
|
+
CLAIM_FILENAME = ClaimLease::CLAIM
|
|
114
116
|
|
|
115
117
|
# A daemon cycle is milliseconds; a manual full extraction is seconds to
|
|
116
118
|
# minutes. This bounds how long a crashed writer can block the daemon.
|
|
@@ -423,7 +425,7 @@ module Woods
|
|
|
423
425
|
paths = paths.reject { |path| startup_covered?(path) } if startup
|
|
424
426
|
relative = ChangeSet.new(paths: paths, root: @root).relative_paths
|
|
425
427
|
action = @policy.classify_all(relative)
|
|
426
|
-
action = :reextract if action == :ignore &&
|
|
428
|
+
action = :reextract if action == :ignore && reconciliation_required?
|
|
427
429
|
return :restart if action == :reload && !@reloader.enabled?
|
|
428
430
|
|
|
429
431
|
action
|
|
@@ -448,6 +450,7 @@ module Woods
|
|
|
448
450
|
@startup_paths = Set.new
|
|
449
451
|
@live_paths = Set.new
|
|
450
452
|
@startup_full = false
|
|
453
|
+
@reconciliation_pending = false
|
|
451
454
|
@pending_mutex = Mutex.new
|
|
452
455
|
@stop_reason = nil
|
|
453
456
|
@drain_mutex = Mutex.new
|
|
@@ -492,7 +495,12 @@ module Woods
|
|
|
492
495
|
end
|
|
493
496
|
|
|
494
497
|
def carry_forward(change_set)
|
|
495
|
-
@pending_mutex.synchronize
|
|
498
|
+
@pending_mutex.synchronize do
|
|
499
|
+
@pending.merge(change_set.absolute_paths)
|
|
500
|
+
# Deletion-only startup intentionally names no paths: its bounded
|
|
501
|
+
# sweep must not treat nominal framework paths as deleted sources.
|
|
502
|
+
@reconciliation_pending = true if change_set.empty?
|
|
503
|
+
end
|
|
496
504
|
end
|
|
497
505
|
|
|
498
506
|
# Run cycles until the pending set is empty. One call per watcher batch;
|
|
@@ -542,7 +550,11 @@ module Woods
|
|
|
542
550
|
end
|
|
543
551
|
|
|
544
552
|
def pending_empty?
|
|
545
|
-
@pending_mutex.synchronize { @pending.empty? }
|
|
553
|
+
@pending_mutex.synchronize { @pending.empty? && !@reconciliation_pending }
|
|
554
|
+
end
|
|
555
|
+
|
|
556
|
+
def reconciliation_required?
|
|
557
|
+
@pending_mutex.synchronize { @startup_full || @reconciliation_pending }
|
|
546
558
|
end
|
|
547
559
|
|
|
548
560
|
# Only a drained startup boundary (or a later recovery) can establish
|
|
@@ -613,24 +625,22 @@ module Woods
|
|
|
613
625
|
|
|
614
626
|
# Reconcile changes that predate this daemon.
|
|
615
627
|
#
|
|
616
|
-
#
|
|
617
|
-
#
|
|
618
|
-
# Anything modified since is uncovered, whoever made the change and
|
|
619
|
-
# whether or not a daemon was watching at the time. With no generation
|
|
620
|
-
# file at all there is no index, and every file is uncovered — which the
|
|
621
|
-
# storm threshold correctly turns into one full extraction.
|
|
628
|
+
# Capture precedes publication. CatchUp keeps edits in that interval
|
|
629
|
+
# visible, including those after the final source-verification scan.
|
|
622
630
|
#
|
|
623
631
|
# @return [void]
|
|
624
632
|
def catch_up
|
|
625
633
|
return unless @catch_up
|
|
626
634
|
|
|
627
635
|
carried = restore_pending
|
|
628
|
-
|
|
636
|
+
scan = catch_up_scan
|
|
637
|
+
paths = (scan.paths + carried + vanished_restart_paths).uniq
|
|
629
638
|
boot_changes = @boot_snapshot ? @boot_snapshot.changed_paths : []
|
|
630
639
|
enqueue(boot_changes) unless boot_changes.empty?
|
|
631
|
-
prepare_startup_reconciliation(paths)
|
|
640
|
+
prepare_startup_reconciliation(paths, rebuild: scan.rebuild?)
|
|
632
641
|
paths |= boot_changes
|
|
633
642
|
if paths.empty?
|
|
643
|
+
return extract(ChangeSet.new(paths: [], root: @root)) if @startup_full
|
|
634
644
|
return reconcile_deletions if stale_deletions?
|
|
635
645
|
|
|
636
646
|
return @logger.info('[Woods] watch: index is current at startup')
|
|
@@ -641,7 +651,8 @@ module Woods
|
|
|
641
651
|
drain
|
|
642
652
|
end
|
|
643
653
|
|
|
644
|
-
def prepare_startup_reconciliation(paths)
|
|
654
|
+
def prepare_startup_reconciliation(paths, rebuild: false)
|
|
655
|
+
@startup_full = rebuild
|
|
645
656
|
return unless @boot_snapshot
|
|
646
657
|
|
|
647
658
|
restart_paths = paths.select do |path|
|
|
@@ -650,7 +661,7 @@ module Woods
|
|
|
650
661
|
end
|
|
651
662
|
@pending_mutex.synchronize do
|
|
652
663
|
@startup_paths.merge(restart_paths)
|
|
653
|
-
@startup_full
|
|
664
|
+
@startup_full ||= restart_paths.any?
|
|
654
665
|
end
|
|
655
666
|
end
|
|
656
667
|
|
|
@@ -721,38 +732,11 @@ module Woods
|
|
|
721
732
|
end
|
|
722
733
|
|
|
723
734
|
def uncovered_paths
|
|
724
|
-
|
|
725
|
-
TreeScan.files(root: @root, ignored: ignored_directories)
|
|
726
|
-
.select { |path| uncovered?(path, watermark) }
|
|
735
|
+
catch_up_scan.paths
|
|
727
736
|
end
|
|
728
737
|
|
|
729
|
-
def
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
File.mtime(path).to_f > watermark
|
|
733
|
-
rescue SystemCallError
|
|
734
|
-
false
|
|
735
|
-
end
|
|
736
|
-
|
|
737
|
-
# When this index was last known good, or nil when there is no usable
|
|
738
|
-
# index — which the storm threshold turns into one full extraction.
|
|
739
|
-
#
|
|
740
|
-
# A dangling payload pointer counts as "no index" (INF-10). The marker
|
|
741
|
-
# can outlive the directory it names (a partial restore from a CI
|
|
742
|
-
# artifact, an external cleanup targeting the large directories), and
|
|
743
|
-
# {Generation#payload_dir} deliberately degrades to the index root for
|
|
744
|
-
# *readers* — so a gutted index would otherwise read "current at
|
|
745
|
-
# startup", publish `running`, and have every caller stand down over a
|
|
746
|
-
# directory holding no index at all. "Alive means covered" is the
|
|
747
|
-
# daemon's contract; here it would have been false.
|
|
748
|
-
#
|
|
749
|
-
# @return [Float, nil]
|
|
750
|
-
def index_watermark
|
|
751
|
-
return nil if dangling_payload_pointer?
|
|
752
|
-
|
|
753
|
-
File.mtime(@generation.path).to_f
|
|
754
|
-
rescue SystemCallError
|
|
755
|
-
nil
|
|
738
|
+
def catch_up_scan
|
|
739
|
+
CatchUp.new(root: @root, output_dir: @output_dir, ignored: ignored_directories)
|
|
756
740
|
end
|
|
757
741
|
|
|
758
742
|
# @return [Boolean] true when the marker names a payload directory that
|
|
@@ -1018,6 +1002,7 @@ module Woods
|
|
|
1018
1002
|
else
|
|
1019
1003
|
extractor.extract_changed(change_set.absolute_paths)
|
|
1020
1004
|
end
|
|
1005
|
+
extractor.raise_on_publication_failure! if extractor.respond_to?(:raise_on_publication_failure!)
|
|
1021
1006
|
|
|
1022
1007
|
action = full ? :full : :incremental
|
|
1023
1008
|
return unpublished(action, change_set, started) if wrote_without_publishing?(touched, before)
|
|
@@ -1037,7 +1022,7 @@ module Woods
|
|
|
1037
1022
|
|
|
1038
1023
|
def full_extraction?(change_set)
|
|
1039
1024
|
if @startup_full
|
|
1040
|
-
@logger.info('[Woods] watch:
|
|
1025
|
+
@logger.info('[Woods] watch: startup coverage requires full extraction')
|
|
1041
1026
|
return true
|
|
1042
1027
|
end
|
|
1043
1028
|
|
|
@@ -1050,6 +1035,7 @@ module Woods
|
|
|
1050
1035
|
|
|
1051
1036
|
def finish_extraction(action, change_set, touched, started)
|
|
1052
1037
|
result = publish(action, change_set, touched, started)
|
|
1038
|
+
@pending_mutex.synchronize { @reconciliation_pending = false }
|
|
1053
1039
|
if action == :full
|
|
1054
1040
|
@startup_full = false
|
|
1055
1041
|
@pending_mutex.synchronize { @startup_paths.clear }
|
|
@@ -1071,8 +1057,10 @@ module Woods
|
|
|
1071
1057
|
# previous index while the daemon says `running`. Not raising was right;
|
|
1072
1058
|
# not noticing was not.
|
|
1073
1059
|
#
|
|
1074
|
-
#
|
|
1075
|
-
# units
|
|
1060
|
+
# The extractor's explicit publication-error check above also covers
|
|
1061
|
+
# payload changes with no touched units, such as withdrawing flows. Keep
|
|
1062
|
+
# this fallback for injected extractors without that reporting method.
|
|
1063
|
+
# A genuine no-op deliberately does not bump.
|
|
1076
1064
|
def wrote_without_publishing?(touched, before)
|
|
1077
1065
|
return false if touched != :all && Array(touched).empty?
|
|
1078
1066
|
|
|
@@ -1188,7 +1176,31 @@ module Woods
|
|
|
1188
1176
|
def claim_startup?
|
|
1189
1177
|
return true if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
|
|
1190
1178
|
|
|
1179
|
+
prepare_claim_lease if @conservative_claims
|
|
1180
|
+
acquire_startup_claim
|
|
1181
|
+
rescue ClaimLease::Unavailable, SystemCallError => e
|
|
1182
|
+
raise unless @conservative_claims
|
|
1183
|
+
|
|
1184
|
+
@logger.warn("[Woods] watch: managed claim unavailable: #{e.message}")
|
|
1185
|
+
false
|
|
1186
|
+
ensure
|
|
1187
|
+
close_unclaimed_lease
|
|
1188
|
+
end
|
|
1189
|
+
|
|
1190
|
+
def close_unclaimed_lease
|
|
1191
|
+
return if @claimed
|
|
1192
|
+
|
|
1193
|
+
@claim_lease&.close
|
|
1194
|
+
end
|
|
1195
|
+
|
|
1196
|
+
def prepare_claim_lease
|
|
1197
|
+
FileUtils.mkdir_p(@output_dir)
|
|
1198
|
+
@claim_lease = ClaimLease.new(@output_dir)
|
|
1199
|
+
end
|
|
1200
|
+
|
|
1201
|
+
def acquire_startup_claim
|
|
1191
1202
|
with_claim_lock do
|
|
1203
|
+
@claim_lease&.acquire
|
|
1192
1204
|
3.times do
|
|
1193
1205
|
return true if create_claim
|
|
1194
1206
|
return false unless reclaim_if_stale
|
|
@@ -1203,11 +1215,13 @@ module Woods
|
|
|
1203
1215
|
# pauses, B deletes S and publishes its live claim, A resumes and
|
|
1204
1216
|
# unlinks B's claim, and both return as owners. An `flock` on a
|
|
1205
1217
|
# sidecar file makes the whole loop one critical section; the kernel
|
|
1206
|
-
# releases it if the holder dies.
|
|
1207
|
-
#
|
|
1208
|
-
def with_claim_lock
|
|
1218
|
+
# releases it if the holder dies. Managed startup refuses unavailable
|
|
1219
|
+
# locks; raw startup retains its earlier unserialized fallback.
|
|
1220
|
+
def with_claim_lock(&block)
|
|
1221
|
+
return @claim_lease.coordinate(&block) if @claim_lease
|
|
1222
|
+
|
|
1209
1223
|
lock = open_claim_lock
|
|
1210
|
-
|
|
1224
|
+
block.call
|
|
1211
1225
|
ensure
|
|
1212
1226
|
lock&.close
|
|
1213
1227
|
end
|
|
@@ -1243,7 +1257,7 @@ module Woods
|
|
|
1243
1257
|
# race this method exists to close.
|
|
1244
1258
|
def create_claim
|
|
1245
1259
|
FileUtils.mkdir_p(@output_dir)
|
|
1246
|
-
content = JSON.generate(pid: Process.pid, host: Status.host_identity)
|
|
1260
|
+
content = JSON.generate(pid: Process.pid, host: Status.host_identity, **(@claim_lease&.fields || {}))
|
|
1247
1261
|
tmp_path = "#{claim_path}.tmp.#{Process.pid}.#{SecureRandom.hex(4)}"
|
|
1248
1262
|
File.write(tmp_path, content)
|
|
1249
1263
|
begin
|
|
@@ -1303,7 +1317,7 @@ module Woods
|
|
|
1303
1317
|
# @return [String, nil] the claim file's current bytes, or nil if it
|
|
1304
1318
|
# doesn't exist
|
|
1305
1319
|
def claim_bytes
|
|
1306
|
-
File.read(claim_path)
|
|
1320
|
+
@claim_lease ? @claim_lease.snapshot : File.read(claim_path)
|
|
1307
1321
|
rescue Errno::ENOENT
|
|
1308
1322
|
nil
|
|
1309
1323
|
end
|
|
@@ -1313,12 +1327,12 @@ module Woods
|
|
|
1313
1327
|
# or already-vanished claim counts as stale too, since it cannot be a
|
|
1314
1328
|
# live daemon's claim
|
|
1315
1329
|
def stale_claim?
|
|
1316
|
-
record = JSON.parse(
|
|
1330
|
+
record = JSON.parse(claim_bytes)
|
|
1317
1331
|
return false if @conservative_claims && !verifiable_managed_claim?(record)
|
|
1318
1332
|
return true unless same_claim_host?(record['host'])
|
|
1319
1333
|
|
|
1320
1334
|
!claim_pid_alive?(record['pid'])
|
|
1321
|
-
rescue JSON::ParserError, SystemCallError
|
|
1335
|
+
rescue JSON::ParserError, SystemCallError, TypeError
|
|
1322
1336
|
!@conservative_claims
|
|
1323
1337
|
end
|
|
1324
1338
|
|
|
@@ -1374,18 +1388,25 @@ module Woods
|
|
|
1374
1388
|
def release_claim
|
|
1375
1389
|
return unless @claimed
|
|
1376
1390
|
|
|
1377
|
-
|
|
1391
|
+
if @claim_lease
|
|
1392
|
+
@claim_lease.coordinate { FileUtils.rm_f(claim_path) if own_claim? }
|
|
1393
|
+
elsif own_claim?
|
|
1394
|
+
FileUtils.rm_f(claim_path)
|
|
1395
|
+
end
|
|
1396
|
+
ensure
|
|
1397
|
+
@claim_lease&.close
|
|
1378
1398
|
@claimed = false
|
|
1379
1399
|
end
|
|
1380
1400
|
|
|
1381
1401
|
# @return [Boolean] whether the claim on disk still records this process
|
|
1382
1402
|
def own_claim?
|
|
1383
|
-
record = JSON.parse(
|
|
1403
|
+
record = JSON.parse(claim_bytes)
|
|
1404
|
+
return record.is_a?(Hash) && record['token'] == @claim_lease.token if @claim_lease
|
|
1384
1405
|
return true unless record.is_a?(Hash)
|
|
1385
1406
|
|
|
1386
1407
|
record['pid'] == Process.pid && same_claim_host?(record['host'])
|
|
1387
|
-
rescue JSON::ParserError, SystemCallError
|
|
1388
|
-
|
|
1408
|
+
rescue JSON::ParserError, SystemCallError, TypeError, ClaimLease::Unavailable
|
|
1409
|
+
!@claim_lease
|
|
1389
1410
|
end
|
|
1390
1411
|
|
|
1391
1412
|
def claim_path
|
|
@@ -92,7 +92,7 @@ module Woods
|
|
|
92
92
|
|
|
93
93
|
def validate_idle!
|
|
94
94
|
value = @environment['WOODS_WATCH_IDLE_TIMEOUT']
|
|
95
|
-
return if mode == 'external' || value.
|
|
95
|
+
return if mode == 'external' || value.to_s.strip.empty?
|
|
96
96
|
|
|
97
97
|
raise Conflict, 'Unset WOODS_WATCH_IDLE_TIMEOUT; managed maintenance needs a resident child'
|
|
98
98
|
end
|
|
@@ -26,7 +26,12 @@ module Woods
|
|
|
26
26
|
# @param expected [Hash] prior content fingerprint and mode
|
|
27
27
|
# @return [void]
|
|
28
28
|
def self.verify_file!(document, expected)
|
|
29
|
-
|
|
29
|
+
actual = document.fingerprint
|
|
30
|
+
# Git records the owner's executable bit, not the checkout's umask.
|
|
31
|
+
# Plan snapshots separately retain exact modes for preview/apply drift.
|
|
32
|
+
same_content = actual.slice('sha256', 'exists') == expected.slice('sha256', 'exists')
|
|
33
|
+
same_executable = (actual.fetch('mode') & 0o100) == (expected.fetch('mode') & 0o100)
|
|
34
|
+
return if same_content && same_executable
|
|
30
35
|
|
|
31
36
|
raise Conflict, "Owned watcher file was edited or removed: #{document.path}"
|
|
32
37
|
end
|
|
@@ -26,7 +26,7 @@ module Woods
|
|
|
26
26
|
descriptor, launcher, attempt = values
|
|
27
27
|
validate_environment!(descriptor, [launcher, attempt])
|
|
28
28
|
|
|
29
|
-
unless env['WOODS_WATCH_IDLE_TIMEOUT'].to_s.empty?
|
|
29
|
+
unless env['WOODS_WATCH_IDLE_TIMEOUT'].to_s.strip.empty?
|
|
30
30
|
raise ArgumentError, 'WOODS_WATCH_IDLE_TIMEOUT must be unset for managed watching'
|
|
31
31
|
end
|
|
32
32
|
|