woods 2.0.0.beta3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +500 -420
- data/CONTRIBUTING.md +29 -17
- data/README.md +78 -178
- data/docs/AGENT_GUIDE.md +52 -11
- data/docs/AGENT_SETUP.md +34 -17
- data/docs/AUTOMATIC_MAINTENANCE.md +222 -0
- data/docs/BACKEND_MATRIX.md +18 -7
- data/docs/CLIENT_HOOKS.md +1 -1
- data/docs/CONFIGURATION_REFERENCE.md +105 -29
- data/docs/CONSOLE_MCP_SETUP.md +54 -9
- data/docs/DOCKER_SETUP.md +16 -1
- data/docs/EVALUATION.md +10 -4
- data/docs/EXTRACTOR_REFERENCE.md +23 -3
- data/docs/FAQ.md +14 -3
- data/docs/GETTING_STARTED.md +18 -17
- data/docs/INCREMENTAL_EXTRACTION.md +37 -8
- data/docs/INDEX_LAYOUT.md +2 -2
- data/docs/MCP_SERVERS.md +79 -7
- data/docs/MCP_TOOL_COOKBOOK.md +5 -5
- data/docs/MCP_WORKTREE_SETUP.md +55 -83
- data/docs/PUBLISHED_INDEX.md +17 -0
- data/docs/README.md +2 -1
- data/docs/RETRIEVAL_GUIDE.md +81 -13
- data/docs/SOURCE_FRESHNESS.md +1 -1
- data/docs/TOKEN_BENCHMARK.md +16 -10
- data/docs/TROUBLESHOOTING.md +142 -47
- data/docs/UPGRADING_TO_2.md +12 -6
- data/docs/WATCH_DAEMON.md +189 -24
- data/docs/WHY_WOODS.md +9 -5
- data/exe/woods-console +13 -11
- data/exe/woods-mcp-start +14 -9
- data/exe/woods-watch +5 -0
- data/lib/generators/woods/pgvector_generator.rb +8 -2
- data/lib/generators/woods/watch_generator.rb +53 -0
- data/lib/puma/plugin/woods.rb +10 -0
- data/lib/tasks/woods.rake +14 -0
- data/lib/woods/agent_configuration/applier.rb +5 -3
- data/lib/woods/agent_configuration/cli.rb +2 -2
- data/lib/woods/agent_configuration/layout.rb +13 -0
- data/lib/woods/cache/cache_middleware.rb +6 -0
- data/lib/woods/console/credential_scanner.rb +4 -3
- data/lib/woods/console/dispatch_pipeline.rb +7 -0
- data/lib/woods/console/embedded_executor.rb +31 -9
- data/lib/woods/console/sql_noise_stripper.rb +9 -7
- data/lib/woods/console/sql_table_scanner.rb +47 -7
- data/lib/woods/console/sql_validator.rb +49 -9
- data/lib/woods/console/sqlite_read_guard.rb +46 -0
- data/lib/woods/console/stdio_transport.rb +27 -0
- data/lib/woods/coordination/pipeline_lock.rb +3 -2
- data/lib/woods/embedding/indexer.rb +24 -14
- data/lib/woods/extractor.rb +70 -19
- data/lib/woods/extractors/declared_parent.rb +55 -0
- data/lib/woods/extractors/graphql_extractor.rb +2 -11
- data/lib/woods/extractors/lib_extractor.rb +10 -8
- data/lib/woods/extractors/mailer_extractor.rb +6 -10
- data/lib/woods/extractors/model_extractor.rb +1 -15
- data/lib/woods/extractors/poro_extractor.rb +10 -8
- data/lib/woods/extractors/shared_utility_methods.rb +22 -5
- data/lib/woods/git_command.rb +6 -7
- data/lib/woods/git_provenance.rb +4 -6
- data/lib/woods/mcp/bearer_auth.rb +2 -1
- data/lib/woods/mcp/bootstrapper.rb +20 -5
- data/lib/woods/mcp/config_resolver.rb +2 -1
- data/lib/woods/mcp/index_reader.rb +11 -2
- data/lib/woods/mcp/initialization_guidance.rb +1 -1
- data/lib/woods/mcp/renderers/markdown_renderer.rb +14 -8
- data/lib/woods/mcp/renderers/plain_renderer.rb +11 -7
- data/lib/woods/mcp/server.rb +63 -37
- data/lib/woods/mcp/tool_contract.rb +1 -1
- data/lib/woods/mcp/tool_response_renderer.rb +16 -0
- data/lib/woods/mcp/traversal_evidence_text.rb +1 -1
- data/lib/woods/mcp/traversal_response.rb +22 -0
- data/lib/woods/path_dispatcher.rb +6 -5
- data/lib/woods/published_index/typed_unit_reader.rb +40 -3
- data/lib/woods/published_index.rb +2 -2
- data/lib/woods/rake_helpers.rb +2 -12
- data/lib/woods/retrieval/corpus_status.rb +46 -0
- data/lib/woods/retrieval/lexical_assembler.rb +14 -3
- data/lib/woods/retrieval/lexical_index.rb +2 -1
- data/lib/woods/retriever.rb +19 -7
- data/lib/woods/session_tracer/file_store.rb +6 -1
- data/lib/woods/source_inputs/consumer_errors.rb +4 -0
- data/lib/woods/storage/local_corpus_stats.rb +32 -0
- data/lib/woods/storage/metadata_store.rb +20 -0
- data/lib/woods/storage/pgvector.rb +6 -2
- data/lib/woods/storage/vector_store.rb +10 -0
- data/lib/woods/temporal/json_snapshot_store.rb +35 -7
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/child_environment.rb +30 -0
- data/lib/woods/watch/cli.rb +91 -0
- data/lib/woods/watch/daemon.rb +73 -11
- data/lib/woods/watch/event_stream.rb +70 -0
- data/lib/woods/watch/guardian.rb +142 -0
- data/lib/woods/watch/installation/layout.rb +70 -0
- data/lib/woods/watch/installation/options.rb +128 -0
- data/lib/woods/watch/installation/planner.rb +128 -0
- data/lib/woods/watch/installation/probe.rb +101 -0
- data/lib/woods/watch/installation/receipt.rb +77 -0
- data/lib/woods/watch/installation/recovery.rb +64 -0
- data/lib/woods/watch/installation/templates.rb +58 -0
- data/lib/woods/watch/installation.rb +56 -0
- data/lib/woods/watch/lifecycle.rb +182 -0
- data/lib/woods/watch/managed_child.rb +113 -0
- data/lib/woods/watch/managed_cleanup.rb +48 -0
- data/lib/woods/watch/managed_process.rb +144 -0
- data/lib/woods/watch/puma_adapter.rb +87 -0
- data/lib/woods/watch/puma_child.rb +66 -0
- data/lib/woods/watch/supervision_records.rb +95 -0
- data/lib/woods/watch/supervision_status.rb +104 -0
- data/lib/woods/watch/supervisor.rb +161 -0
- data/lib/woods/watch/supervisor_reporting.rb +46 -0
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/hooks/woods-input-rules.sh +4 -4
- data/plugin/skills/woods-agent-enable/SKILL.md +7 -1
- data/plugin/skills/woods-diagnose/SKILL.md +134 -34
- data/plugin/skills/woods-investigate/SKILL.md +54 -15
- data/plugin/skills/woods-mcp-config/SKILL.md +38 -11
- data/plugin/skills/woods-setup/SKILL.md +72 -15
- metadata +38 -5
|
@@ -16,9 +16,10 @@ module Woods
|
|
|
16
16
|
SourceEvidence.validate_mode!(evidence)
|
|
17
17
|
raise ArgumentError, 'budget must be a positive Integer' unless budget.is_a?(Integer) && budget.positive?
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
# Reserve the largest included count before selecting evidence. Replacing
|
|
20
|
+
# it afterward can only shrink the text; do not refill or reorder sources.
|
|
21
|
+
notice = count_notice(candidates.size, candidates.size)
|
|
22
|
+
context = notice[0, budget * 4]
|
|
22
23
|
sources = []
|
|
23
24
|
candidates.each do |candidate|
|
|
24
25
|
unit = candidate.metadata
|
|
@@ -50,12 +51,22 @@ module Woods
|
|
|
50
51
|
score: candidate.score, matched_fields: candidate.matched_fields, truncated: truncated }
|
|
51
52
|
break if truncated
|
|
52
53
|
end
|
|
54
|
+
body = context[notice.length..].to_s
|
|
55
|
+
context = (count_notice(candidates.size, sources.size) + body)[0, budget * 4]
|
|
53
56
|
AssembledContext.new(context: context, tokens_used: estimate_tokens(context), budget: budget,
|
|
54
57
|
sources: sources, sections: [:primary], skipped_missing_metadata: 0)
|
|
55
58
|
end
|
|
56
59
|
|
|
57
60
|
private
|
|
58
61
|
|
|
62
|
+
def count_notice(candidates, included)
|
|
63
|
+
text = "Mode: lexical (field-aware BM25; sources included: #{included}; " \
|
|
64
|
+
"candidates considered: #{candidates}; candidate limit: #{LexicalIndex::DEFAULT_LIMIT}; " \
|
|
65
|
+
'token counts estimated).'
|
|
66
|
+
text += "\nNo lexical matches." if candidates.zero?
|
|
67
|
+
text
|
|
68
|
+
end
|
|
69
|
+
|
|
59
70
|
def evidence_text(candidate)
|
|
60
71
|
unit = candidate.metadata
|
|
61
72
|
source = unit['source_code'].to_s
|
|
@@ -13,6 +13,7 @@ module Woods
|
|
|
13
13
|
RUNTIME_FIELDS = %w[callbacks associations validations scopes concerns included_modules
|
|
14
14
|
methods instance_methods class_methods actions routes columns table_name
|
|
15
15
|
description purpose dependencies].freeze
|
|
16
|
+
DEFAULT_LIMIT = 20
|
|
16
17
|
K1 = 1.2
|
|
17
18
|
B = 0.75
|
|
18
19
|
Document = Struct.new(:key, :unit, :fields, keyword_init: true)
|
|
@@ -36,7 +37,7 @@ module Woods
|
|
|
36
37
|
@frequencies.freeze
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
def execute(query:, limit:
|
|
40
|
+
def execute(query:, limit: DEFAULT_LIMIT, type_filter: nil, exclude_types: nil)
|
|
40
41
|
terms = tokenize(query).uniq
|
|
41
42
|
candidates = @documents.filter_map do |doc|
|
|
42
43
|
next unless eligible?(doc.unit, type_filter, exclude_types)
|
data/lib/woods/retriever.rb
CHANGED
|
@@ -13,6 +13,7 @@ require_relative 'retrieval/lexical_assembler'
|
|
|
13
13
|
require_relative 'retrieval/scope'
|
|
14
14
|
require_relative 'retrieval/scoped_vector_store'
|
|
15
15
|
require_relative 'retrieval/scoped_graph_store'
|
|
16
|
+
require_relative 'retrieval/corpus_status'
|
|
16
17
|
require_relative 'embedding/token_counter'
|
|
17
18
|
require_relative 'token_utils'
|
|
18
19
|
|
|
@@ -85,7 +86,7 @@ module Woods
|
|
|
85
86
|
# source: :in_top_k, # see enum below
|
|
86
87
|
# top_of_type_global_rank: 3, # 1-based rank in unfiltered ranked, or nil
|
|
87
88
|
# global_k: 20, # size of the unfiltered ranked list
|
|
88
|
-
# total_of_type: 183 #
|
|
89
|
+
# total_of_type: 183 # retrieval metadata records of that type
|
|
89
90
|
# }
|
|
90
91
|
# }
|
|
91
92
|
#
|
|
@@ -97,10 +98,11 @@ module Woods
|
|
|
97
98
|
# the fallback vector search returned
|
|
98
99
|
# candidates of this type. Weak match.
|
|
99
100
|
# :outside_top_k — type NOT in the unfiltered ranked list, has
|
|
100
|
-
#
|
|
101
|
+
# retrieval metadata, but the fallback did
|
|
101
102
|
# not run (other requested types filled the
|
|
102
103
|
# result). No results of this type.
|
|
103
|
-
# :absent — type has zero
|
|
104
|
+
# :absent — type has zero retrieval metadata records;
|
|
105
|
+
# structural extraction may still contain it.
|
|
104
106
|
#
|
|
105
107
|
# Nil for unfiltered queries.
|
|
106
108
|
RetrievalResult = Struct.new(:context, :sources, :classification, :strategy, :tokens_used, :budget, :trace,
|
|
@@ -199,6 +201,16 @@ module Woods
|
|
|
199
201
|
# @return [Pipeline]
|
|
200
202
|
attr_reader :pipeline, :mode, :default_budget
|
|
201
203
|
|
|
204
|
+
# Diagnose the currently served semantic stores without provider or remote
|
|
205
|
+
# store requests. Capture one pipeline so a reload cannot mix store counts.
|
|
206
|
+
# @return [Hash, nil] local corpus statistics, or nil for lexical retrieval
|
|
207
|
+
def corpus_status(include_types: true)
|
|
208
|
+
return unless mode == :semantic
|
|
209
|
+
|
|
210
|
+
current = @pipeline
|
|
211
|
+
Retrieval::CorpusStatus.build(current.vector_store, current.metadata_store, include_types: include_types)
|
|
212
|
+
end
|
|
213
|
+
|
|
202
214
|
# Optional callback invoked with the pipeline struct the moment
|
|
203
215
|
# {#retrieve} resolves it, before any pipeline work runs. Nil in
|
|
204
216
|
# production. Observability seam for the reload transaction's old-or-new
|
|
@@ -642,9 +654,9 @@ module Woods
|
|
|
642
654
|
#
|
|
643
655
|
# +top_of_type_global_rank+ is the 1-based position of the first
|
|
644
656
|
# candidate of that type in the ranked list, or nil when no candidate
|
|
645
|
-
# of that type survived ranking. +total_of_type+
|
|
646
|
-
#
|
|
647
|
-
#
|
|
657
|
+
# of that type survived ranking. +total_of_type+ counts retrieval metadata
|
|
658
|
+
# records, including chunk records and units without vectors. It does not
|
|
659
|
+
# measure structural unit coverage or embedded units. +source+ labels the bucket
|
|
648
660
|
# the type landed in so the caller doesn't infer it from a nil rank;
|
|
649
661
|
# see the RetrievalResult docstring for the four-value enum.
|
|
650
662
|
#
|
|
@@ -727,7 +739,7 @@ module Woods
|
|
|
727
739
|
return context if type_rank_context.empty?
|
|
728
740
|
|
|
729
741
|
lines = ['', '### Type rank context', '',
|
|
730
|
-
'| Type | Source | Rank in unfiltered top-K | Global K |
|
|
742
|
+
'| Type | Source | Rank in unfiltered top-K | Global K | Retrieval metadata records |',
|
|
731
743
|
'|------|--------|--------------------------|----------|----------------|']
|
|
732
744
|
type_rank_context.each do |type, info|
|
|
733
745
|
rank = info[:top_of_type_global_rank] || '—'
|
|
@@ -116,7 +116,8 @@ module Woods
|
|
|
116
116
|
def clear(session_id)
|
|
117
117
|
with_store_lock do
|
|
118
118
|
FileUtils.rm_f(session_path(session_id))
|
|
119
|
-
|
|
119
|
+
legacy = legacy_session_path(session_id)
|
|
120
|
+
FileUtils.rm_f(legacy) if legacy
|
|
120
121
|
end
|
|
121
122
|
end
|
|
122
123
|
|
|
@@ -158,6 +159,10 @@ module Woods
|
|
|
158
159
|
def migrate_legacy_session!(session_id)
|
|
159
160
|
target = session_path(session_id)
|
|
160
161
|
legacy = legacy_session_path(session_id)
|
|
162
|
+
# Expire each half before a merge or append can refresh its mtime.
|
|
163
|
+
[target, legacy].compact.each do |path|
|
|
164
|
+
FileUtils.rm_f(path) if File.exist?(path) && expired?(path)
|
|
165
|
+
end
|
|
161
166
|
return target unless legacy && File.exist?(legacy)
|
|
162
167
|
|
|
163
168
|
if File.exist?(target)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Storage
|
|
5
|
+
# Counts local store entries without loading vectors or source bodies.
|
|
6
|
+
# Counts describe records (including chunks), not canonical unit coverage.
|
|
7
|
+
module LocalCorpusStats
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# @param types [Array<String, Symbol, nil>] one type per live entry
|
|
11
|
+
# @return [Hash] total, known types, and entries without a usable type
|
|
12
|
+
def from_types(types)
|
|
13
|
+
from_counts(types.tally)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# @param counts [Hash] locally grouped type => entry counts
|
|
17
|
+
# @return [Hash] entry counts with unknown types kept separate
|
|
18
|
+
def from_counts(counts)
|
|
19
|
+
by_type = Hash.new(0)
|
|
20
|
+
untyped = 0
|
|
21
|
+
counts.each do |type, count|
|
|
22
|
+
if type.nil? || type.to_s.strip.empty?
|
|
23
|
+
untyped += count
|
|
24
|
+
else
|
|
25
|
+
by_type[type.to_s] += count
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
{ count: by_type.values.sum + untyped, by_type: by_type.sort.to_h, untyped_count: untyped }
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'local_corpus_stats'
|
|
4
|
+
|
|
3
5
|
require 'json'
|
|
4
6
|
require 'fileutils'
|
|
5
7
|
require 'time'
|
|
@@ -265,6 +267,14 @@ module Woods
|
|
|
265
267
|
@data.size
|
|
266
268
|
end
|
|
267
269
|
|
|
270
|
+
# Local diagnostic capability; counts records, not embedded units.
|
|
271
|
+
# @return [Hash] metadata entry counts, including source-empty records
|
|
272
|
+
def local_corpus_stats(include_types: true)
|
|
273
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
274
|
+
|
|
275
|
+
LocalCorpusStats.from_types(@data.values.map { |record| record['type'] })
|
|
276
|
+
end
|
|
277
|
+
|
|
268
278
|
# Iterate over every stored entry, yielding +(id, metadata)+ pairs.
|
|
269
279
|
#
|
|
270
280
|
# Persistence seam for {Snapshotter::Metadata}. Yields the raw internal
|
|
@@ -465,6 +475,16 @@ module Woods
|
|
|
465
475
|
@db.get_first_value('SELECT COUNT(*) FROM units')
|
|
466
476
|
end
|
|
467
477
|
|
|
478
|
+
# Reads only grouped type counts from the local SQLite database.
|
|
479
|
+
# @return [Hash] metadata entry counts, independent of vector coverage
|
|
480
|
+
def local_corpus_stats(include_types: true)
|
|
481
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
482
|
+
|
|
483
|
+
counts = @db.execute('SELECT type, COUNT(*) AS entry_count FROM units GROUP BY type')
|
|
484
|
+
.to_h { |row| [row['type'], row['entry_count']] }
|
|
485
|
+
LocalCorpusStats.from_counts(counts)
|
|
486
|
+
end
|
|
487
|
+
|
|
468
488
|
private
|
|
469
489
|
|
|
470
490
|
# Bounded retry around a contended write, mirroring
|
|
@@ -26,6 +26,7 @@ module Woods
|
|
|
26
26
|
class Pgvector # rubocop:disable Metrics/ClassLength
|
|
27
27
|
include Interface
|
|
28
28
|
|
|
29
|
+
MAX_HNSW_DIMENSIONS = 2000
|
|
29
30
|
TABLE = 'woods_vectors'
|
|
30
31
|
TABLE_NAME_PATTERN = /\A[a-z_][a-z0-9_]*\z/
|
|
31
32
|
|
|
@@ -219,9 +220,12 @@ module Woods
|
|
|
219
220
|
private
|
|
220
221
|
|
|
221
222
|
def normalize_dimensions(value)
|
|
222
|
-
|
|
223
|
+
raise ArgumentError, 'dimensions must be a positive Integer' unless value.is_a?(Integer) && value.positive?
|
|
224
|
+
return value if value <= MAX_HNSW_DIMENSIONS
|
|
223
225
|
|
|
224
|
-
raise ArgumentError,
|
|
226
|
+
raise ArgumentError,
|
|
227
|
+
"pgvector HNSW vector dimensions must be at most #{MAX_HNSW_DIMENSIONS}, got #{value}. " \
|
|
228
|
+
'Request a supported provider output width or choose another vector backend.'
|
|
225
229
|
end
|
|
226
230
|
|
|
227
231
|
def validate_identifier!(name, value, original)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'set'
|
|
4
|
+
require_relative 'local_corpus_stats'
|
|
4
5
|
|
|
5
6
|
module Woods
|
|
6
7
|
module Storage
|
|
@@ -263,6 +264,15 @@ module Woods
|
|
|
263
264
|
@ids.size - @tombstones.size
|
|
264
265
|
end
|
|
265
266
|
|
|
267
|
+
# Local diagnostic capability; does not load or copy vector values.
|
|
268
|
+
# @return [Hash] vector entry counts, including chunk entries
|
|
269
|
+
def local_corpus_stats(include_types: true)
|
|
270
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
271
|
+
|
|
272
|
+
types = @id_to_index.values.map { |idx| metadata_value(@metadata[idx], :type) }
|
|
273
|
+
LocalCorpusStats.from_types(types)
|
|
274
|
+
end
|
|
275
|
+
|
|
266
276
|
private
|
|
267
277
|
|
|
268
278
|
# Match a filter value against a metadata value. Arrays are
|
|
@@ -22,7 +22,7 @@ module Woods
|
|
|
22
22
|
#
|
|
23
23
|
# Implements the same public interface as SnapshotStore so the MCP server
|
|
24
24
|
# tools work identically.
|
|
25
|
-
# Malformed,
|
|
25
|
+
# Malformed, invalidly shaped, or unreadable retained snapshots (including files
|
|
26
26
|
# removed during retention) are warned about and treated as absent:
|
|
27
27
|
# +find+ returns nil, +diff+ returns an empty result, and history/list scans
|
|
28
28
|
# omit the corrupt file.
|
|
@@ -95,7 +95,6 @@ module Woods
|
|
|
95
95
|
snapshots = load_all_with_units
|
|
96
96
|
.sort_by { |s| s[:extracted_at] || '' }
|
|
97
97
|
.reverse
|
|
98
|
-
.first(limit)
|
|
99
98
|
|
|
100
99
|
entries = snapshots.flat_map do |snap|
|
|
101
100
|
snap[:units].values.select { |unit| unit[:identifier] == identifier }.map do |unit|
|
|
@@ -150,10 +149,8 @@ module Woods
|
|
|
150
149
|
sha = File.basename(path, '.json')
|
|
151
150
|
next unless sha.match?(/\A[0-9a-f]+\z/i) && File.file?(path)
|
|
152
151
|
|
|
153
|
-
data = read_snapshot(path)
|
|
154
|
-
|
|
155
|
-
valid = data['git_sha'] == sha && timestamp.is_a?(String)
|
|
156
|
-
{ git_sha: sha, extracted_at: valid ? timestamp : '', corrupt: !valid }
|
|
152
|
+
data = read_snapshot(path)
|
|
153
|
+
{ git_sha: sha, extracted_at: data&.[]('extracted_at') || '', corrupt: data.nil? }
|
|
157
154
|
end
|
|
158
155
|
end
|
|
159
156
|
|
|
@@ -280,7 +277,10 @@ module Woods
|
|
|
280
277
|
|
|
281
278
|
def read_snapshot(path)
|
|
282
279
|
data = JSON.parse(AtomicFile.read(path))
|
|
283
|
-
|
|
280
|
+
validate_snapshot_shape!(data)
|
|
281
|
+
unless data['git_sha'] == File.basename(path, '.json')
|
|
282
|
+
raise JSON::ParserError, 'git_sha does not match the snapshot filename'
|
|
283
|
+
end
|
|
284
284
|
|
|
285
285
|
data
|
|
286
286
|
rescue JSON::ParserError => e
|
|
@@ -291,6 +291,34 @@ module Woods
|
|
|
291
291
|
nil
|
|
292
292
|
end
|
|
293
293
|
|
|
294
|
+
# Guard only shapes consumed by conversion, sorting, and the next capture.
|
|
295
|
+
# Legacy files can omit units/timestamps and optional per-unit hashes;
|
|
296
|
+
# bare unit keys still supply identifiers when the record does not.
|
|
297
|
+
def validate_snapshot_shape!(data)
|
|
298
|
+
raise JSON::ParserError, 'expected a JSON object' unless data.is_a?(Hash)
|
|
299
|
+
|
|
300
|
+
sha = data['git_sha']
|
|
301
|
+
unless sha.is_a?(String) && sha.match?(/\A[0-9a-f]+\z/i)
|
|
302
|
+
raise JSON::ParserError, 'expected git_sha to be a hexadecimal string'
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
timestamp = data['extracted_at']
|
|
306
|
+
unless timestamp.nil? || timestamp.is_a?(String)
|
|
307
|
+
raise JSON::ParserError, 'expected extracted_at to be a string or null'
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
validate_snapshot_units!(data['units'])
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def validate_snapshot_units!(units)
|
|
314
|
+
return if units.nil?
|
|
315
|
+
|
|
316
|
+
raise JSON::ParserError, 'expected units to be an object or null' unless units.is_a?(Hash)
|
|
317
|
+
return if units.each_value.all?(Hash)
|
|
318
|
+
|
|
319
|
+
raise JSON::ParserError, 'expected every unit record to be an object'
|
|
320
|
+
end
|
|
321
|
+
|
|
294
322
|
# @param exclude_sha [String, nil] SHA to leave out of the result
|
|
295
323
|
# @return [Hash, nil]
|
|
296
324
|
def find_latest(exclude_sha: nil)
|
data/lib/woods/version.rb
CHANGED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Watch
|
|
5
|
+
# Restore Bundler's original environment while preserving the selected
|
|
6
|
+
# application Gemfile. Each child must resolve the current lockfile anew.
|
|
7
|
+
module ChildEnvironment
|
|
8
|
+
NIL_VALUE = 'BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL'
|
|
9
|
+
ACTIVATION_KEYS = %w[BUNDLE_BIN_PATH BUNDLER_VERSION BUNDLE_LOCKFILE].freeze
|
|
10
|
+
|
|
11
|
+
# @param env [Hash] complete intended application environment
|
|
12
|
+
# @param root [String] application working directory
|
|
13
|
+
# @return [Hash] independent environment for exec with unsetenv_others
|
|
14
|
+
def self.build(env, root:)
|
|
15
|
+
result = env.dup
|
|
16
|
+
gemfile = result['BUNDLE_GEMFILE']
|
|
17
|
+
ACTIVATION_KEYS.each { |key| result.delete(key) }
|
|
18
|
+
env.each do |key, value|
|
|
19
|
+
next unless key.start_with?('BUNDLER_ORIG_')
|
|
20
|
+
|
|
21
|
+
original = key.delete_prefix('BUNDLER_ORIG_')
|
|
22
|
+
value == NIL_VALUE ? result.delete(original) : result[original] = value
|
|
23
|
+
result.delete(key)
|
|
24
|
+
end
|
|
25
|
+
result['BUNDLE_GEMFILE'] = File.expand_path(gemfile, root) if gemfile && !gemfile.empty?
|
|
26
|
+
result
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
require_relative 'supervisor'
|
|
5
|
+
|
|
6
|
+
module Woods
|
|
7
|
+
module Watch
|
|
8
|
+
# Parses the foreground launcher contract without booting Rails.
|
|
9
|
+
class CLI
|
|
10
|
+
# @param output [#puts] diagnostic stream
|
|
11
|
+
def initialize(output: $stderr)
|
|
12
|
+
@output = output
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param argv [Array<String>] launcher options and explicit child arguments
|
|
16
|
+
# @return [Integer] process exit status
|
|
17
|
+
def run(argv)
|
|
18
|
+
options, command = parse(argv.dup)
|
|
19
|
+
return 0 if @help
|
|
20
|
+
|
|
21
|
+
validate_platform!
|
|
22
|
+
validate_command!(command, options[:root])
|
|
23
|
+
supervisor = Supervisor.new(command: command, logger: @output, **options)
|
|
24
|
+
with_signals(supervisor) { supervisor.run }
|
|
25
|
+
rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
|
|
26
|
+
@output.puts("[woods-watch] #{e.message}")
|
|
27
|
+
2
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def parse(argv)
|
|
33
|
+
@help = false
|
|
34
|
+
options = { root: Dir.pwd, boot_timeout: 300, shutdown_timeout: 10 }
|
|
35
|
+
parser = OptionParser.new do |flags|
|
|
36
|
+
flags.banner = 'Usage: woods-watch [options] -- [bin/rails woods:watch]'
|
|
37
|
+
flags.on('--root PATH', 'Application root (default: current directory)') do |path|
|
|
38
|
+
options[:root] = File.realpath(path)
|
|
39
|
+
end
|
|
40
|
+
flags.on('--boot-timeout SECONDS', 'Boot/identity deadline (default: 300)') do |value|
|
|
41
|
+
options[:boot_timeout] = positive_seconds(value)
|
|
42
|
+
end
|
|
43
|
+
flags.on('--shutdown-timeout SECONDS', 'Graceful shutdown limit (default: 10)') do |value|
|
|
44
|
+
options[:shutdown_timeout] = positive_seconds(value)
|
|
45
|
+
end
|
|
46
|
+
flags.on('-h', '--help', 'Show help without booting Rails') { @help = true }
|
|
47
|
+
end
|
|
48
|
+
parser.order!(argv)
|
|
49
|
+
@output.puts(parser) if @help
|
|
50
|
+
[options, argv.empty? ? ['bin/rails', 'woods:watch'] : argv]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def positive_seconds(value)
|
|
54
|
+
number = Float(value)
|
|
55
|
+
return number if number.finite? && number.positive?
|
|
56
|
+
|
|
57
|
+
raise ArgumentError, 'timeout must be a positive finite number of seconds'
|
|
58
|
+
rescue ArgumentError, TypeError
|
|
59
|
+
raise ArgumentError, 'timeout must be a positive finite number of seconds'
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def validate_platform!
|
|
63
|
+
return if Process.respond_to?(:fork) && !RUBY_PLATFORM.match?(/mswin|mingw|java/)
|
|
64
|
+
|
|
65
|
+
raise ArgumentError,
|
|
66
|
+
'managed watching requires POSIX process groups and fork; use an external raw-task supervisor'
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def validate_command!(command, root)
|
|
70
|
+
executable = command.first
|
|
71
|
+
paths = if executable.include?(File::SEPARATOR)
|
|
72
|
+
[File.expand_path(executable, root)]
|
|
73
|
+
else
|
|
74
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).map do |entry|
|
|
75
|
+
File.expand_path(File.join(entry, executable), root)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
return if paths.any? { |path| File.file?(path) && File.executable?(path) }
|
|
79
|
+
|
|
80
|
+
raise ArgumentError, "child executable not found or not executable: #{executable}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def with_signals(supervisor)
|
|
84
|
+
handlers = %w[INT TERM].to_h { |signal| [signal, Signal.trap(signal) { supervisor.stop }] }
|
|
85
|
+
yield
|
|
86
|
+
ensure
|
|
87
|
+
handlers&.each { |signal, handler| Signal.trap(signal, handler) }
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
data/lib/woods/watch/daemon.rb
CHANGED
|
@@ -167,6 +167,8 @@ module Woods
|
|
|
167
167
|
# @param force_polling [Boolean] never use the `listen` backend — the
|
|
168
168
|
# right choice across a container bind mount, where native FS events
|
|
169
169
|
# do not propagate
|
|
170
|
+
# @param lifecycle [#call, nil] private managed-child boundary reporter
|
|
171
|
+
# @param conservative_claims [Boolean] refuse unknown/foreign managed ownership
|
|
170
172
|
# @param logger [#info, #warn, #error]
|
|
171
173
|
# rubocop:disable-next Metrics/ParameterLists -- every collaborator is
|
|
172
174
|
# injectable on purpose; that is what makes the daemon placement-agnostic
|
|
@@ -175,7 +177,8 @@ module Woods
|
|
|
175
177
|
policy: ReloadPolicy.new, debounce: DEFAULT_DEBOUNCE,
|
|
176
178
|
full_extraction_threshold: DEFAULT_FULL_EXTRACTION_THRESHOLD,
|
|
177
179
|
idle_timeout: nil, lock: nil, catch_up: true, force_polling: false,
|
|
178
|
-
boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil
|
|
180
|
+
boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil,
|
|
181
|
+
lifecycle: nil, conservative_claims: false)
|
|
179
182
|
@poll_interval = validated_poll_interval(poll_interval)
|
|
180
183
|
@output_dir = output_dir.to_s
|
|
181
184
|
@root = (root || (defined?(Rails) ? Rails.root : Dir.pwd)).to_s
|
|
@@ -190,6 +193,8 @@ module Woods
|
|
|
190
193
|
@boot_snapshot = boot_snapshot
|
|
191
194
|
@force_polling = force_polling
|
|
192
195
|
@logger = logger || default_logger
|
|
196
|
+
@lifecycle = lifecycle
|
|
197
|
+
@conservative_claims = conservative_claims
|
|
193
198
|
@generation = Generation.new(output_dir: @output_dir)
|
|
194
199
|
@status = Status.new(output_dir: @output_dir)
|
|
195
200
|
@lock = lock || default_lock
|
|
@@ -307,9 +312,7 @@ module Woods
|
|
|
307
312
|
enqueue(paths)
|
|
308
313
|
drain
|
|
309
314
|
end
|
|
310
|
-
|
|
311
|
-
catch_up unless @stop_reason
|
|
312
|
-
drain_cycles
|
|
315
|
+
reconcile_startup
|
|
313
316
|
@last_event_at = monotonic_now
|
|
314
317
|
end
|
|
315
318
|
watcher_thread.join
|
|
@@ -320,6 +323,15 @@ module Woods
|
|
|
320
323
|
shut_down(heartbeat, watcher_thread)
|
|
321
324
|
end
|
|
322
325
|
|
|
326
|
+
def reconcile_startup
|
|
327
|
+
await_watcher_ready
|
|
328
|
+
@lifecycle&.call(:backend_ready)
|
|
329
|
+
catch_up unless @stop_reason
|
|
330
|
+
drain_cycles
|
|
331
|
+
@lifecycle_started = true
|
|
332
|
+
report_lifecycle
|
|
333
|
+
end
|
|
334
|
+
|
|
323
335
|
# @param heartbeat [Thread, nil]
|
|
324
336
|
# @param watcher_thread [Thread, nil]
|
|
325
337
|
# @return [void]
|
|
@@ -499,6 +511,7 @@ module Woods
|
|
|
499
511
|
|
|
500
512
|
begin
|
|
501
513
|
drain_cycles
|
|
514
|
+
report_lifecycle if @lifecycle_started
|
|
502
515
|
ensure
|
|
503
516
|
# An extraction is work, not idleness. Stamping only on the event
|
|
504
517
|
# would let a cycle longer than `idle_timeout` read as a quiet
|
|
@@ -532,6 +545,35 @@ module Woods
|
|
|
532
545
|
@pending_mutex.synchronize { @pending.empty? }
|
|
533
546
|
end
|
|
534
547
|
|
|
548
|
+
# Only a drained startup boundary (or a later recovery) can establish
|
|
549
|
+
# maintenance readiness. The early running heartbeat does not prove it.
|
|
550
|
+
def report_lifecycle
|
|
551
|
+
return unless @lifecycle
|
|
552
|
+
return if @stop_reason && @stop_reason != :restart_required
|
|
553
|
+
|
|
554
|
+
reason = lifecycle_reason
|
|
555
|
+
state = reason == 'reconciled' ? 'ready' : 'degraded'
|
|
556
|
+
return if @lifecycle_state == [state, reason]
|
|
557
|
+
|
|
558
|
+
@lifecycle.call(:startup, state: state, generation: @generation.current.number, reason: reason)
|
|
559
|
+
@lifecycle_state = [state, reason]
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
def lifecycle_reason
|
|
563
|
+
return 'restart_required' if @stop_reason == :restart_required
|
|
564
|
+
return 'startup_failed' if @degraded_reason
|
|
565
|
+
return 'pending_work' unless pending_empty?
|
|
566
|
+
return 'catch_up_disabled' unless @catch_up
|
|
567
|
+
return 'no_index' unless managed_index_published?
|
|
568
|
+
|
|
569
|
+
'reconciled'
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
def managed_index_published?
|
|
573
|
+
@generation.current.number.positive? && !dangling_payload_pointer? &&
|
|
574
|
+
@generation.payload_dir.join('manifest.json').file?
|
|
575
|
+
end
|
|
576
|
+
|
|
535
577
|
# Carry the pending set across a restart.
|
|
536
578
|
#
|
|
537
579
|
# Within a run, carried paths survive; at shutdown they were dropped, and
|
|
@@ -583,7 +625,7 @@ module Woods
|
|
|
583
625
|
return unless @catch_up
|
|
584
626
|
|
|
585
627
|
carried = restore_pending
|
|
586
|
-
paths = (uncovered_paths + carried).uniq
|
|
628
|
+
paths = (uncovered_paths + carried + vanished_restart_paths).uniq
|
|
587
629
|
boot_changes = @boot_snapshot ? @boot_snapshot.changed_paths : []
|
|
588
630
|
enqueue(boot_changes) unless boot_changes.empty?
|
|
589
631
|
prepare_startup_reconciliation(paths)
|
|
@@ -623,7 +665,8 @@ module Woods
|
|
|
623
665
|
# graph knows every path it attributed a unit to; any of those gone from
|
|
624
666
|
# disk means the extractor's sweep has reconciling to do.
|
|
625
667
|
#
|
|
626
|
-
#
|
|
668
|
+
# Except for boot inputs handled by vanished_restart_paths, the daemon
|
|
669
|
+
# only detects; it does not name the paths. Naming them
|
|
627
670
|
# would put them in the change set, whose deletions are authoritative for
|
|
628
671
|
# any unit type — and some registered paths are nominal (on Rails < 7.1,
|
|
629
672
|
# `ActiveRecord::SchemaMigration` registers a convention path no app
|
|
@@ -631,9 +674,22 @@ module Woods
|
|
|
631
674
|
# still produces. An empty-change-set run reaches the same ghosts through
|
|
632
675
|
# the sweep, which carries the bounds that make it safe.
|
|
633
676
|
def stale_deletions?
|
|
634
|
-
|
|
677
|
+
vanished_registered_paths.any?
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
# Deleted boot inputs have no mtime and are absent from a fresh snapshot.
|
|
681
|
+
# Preserve their restart obligation so a boot that already omitted them
|
|
682
|
+
# performs a full extraction. Do not promote nominal model paths to
|
|
683
|
+
# authoritative deletions, even when application reloading is disabled.
|
|
684
|
+
def vanished_restart_paths
|
|
685
|
+
vanished_registered_paths.select do |path|
|
|
686
|
+
@policy.classify(path.delete_prefix("#{@root}/")) == :restart
|
|
687
|
+
end
|
|
688
|
+
end
|
|
635
689
|
|
|
636
|
-
|
|
690
|
+
def vanished_registered_paths
|
|
691
|
+
root_prefix = "#{@root}/"
|
|
692
|
+
persisted_registered_paths.select do |path|
|
|
637
693
|
path.start_with?(root_prefix) && !File.exist?(path)
|
|
638
694
|
end
|
|
639
695
|
end
|
|
@@ -1094,7 +1150,7 @@ module Woods
|
|
|
1094
1150
|
#
|
|
1095
1151
|
# @return [Boolean]
|
|
1096
1152
|
def another_daemon_alive?
|
|
1097
|
-
return false if ENV['WOODS_IGNORE_WATCH'] == '1'
|
|
1153
|
+
return false if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
|
|
1098
1154
|
return false unless @status.alive?
|
|
1099
1155
|
|
|
1100
1156
|
record = @status.read
|
|
@@ -1130,7 +1186,7 @@ module Woods
|
|
|
1130
1186
|
# @return [Boolean] true when this instance now holds the claim (or
|
|
1131
1187
|
# was told to skip claiming entirely)
|
|
1132
1188
|
def claim_startup?
|
|
1133
|
-
return true if ENV['WOODS_IGNORE_WATCH'] == '1'
|
|
1189
|
+
return true if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
|
|
1134
1190
|
|
|
1135
1191
|
with_claim_lock do
|
|
1136
1192
|
3.times do
|
|
@@ -1258,11 +1314,17 @@ module Woods
|
|
|
1258
1314
|
# live daemon's claim
|
|
1259
1315
|
def stale_claim?
|
|
1260
1316
|
record = JSON.parse(File.read(claim_path))
|
|
1317
|
+
return false if @conservative_claims && !verifiable_managed_claim?(record)
|
|
1261
1318
|
return true unless same_claim_host?(record['host'])
|
|
1262
1319
|
|
|
1263
1320
|
!claim_pid_alive?(record['pid'])
|
|
1264
1321
|
rescue JSON::ParserError, SystemCallError
|
|
1265
|
-
|
|
1322
|
+
!@conservative_claims
|
|
1323
|
+
end
|
|
1324
|
+
|
|
1325
|
+
def verifiable_managed_claim?(record)
|
|
1326
|
+
record.is_a?(Hash) && record['host'] == Status.host_identity &&
|
|
1327
|
+
record['pid'].is_a?(Integer) && record['pid'].positive?
|
|
1266
1328
|
end
|
|
1267
1329
|
|
|
1268
1330
|
# Mirrors {Status#alive?}'s host check: a pid is only meaningful inside
|