woods 2.0.0.beta4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +480 -477
- data/CONTRIBUTING.md +2 -2
- data/README.md +11 -26
- data/docs/AGENT_GUIDE.md +31 -12
- data/docs/AGENT_SETUP.md +17 -10
- data/docs/AUTOMATIC_MAINTENANCE.md +222 -0
- data/docs/BACKEND_MATRIX.md +13 -7
- data/docs/CLIENT_HOOKS.md +1 -1
- data/docs/CONFIGURATION_REFERENCE.md +36 -26
- data/docs/CONSOLE_MCP_SETUP.md +10 -8
- data/docs/DOCKER_SETUP.md +15 -0
- data/docs/EVALUATION.md +10 -4
- data/docs/EXTRACTOR_REFERENCE.md +14 -2
- data/docs/FAQ.md +14 -3
- data/docs/GETTING_STARTED.md +18 -17
- data/docs/INCREMENTAL_EXTRACTION.md +8 -3
- data/docs/INDEX_LAYOUT.md +2 -2
- data/docs/MCP_SERVERS.md +28 -11
- data/docs/MCP_TOOL_COOKBOOK.md +1 -1
- data/docs/MCP_WORKTREE_SETUP.md +13 -1
- data/docs/PUBLISHED_INDEX.md +1 -1
- data/docs/README.md +2 -1
- data/docs/RETRIEVAL_GUIDE.md +57 -8
- data/docs/SOURCE_FRESHNESS.md +1 -1
- data/docs/TOKEN_BENCHMARK.md +16 -10
- data/docs/TROUBLESHOOTING.md +133 -37
- data/docs/UPGRADING_TO_2.md +9 -7
- data/docs/WATCH_DAEMON.md +172 -17
- data/docs/WHY_WOODS.md +9 -5
- data/exe/woods-console +13 -11
- data/exe/woods-watch +5 -0
- data/lib/generators/woods/watch_generator.rb +53 -0
- data/lib/puma/plugin/woods.rb +10 -0
- data/lib/tasks/woods.rake +14 -0
- data/lib/woods/cache/cache_middleware.rb +6 -0
- data/lib/woods/console/stdio_transport.rb +27 -0
- data/lib/woods/extractor.rb +25 -7
- data/lib/woods/git_command.rb +6 -7
- data/lib/woods/git_provenance.rb +4 -6
- data/lib/woods/mcp/bootstrapper.rb +3 -1
- data/lib/woods/mcp/initialization_guidance.rb +1 -1
- data/lib/woods/mcp/server.rb +41 -9
- data/lib/woods/retrieval/corpus_status.rb +46 -0
- data/lib/woods/retriever.rb +19 -7
- data/lib/woods/storage/local_corpus_stats.rb +32 -0
- data/lib/woods/storage/metadata_store.rb +20 -0
- data/lib/woods/storage/vector_store.rb +10 -0
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/child_environment.rb +30 -0
- data/lib/woods/watch/cli.rb +91 -0
- data/lib/woods/watch/daemon.rb +55 -7
- data/lib/woods/watch/event_stream.rb +70 -0
- data/lib/woods/watch/guardian.rb +142 -0
- data/lib/woods/watch/installation/layout.rb +70 -0
- data/lib/woods/watch/installation/options.rb +128 -0
- data/lib/woods/watch/installation/planner.rb +128 -0
- data/lib/woods/watch/installation/probe.rb +101 -0
- data/lib/woods/watch/installation/receipt.rb +77 -0
- data/lib/woods/watch/installation/recovery.rb +64 -0
- data/lib/woods/watch/installation/templates.rb +58 -0
- data/lib/woods/watch/installation.rb +56 -0
- data/lib/woods/watch/lifecycle.rb +182 -0
- data/lib/woods/watch/managed_child.rb +113 -0
- data/lib/woods/watch/managed_cleanup.rb +48 -0
- data/lib/woods/watch/managed_process.rb +144 -0
- data/lib/woods/watch/puma_adapter.rb +87 -0
- data/lib/woods/watch/puma_child.rb +66 -0
- data/lib/woods/watch/supervision_records.rb +95 -0
- data/lib/woods/watch/supervision_status.rb +104 -0
- data/lib/woods/watch/supervisor.rb +161 -0
- data/lib/woods/watch/supervisor_reporting.rb +46 -0
- data/plugin/.claude-plugin/plugin.json +1 -1
- data/plugin/skills/woods-agent-enable/SKILL.md +1 -1
- data/plugin/skills/woods-diagnose/SKILL.md +77 -8
- data/plugin/skills/woods-investigate/SKILL.md +6 -6
- data/plugin/skills/woods-mcp-config/SKILL.md +28 -1
- data/plugin/skills/woods-setup/SKILL.md +58 -4
- metadata +35 -5
data/lib/woods/mcp/server.rb
CHANGED
|
@@ -12,6 +12,7 @@ require_relative '../atomic_file'
|
|
|
12
12
|
require_relative '../generation'
|
|
13
13
|
require_relative '../tasks'
|
|
14
14
|
require_relative '../watch/status'
|
|
15
|
+
require_relative '../watch/supervision_status'
|
|
15
16
|
require_relative '../filename_utils'
|
|
16
17
|
require_relative '../update_check'
|
|
17
18
|
require_relative '../retrieval/source_evidence'
|
|
@@ -928,6 +929,7 @@ module Woods
|
|
|
928
929
|
coerce = method(:coerce_array)
|
|
929
930
|
stale_check = method(:stale_index_result?)
|
|
930
931
|
degraded_response = method(:degraded_retrieval_response)
|
|
932
|
+
corpus_status = method(:retriever_corpus_status)
|
|
931
933
|
retrieval_mode = retriever.respond_to?(:mode) ? retriever.mode : :semantic
|
|
932
934
|
server.define_tool(
|
|
933
935
|
name: 'codebase_retrieve',
|
|
@@ -954,13 +956,13 @@ module Woods
|
|
|
954
956
|
'rails_source, test_mapping, etc.). Overrides the default test_mapping exclusion. ' \
|
|
955
957
|
'Lexical mode and explicit package/path scopes filter before limits and omit the global rank table. ' \
|
|
956
958
|
'In semantic mode, when the unfiltered top-K has no requested type, the retriever ' \
|
|
957
|
-
'falls back to rank-within-type
|
|
958
|
-
'the requested type exist in the index. The response appends a "Type rank ' \
|
|
959
|
+
'falls back to rank-within-type over the available vectors. The response appends a "Type rank ' \
|
|
959
960
|
'context" table with per-type: source, rank in unfiltered top-K, global_k, ' \
|
|
960
|
-
'total_of_type
|
|
961
|
+
'total_of_type (retrieval metadata records, not structural or embedded units). ' \
|
|
962
|
+
'Read source to tell the cases apart: in_top_k (strong match), ' \
|
|
961
963
|
'within_type_fallback (weak match surfaced by the fallback), outside_top_k ' \
|
|
962
|
-
'(
|
|
963
|
-
'(zero
|
|
964
|
+
'(retrieval metadata has this type but other requested types filled the result), absent ' \
|
|
965
|
+
'(zero retrieval metadata records of this type; structural units may still exist).'
|
|
964
966
|
},
|
|
965
967
|
packages: { type: 'array', items: { type: 'string' },
|
|
966
968
|
description: 'Exact published nearest package owners. OR within the list; AND with paths and type eligibility.' },
|
|
@@ -1015,6 +1017,16 @@ module Woods
|
|
|
1015
1017
|
)
|
|
1016
1018
|
end
|
|
1017
1019
|
if retriever
|
|
1020
|
+
corpus = corpus_status.call(retriever, include_types: false)
|
|
1021
|
+
if corpus && corpus[:state] == 'empty'
|
|
1022
|
+
next respond_err.call(
|
|
1023
|
+
'Semantic retrieval has no vectors or retrieval metadata. The structural index may still be ready. ' \
|
|
1024
|
+
'Run `woods:embed` in the application environment and reload the index, or use ' \
|
|
1025
|
+
'explicit `WOODS_RETRIEVAL_MODE=lexical` and restart for retrieval without embeddings. ' \
|
|
1026
|
+
'Use `search` and `lookup` for structural discovery.',
|
|
1027
|
+
code: :empty_index, tool: 'codebase_retrieve', corpus: corpus
|
|
1028
|
+
)
|
|
1029
|
+
end
|
|
1018
1030
|
begin
|
|
1019
1031
|
scope_options = if Retrieval::Scope.requested?(packages: packages, source_paths: source_paths)
|
|
1020
1032
|
{ packages: packages, source_paths: source_paths }
|
|
@@ -1138,12 +1150,15 @@ module Woods
|
|
|
1138
1150
|
|
|
1139
1151
|
server.define_tool(
|
|
1140
1152
|
name: 'trace_flow',
|
|
1141
|
-
description: 'Trace
|
|
1153
|
+
description: 'Trace a source-derived flow from an exact indexed unit, optionally scoped by #method. ' \
|
|
1154
|
+
'Receiverless local calls may remain unexpanded; this is not proof of runtime execution.',
|
|
1142
1155
|
input_schema: {
|
|
1143
1156
|
properties: {
|
|
1144
1157
|
entry_point: {
|
|
1145
1158
|
type: 'string',
|
|
1146
|
-
description: '
|
|
1159
|
+
description: 'Exact UnitIdentifier, optionally followed by #method (e.g., UsersController#create ' \
|
|
1160
|
+
'or CheckoutService#order). Bare names identify units, including factories; ' \
|
|
1161
|
+
'a bare method name does not locate its owning class. Use search and lookup first.'
|
|
1147
1162
|
},
|
|
1148
1163
|
depth: {
|
|
1149
1164
|
type: 'integer',
|
|
@@ -2075,7 +2090,10 @@ module Woods
|
|
|
2075
2090
|
name: 'woods_status',
|
|
2076
2091
|
description: 'Diagnose whether the Woods index and server are healthy. Returns extraction metadata ' \
|
|
2077
2092
|
'(last run, unit counts, git SHA, staleness in seconds), retriever/embedding configuration, ' \
|
|
2078
|
-
'bootstrap state (hydrated / degraded / failed + reason), feature flags
|
|
2093
|
+
'bootstrap state (hydrated / degraded / failed + reason), and feature flags. ' \
|
|
2094
|
+
'Top-level `ready` describes structural index availability. ' \
|
|
2095
|
+
'Semantic corpus diagnostics report local vector/metadata record counts separately; ' \
|
|
2096
|
+
'unknown counts are null, and nonempty counts do not prove complete embedding coverage. ' \
|
|
2079
2097
|
'Includes source-content freshness; quick scans have a 250ms budget, explicit deep scans have 5s. ' \
|
|
2080
2098
|
'Incomplete evidence is unknown. Call this first on cold connect.',
|
|
2081
2099
|
input_schema: { type: 'object', properties: {
|
|
@@ -2139,10 +2157,11 @@ module Woods
|
|
|
2139
2157
|
},
|
|
2140
2158
|
index: index_section(manifest, extracted_at, staleness, index_dir, reader, source_check),
|
|
2141
2159
|
watch: watch_section(index_dir),
|
|
2160
|
+
supervision: index_dir ? Woods::Watch::SupervisionStatus.read(index_dir.to_s) : { records: [] },
|
|
2142
2161
|
retriever: {
|
|
2143
2162
|
configured: !retriever.nil?,
|
|
2144
2163
|
class: retriever&.class&.name,
|
|
2145
|
-
**(retriever
|
|
2164
|
+
**retriever_status_fields(retriever)
|
|
2146
2165
|
},
|
|
2147
2166
|
bootstrap: bootstrap_state&.to_h,
|
|
2148
2167
|
features: retrieval_features(config, resolved, retriever)
|
|
@@ -2151,6 +2170,19 @@ module Woods
|
|
|
2151
2170
|
|
|
2152
2171
|
private
|
|
2153
2172
|
|
|
2173
|
+
def retriever_status_fields(retriever)
|
|
2174
|
+
return { mode: 'lexical' } if retriever.respond_to?(:mode) && retriever.mode == :lexical
|
|
2175
|
+
|
|
2176
|
+
corpus = retriever_corpus_status(retriever)
|
|
2177
|
+
corpus ? { corpus: corpus } : {}
|
|
2178
|
+
end
|
|
2179
|
+
|
|
2180
|
+
def retriever_corpus_status(retriever, include_types: true)
|
|
2181
|
+
return if retriever.respond_to?(:mode) && retriever.mode == :lexical
|
|
2182
|
+
|
|
2183
|
+
retriever.corpus_status(include_types: include_types) if retriever.respond_to?(:corpus_status)
|
|
2184
|
+
end
|
|
2185
|
+
|
|
2154
2186
|
# Assemble the +index+ sub-hash of woods_status, including a staleness
|
|
2155
2187
|
# gate that compares +manifest.git_sha+ against the current HEAD. The
|
|
2156
2188
|
# manifest captures +git_sha+ / +gemfile_lock_sha+ / +schema_sha+ at
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Retrieval
|
|
5
|
+
# Bounded, local diagnostics: never probe remote stores or providers.
|
|
6
|
+
# Nonempty counts establish presence, not alignment, completeness, or
|
|
7
|
+
# provider health. Metadata-only corpora can still serve non-vector paths.
|
|
8
|
+
module CorpusStatus
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# @return [Hash] state and independently known local store statistics
|
|
12
|
+
def build(vector_store, metadata_store, include_types: true)
|
|
13
|
+
vectors = local_stats(vector_store, include_types: include_types)
|
|
14
|
+
metadata = local_stats(metadata_store, include_types: include_types)
|
|
15
|
+
{ state: state(vectors[:count], metadata[:count]), vectors: vectors, metadata: metadata }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def local_stats(store, include_types:)
|
|
19
|
+
return unknown unless store.respond_to?(:local_corpus_stats)
|
|
20
|
+
|
|
21
|
+
stats = store.local_corpus_stats(include_types: include_types)
|
|
22
|
+
return unknown unless stats[:count].is_a?(Integer) && stats[:count] >= 0
|
|
23
|
+
|
|
24
|
+
stats
|
|
25
|
+
rescue StandardError, NotImplementedError
|
|
26
|
+
unknown
|
|
27
|
+
end
|
|
28
|
+
private_class_method :local_stats
|
|
29
|
+
|
|
30
|
+
def unknown
|
|
31
|
+
{ count: nil, by_type: nil, untyped_count: nil }
|
|
32
|
+
end
|
|
33
|
+
private_class_method :unknown
|
|
34
|
+
|
|
35
|
+
def state(vectors, metadata)
|
|
36
|
+
return 'unknown' if vectors.nil? || metadata.nil?
|
|
37
|
+
return 'empty' if vectors.zero? && metadata.zero?
|
|
38
|
+
return 'metadata_only' if vectors.zero?
|
|
39
|
+
return 'vectors_only' if metadata.zero?
|
|
40
|
+
|
|
41
|
+
'nonempty'
|
|
42
|
+
end
|
|
43
|
+
private_class_method :state
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
data/lib/woods/retriever.rb
CHANGED
|
@@ -13,6 +13,7 @@ require_relative 'retrieval/lexical_assembler'
|
|
|
13
13
|
require_relative 'retrieval/scope'
|
|
14
14
|
require_relative 'retrieval/scoped_vector_store'
|
|
15
15
|
require_relative 'retrieval/scoped_graph_store'
|
|
16
|
+
require_relative 'retrieval/corpus_status'
|
|
16
17
|
require_relative 'embedding/token_counter'
|
|
17
18
|
require_relative 'token_utils'
|
|
18
19
|
|
|
@@ -85,7 +86,7 @@ module Woods
|
|
|
85
86
|
# source: :in_top_k, # see enum below
|
|
86
87
|
# top_of_type_global_rank: 3, # 1-based rank in unfiltered ranked, or nil
|
|
87
88
|
# global_k: 20, # size of the unfiltered ranked list
|
|
88
|
-
# total_of_type: 183 #
|
|
89
|
+
# total_of_type: 183 # retrieval metadata records of that type
|
|
89
90
|
# }
|
|
90
91
|
# }
|
|
91
92
|
#
|
|
@@ -97,10 +98,11 @@ module Woods
|
|
|
97
98
|
# the fallback vector search returned
|
|
98
99
|
# candidates of this type. Weak match.
|
|
99
100
|
# :outside_top_k — type NOT in the unfiltered ranked list, has
|
|
100
|
-
#
|
|
101
|
+
# retrieval metadata, but the fallback did
|
|
101
102
|
# not run (other requested types filled the
|
|
102
103
|
# result). No results of this type.
|
|
103
|
-
# :absent — type has zero
|
|
104
|
+
# :absent — type has zero retrieval metadata records;
|
|
105
|
+
# structural extraction may still contain it.
|
|
104
106
|
#
|
|
105
107
|
# Nil for unfiltered queries.
|
|
106
108
|
RetrievalResult = Struct.new(:context, :sources, :classification, :strategy, :tokens_used, :budget, :trace,
|
|
@@ -199,6 +201,16 @@ module Woods
|
|
|
199
201
|
# @return [Pipeline]
|
|
200
202
|
attr_reader :pipeline, :mode, :default_budget
|
|
201
203
|
|
|
204
|
+
# Diagnose the currently served semantic stores without provider or remote
|
|
205
|
+
# store requests. Capture one pipeline so a reload cannot mix store counts.
|
|
206
|
+
# @return [Hash, nil] local corpus statistics, or nil for lexical retrieval
|
|
207
|
+
def corpus_status(include_types: true)
|
|
208
|
+
return unless mode == :semantic
|
|
209
|
+
|
|
210
|
+
current = @pipeline
|
|
211
|
+
Retrieval::CorpusStatus.build(current.vector_store, current.metadata_store, include_types: include_types)
|
|
212
|
+
end
|
|
213
|
+
|
|
202
214
|
# Optional callback invoked with the pipeline struct the moment
|
|
203
215
|
# {#retrieve} resolves it, before any pipeline work runs. Nil in
|
|
204
216
|
# production. Observability seam for the reload transaction's old-or-new
|
|
@@ -642,9 +654,9 @@ module Woods
|
|
|
642
654
|
#
|
|
643
655
|
# +top_of_type_global_rank+ is the 1-based position of the first
|
|
644
656
|
# candidate of that type in the ranked list, or nil when no candidate
|
|
645
|
-
# of that type survived ranking. +total_of_type+
|
|
646
|
-
#
|
|
647
|
-
#
|
|
657
|
+
# of that type survived ranking. +total_of_type+ counts retrieval metadata
|
|
658
|
+
# records, including chunk records and units without vectors. It does not
|
|
659
|
+
# measure structural unit coverage or embedded units. +source+ labels the bucket
|
|
648
660
|
# the type landed in so the caller doesn't infer it from a nil rank;
|
|
649
661
|
# see the RetrievalResult docstring for the four-value enum.
|
|
650
662
|
#
|
|
@@ -727,7 +739,7 @@ module Woods
|
|
|
727
739
|
return context if type_rank_context.empty?
|
|
728
740
|
|
|
729
741
|
lines = ['', '### Type rank context', '',
|
|
730
|
-
'| Type | Source | Rank in unfiltered top-K | Global K |
|
|
742
|
+
'| Type | Source | Rank in unfiltered top-K | Global K | Retrieval metadata records |',
|
|
731
743
|
'|------|--------|--------------------------|----------|----------------|']
|
|
732
744
|
type_rank_context.each do |type, info|
|
|
733
745
|
rank = info[:top_of_type_global_rank] || '—'
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Storage
|
|
5
|
+
# Counts local store entries without loading vectors or source bodies.
|
|
6
|
+
# Counts describe records (including chunks), not canonical unit coverage.
|
|
7
|
+
module LocalCorpusStats
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# @param types [Array<String, Symbol, nil>] one type per live entry
|
|
11
|
+
# @return [Hash] total, known types, and entries without a usable type
|
|
12
|
+
def from_types(types)
|
|
13
|
+
from_counts(types.tally)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# @param counts [Hash] locally grouped type => entry counts
|
|
17
|
+
# @return [Hash] entry counts with unknown types kept separate
|
|
18
|
+
def from_counts(counts)
|
|
19
|
+
by_type = Hash.new(0)
|
|
20
|
+
untyped = 0
|
|
21
|
+
counts.each do |type, count|
|
|
22
|
+
if type.nil? || type.to_s.strip.empty?
|
|
23
|
+
untyped += count
|
|
24
|
+
else
|
|
25
|
+
by_type[type.to_s] += count
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
{ count: by_type.values.sum + untyped, by_type: by_type.sort.to_h, untyped_count: untyped }
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'local_corpus_stats'
|
|
4
|
+
|
|
3
5
|
require 'json'
|
|
4
6
|
require 'fileutils'
|
|
5
7
|
require 'time'
|
|
@@ -265,6 +267,14 @@ module Woods
|
|
|
265
267
|
@data.size
|
|
266
268
|
end
|
|
267
269
|
|
|
270
|
+
# Local diagnostic capability; counts records, not embedded units.
|
|
271
|
+
# @return [Hash] metadata entry counts, including source-empty records
|
|
272
|
+
def local_corpus_stats(include_types: true)
|
|
273
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
274
|
+
|
|
275
|
+
LocalCorpusStats.from_types(@data.values.map { |record| record['type'] })
|
|
276
|
+
end
|
|
277
|
+
|
|
268
278
|
# Iterate over every stored entry, yielding +(id, metadata)+ pairs.
|
|
269
279
|
#
|
|
270
280
|
# Persistence seam for {Snapshotter::Metadata}. Yields the raw internal
|
|
@@ -465,6 +475,16 @@ module Woods
|
|
|
465
475
|
@db.get_first_value('SELECT COUNT(*) FROM units')
|
|
466
476
|
end
|
|
467
477
|
|
|
478
|
+
# Reads only grouped type counts from the local SQLite database.
|
|
479
|
+
# @return [Hash] metadata entry counts, independent of vector coverage
|
|
480
|
+
def local_corpus_stats(include_types: true)
|
|
481
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
482
|
+
|
|
483
|
+
counts = @db.execute('SELECT type, COUNT(*) AS entry_count FROM units GROUP BY type')
|
|
484
|
+
.to_h { |row| [row['type'], row['entry_count']] }
|
|
485
|
+
LocalCorpusStats.from_counts(counts)
|
|
486
|
+
end
|
|
487
|
+
|
|
468
488
|
private
|
|
469
489
|
|
|
470
490
|
# Bounded retry around a contended write, mirroring
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'set'
|
|
4
|
+
require_relative 'local_corpus_stats'
|
|
4
5
|
|
|
5
6
|
module Woods
|
|
6
7
|
module Storage
|
|
@@ -263,6 +264,15 @@ module Woods
|
|
|
263
264
|
@ids.size - @tombstones.size
|
|
264
265
|
end
|
|
265
266
|
|
|
267
|
+
# Local diagnostic capability; does not load or copy vector values.
|
|
268
|
+
# @return [Hash] vector entry counts, including chunk entries
|
|
269
|
+
def local_corpus_stats(include_types: true)
|
|
270
|
+
return { count: count, by_type: nil, untyped_count: nil } unless include_types
|
|
271
|
+
|
|
272
|
+
types = @id_to_index.values.map { |idx| metadata_value(@metadata[idx], :type) }
|
|
273
|
+
LocalCorpusStats.from_types(types)
|
|
274
|
+
end
|
|
275
|
+
|
|
266
276
|
private
|
|
267
277
|
|
|
268
278
|
# Match a filter value against a metadata value. Arrays are
|
data/lib/woods/version.rb
CHANGED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Woods
|
|
4
|
+
module Watch
|
|
5
|
+
# Restore Bundler's original environment while preserving the selected
|
|
6
|
+
# application Gemfile. Each child must resolve the current lockfile anew.
|
|
7
|
+
module ChildEnvironment
|
|
8
|
+
NIL_VALUE = 'BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL'
|
|
9
|
+
ACTIVATION_KEYS = %w[BUNDLE_BIN_PATH BUNDLER_VERSION BUNDLE_LOCKFILE].freeze
|
|
10
|
+
|
|
11
|
+
# @param env [Hash] complete intended application environment
|
|
12
|
+
# @param root [String] application working directory
|
|
13
|
+
# @return [Hash] independent environment for exec with unsetenv_others
|
|
14
|
+
def self.build(env, root:)
|
|
15
|
+
result = env.dup
|
|
16
|
+
gemfile = result['BUNDLE_GEMFILE']
|
|
17
|
+
ACTIVATION_KEYS.each { |key| result.delete(key) }
|
|
18
|
+
env.each do |key, value|
|
|
19
|
+
next unless key.start_with?('BUNDLER_ORIG_')
|
|
20
|
+
|
|
21
|
+
original = key.delete_prefix('BUNDLER_ORIG_')
|
|
22
|
+
value == NIL_VALUE ? result.delete(original) : result[original] = value
|
|
23
|
+
result.delete(key)
|
|
24
|
+
end
|
|
25
|
+
result['BUNDLE_GEMFILE'] = File.expand_path(gemfile, root) if gemfile && !gemfile.empty?
|
|
26
|
+
result
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
require_relative 'supervisor'
|
|
5
|
+
|
|
6
|
+
module Woods
|
|
7
|
+
module Watch
|
|
8
|
+
# Parses the foreground launcher contract without booting Rails.
|
|
9
|
+
class CLI
|
|
10
|
+
# @param output [#puts] diagnostic stream
|
|
11
|
+
def initialize(output: $stderr)
|
|
12
|
+
@output = output
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param argv [Array<String>] launcher options and explicit child arguments
|
|
16
|
+
# @return [Integer] process exit status
|
|
17
|
+
def run(argv)
|
|
18
|
+
options, command = parse(argv.dup)
|
|
19
|
+
return 0 if @help
|
|
20
|
+
|
|
21
|
+
validate_platform!
|
|
22
|
+
validate_command!(command, options[:root])
|
|
23
|
+
supervisor = Supervisor.new(command: command, logger: @output, **options)
|
|
24
|
+
with_signals(supervisor) { supervisor.run }
|
|
25
|
+
rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
|
|
26
|
+
@output.puts("[woods-watch] #{e.message}")
|
|
27
|
+
2
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def parse(argv)
|
|
33
|
+
@help = false
|
|
34
|
+
options = { root: Dir.pwd, boot_timeout: 300, shutdown_timeout: 10 }
|
|
35
|
+
parser = OptionParser.new do |flags|
|
|
36
|
+
flags.banner = 'Usage: woods-watch [options] -- [bin/rails woods:watch]'
|
|
37
|
+
flags.on('--root PATH', 'Application root (default: current directory)') do |path|
|
|
38
|
+
options[:root] = File.realpath(path)
|
|
39
|
+
end
|
|
40
|
+
flags.on('--boot-timeout SECONDS', 'Boot/identity deadline (default: 300)') do |value|
|
|
41
|
+
options[:boot_timeout] = positive_seconds(value)
|
|
42
|
+
end
|
|
43
|
+
flags.on('--shutdown-timeout SECONDS', 'Graceful shutdown limit (default: 10)') do |value|
|
|
44
|
+
options[:shutdown_timeout] = positive_seconds(value)
|
|
45
|
+
end
|
|
46
|
+
flags.on('-h', '--help', 'Show help without booting Rails') { @help = true }
|
|
47
|
+
end
|
|
48
|
+
parser.order!(argv)
|
|
49
|
+
@output.puts(parser) if @help
|
|
50
|
+
[options, argv.empty? ? ['bin/rails', 'woods:watch'] : argv]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def positive_seconds(value)
|
|
54
|
+
number = Float(value)
|
|
55
|
+
return number if number.finite? && number.positive?
|
|
56
|
+
|
|
57
|
+
raise ArgumentError, 'timeout must be a positive finite number of seconds'
|
|
58
|
+
rescue ArgumentError, TypeError
|
|
59
|
+
raise ArgumentError, 'timeout must be a positive finite number of seconds'
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def validate_platform!
|
|
63
|
+
return if Process.respond_to?(:fork) && !RUBY_PLATFORM.match?(/mswin|mingw|java/)
|
|
64
|
+
|
|
65
|
+
raise ArgumentError,
|
|
66
|
+
'managed watching requires POSIX process groups and fork; use an external raw-task supervisor'
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def validate_command!(command, root)
|
|
70
|
+
executable = command.first
|
|
71
|
+
paths = if executable.include?(File::SEPARATOR)
|
|
72
|
+
[File.expand_path(executable, root)]
|
|
73
|
+
else
|
|
74
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).map do |entry|
|
|
75
|
+
File.expand_path(File.join(entry, executable), root)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
return if paths.any? { |path| File.file?(path) && File.executable?(path) }
|
|
79
|
+
|
|
80
|
+
raise ArgumentError, "child executable not found or not executable: #{executable}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def with_signals(supervisor)
|
|
84
|
+
handlers = %w[INT TERM].to_h { |signal| [signal, Signal.trap(signal) { supervisor.stop }] }
|
|
85
|
+
yield
|
|
86
|
+
ensure
|
|
87
|
+
handlers&.each { |signal, handler| Signal.trap(signal, handler) }
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
data/lib/woods/watch/daemon.rb
CHANGED
|
@@ -167,6 +167,8 @@ module Woods
|
|
|
167
167
|
# @param force_polling [Boolean] never use the `listen` backend — the
|
|
168
168
|
# right choice across a container bind mount, where native FS events
|
|
169
169
|
# do not propagate
|
|
170
|
+
# @param lifecycle [#call, nil] private managed-child boundary reporter
|
|
171
|
+
# @param conservative_claims [Boolean] refuse unknown/foreign managed ownership
|
|
170
172
|
# @param logger [#info, #warn, #error]
|
|
171
173
|
# rubocop:disable-next Metrics/ParameterLists -- every collaborator is
|
|
172
174
|
# injectable on purpose; that is what makes the daemon placement-agnostic
|
|
@@ -175,7 +177,8 @@ module Woods
|
|
|
175
177
|
policy: ReloadPolicy.new, debounce: DEFAULT_DEBOUNCE,
|
|
176
178
|
full_extraction_threshold: DEFAULT_FULL_EXTRACTION_THRESHOLD,
|
|
177
179
|
idle_timeout: nil, lock: nil, catch_up: true, force_polling: false,
|
|
178
|
-
boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil
|
|
180
|
+
boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil,
|
|
181
|
+
lifecycle: nil, conservative_claims: false)
|
|
179
182
|
@poll_interval = validated_poll_interval(poll_interval)
|
|
180
183
|
@output_dir = output_dir.to_s
|
|
181
184
|
@root = (root || (defined?(Rails) ? Rails.root : Dir.pwd)).to_s
|
|
@@ -190,6 +193,8 @@ module Woods
|
|
|
190
193
|
@boot_snapshot = boot_snapshot
|
|
191
194
|
@force_polling = force_polling
|
|
192
195
|
@logger = logger || default_logger
|
|
196
|
+
@lifecycle = lifecycle
|
|
197
|
+
@conservative_claims = conservative_claims
|
|
193
198
|
@generation = Generation.new(output_dir: @output_dir)
|
|
194
199
|
@status = Status.new(output_dir: @output_dir)
|
|
195
200
|
@lock = lock || default_lock
|
|
@@ -307,9 +312,7 @@ module Woods
|
|
|
307
312
|
enqueue(paths)
|
|
308
313
|
drain
|
|
309
314
|
end
|
|
310
|
-
|
|
311
|
-
catch_up unless @stop_reason
|
|
312
|
-
drain_cycles
|
|
315
|
+
reconcile_startup
|
|
313
316
|
@last_event_at = monotonic_now
|
|
314
317
|
end
|
|
315
318
|
watcher_thread.join
|
|
@@ -320,6 +323,15 @@ module Woods
|
|
|
320
323
|
shut_down(heartbeat, watcher_thread)
|
|
321
324
|
end
|
|
322
325
|
|
|
326
|
+
def reconcile_startup
|
|
327
|
+
await_watcher_ready
|
|
328
|
+
@lifecycle&.call(:backend_ready)
|
|
329
|
+
catch_up unless @stop_reason
|
|
330
|
+
drain_cycles
|
|
331
|
+
@lifecycle_started = true
|
|
332
|
+
report_lifecycle
|
|
333
|
+
end
|
|
334
|
+
|
|
323
335
|
# @param heartbeat [Thread, nil]
|
|
324
336
|
# @param watcher_thread [Thread, nil]
|
|
325
337
|
# @return [void]
|
|
@@ -499,6 +511,7 @@ module Woods
|
|
|
499
511
|
|
|
500
512
|
begin
|
|
501
513
|
drain_cycles
|
|
514
|
+
report_lifecycle if @lifecycle_started
|
|
502
515
|
ensure
|
|
503
516
|
# An extraction is work, not idleness. Stamping only on the event
|
|
504
517
|
# would let a cycle longer than `idle_timeout` read as a quiet
|
|
@@ -532,6 +545,35 @@ module Woods
|
|
|
532
545
|
@pending_mutex.synchronize { @pending.empty? }
|
|
533
546
|
end
|
|
534
547
|
|
|
548
|
+
# Only a drained startup boundary (or a later recovery) can establish
|
|
549
|
+
# maintenance readiness. The early running heartbeat does not prove it.
|
|
550
|
+
def report_lifecycle
|
|
551
|
+
return unless @lifecycle
|
|
552
|
+
return if @stop_reason && @stop_reason != :restart_required
|
|
553
|
+
|
|
554
|
+
reason = lifecycle_reason
|
|
555
|
+
state = reason == 'reconciled' ? 'ready' : 'degraded'
|
|
556
|
+
return if @lifecycle_state == [state, reason]
|
|
557
|
+
|
|
558
|
+
@lifecycle.call(:startup, state: state, generation: @generation.current.number, reason: reason)
|
|
559
|
+
@lifecycle_state = [state, reason]
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
def lifecycle_reason
|
|
563
|
+
return 'restart_required' if @stop_reason == :restart_required
|
|
564
|
+
return 'startup_failed' if @degraded_reason
|
|
565
|
+
return 'pending_work' unless pending_empty?
|
|
566
|
+
return 'catch_up_disabled' unless @catch_up
|
|
567
|
+
return 'no_index' unless managed_index_published?
|
|
568
|
+
|
|
569
|
+
'reconciled'
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
def managed_index_published?
|
|
573
|
+
@generation.current.number.positive? && !dangling_payload_pointer? &&
|
|
574
|
+
@generation.payload_dir.join('manifest.json').file?
|
|
575
|
+
end
|
|
576
|
+
|
|
535
577
|
# Carry the pending set across a restart.
|
|
536
578
|
#
|
|
537
579
|
# Within a run, carried paths survive; at shutdown they were dropped, and
|
|
@@ -1108,7 +1150,7 @@ module Woods
|
|
|
1108
1150
|
#
|
|
1109
1151
|
# @return [Boolean]
|
|
1110
1152
|
def another_daemon_alive?
|
|
1111
|
-
return false if ENV['WOODS_IGNORE_WATCH'] == '1'
|
|
1153
|
+
return false if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
|
|
1112
1154
|
return false unless @status.alive?
|
|
1113
1155
|
|
|
1114
1156
|
record = @status.read
|
|
@@ -1144,7 +1186,7 @@ module Woods
|
|
|
1144
1186
|
# @return [Boolean] true when this instance now holds the claim (or
|
|
1145
1187
|
# was told to skip claiming entirely)
|
|
1146
1188
|
def claim_startup?
|
|
1147
|
-
return true if ENV['WOODS_IGNORE_WATCH'] == '1'
|
|
1189
|
+
return true if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
|
|
1148
1190
|
|
|
1149
1191
|
with_claim_lock do
|
|
1150
1192
|
3.times do
|
|
@@ -1272,11 +1314,17 @@ module Woods
|
|
|
1272
1314
|
# live daemon's claim
|
|
1273
1315
|
def stale_claim?
|
|
1274
1316
|
record = JSON.parse(File.read(claim_path))
|
|
1317
|
+
return false if @conservative_claims && !verifiable_managed_claim?(record)
|
|
1275
1318
|
return true unless same_claim_host?(record['host'])
|
|
1276
1319
|
|
|
1277
1320
|
!claim_pid_alive?(record['pid'])
|
|
1278
1321
|
rescue JSON::ParserError, SystemCallError
|
|
1279
|
-
|
|
1322
|
+
!@conservative_claims
|
|
1323
|
+
end
|
|
1324
|
+
|
|
1325
|
+
def verifiable_managed_claim?(record)
|
|
1326
|
+
record.is_a?(Hash) && record['host'] == Status.host_identity &&
|
|
1327
|
+
record['pid'].is_a?(Integer) && record['pid'].positive?
|
|
1280
1328
|
end
|
|
1281
1329
|
|
|
1282
1330
|
# Mirrors {Status#alive?}'s host check: a pid is only meaningful inside
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Woods
|
|
6
|
+
module Watch
|
|
7
|
+
# Bounded framing plus the guardian-issued child identity used for cleanup.
|
|
8
|
+
class EventStream
|
|
9
|
+
MAX_BYTES = 4096
|
|
10
|
+
|
|
11
|
+
def initialize(reader:, guardian:, launcher:, attempt:)
|
|
12
|
+
@reader = reader
|
|
13
|
+
@guardian = guardian
|
|
14
|
+
@identity = { 'launcher' => launcher, 'attempt' => attempt, 'pid' => guardian, 'event' => 'spawned' }
|
|
15
|
+
@buffer = +''
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
attr_reader :child_pid
|
|
19
|
+
|
|
20
|
+
def read
|
|
21
|
+
records = []
|
|
22
|
+
return records if @reader.closed?
|
|
23
|
+
|
|
24
|
+
16.times do
|
|
25
|
+
part = @reader.read_nonblock(MAX_BYTES, exception: false)
|
|
26
|
+
break if part == :wait_readable
|
|
27
|
+
return finish(records) if part.nil?
|
|
28
|
+
|
|
29
|
+
@buffer << part
|
|
30
|
+
drain_lines(records)
|
|
31
|
+
end
|
|
32
|
+
records
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def eof?
|
|
36
|
+
@eof == true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def finish(records)
|
|
42
|
+
@eof = true
|
|
43
|
+
raise ArgumentError, 'truncated Woods lifecycle record' unless @buffer.empty?
|
|
44
|
+
|
|
45
|
+
records
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def drain_lines(records)
|
|
49
|
+
while @buffer.include?("\n")
|
|
50
|
+
record = @buffer.slice!(0..@buffer.index("\n"))
|
|
51
|
+
raise ArgumentError, 'oversized Woods lifecycle record' if record.bytesize > MAX_BYTES
|
|
52
|
+
|
|
53
|
+
remember_child(record)
|
|
54
|
+
records << record
|
|
55
|
+
end
|
|
56
|
+
raise ArgumentError, 'oversized Woods lifecycle record' if @buffer.bytesize > MAX_BYTES
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def remember_child(line)
|
|
60
|
+
record = JSON.parse(line)
|
|
61
|
+
return unless record.is_a?(Hash) && @identity.all? { |key, value| record[key] == value }
|
|
62
|
+
|
|
63
|
+
child = record['child_pid']
|
|
64
|
+
@child_pid = child if child.is_a?(Integer) && child.positive?
|
|
65
|
+
rescue JSON::ParserError
|
|
66
|
+
nil # The lifecycle validator reports malformed records.
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|