woods 2.0.0.beta4 → 2.0.1

data/lib/woods/retriever.rb

@@ -13,6 +13,7 @@ require_relative 'retrieval/lexical_assembler'
 require_relative 'retrieval/scope'
 require_relative 'retrieval/scoped_vector_store'
 require_relative 'retrieval/scoped_graph_store'
+require_relative 'retrieval/corpus_status'
 require_relative 'embedding/token_counter'
 require_relative 'token_utils'
 
@@ -85,7 +86,7 @@ module Woods
     #       source: :in_top_k,            # see enum below
     #       top_of_type_global_rank: 3,   # 1-based rank in unfiltered ranked, or nil
     #       global_k: 20,                 # size of the unfiltered ranked list
-    #       total_of_type: 183            # total units of that type in the index
+    #       total_of_type: 183            # retrieval metadata records of that type
     #     }
     #   }
     #
@@ -97,10 +98,11 @@ module Woods
     #                            the fallback vector search returned
     #                            candidates of this type. Weak match.
     #   :outside_top_k         — type NOT in the unfiltered ranked list, has
-    #                            units in the index, but the fallback did
+    #                            retrieval metadata, but the fallback did
     #                            not run (other requested types filled the
     #                            result). No results of this type.
-    #   :absent                — type has zero units in the index.
+    #   :absent                — type has zero retrieval metadata records;
+    #                            structural extraction may still contain it.
     #
     # Nil for unfiltered queries.
     RetrievalResult = Struct.new(:context, :sources, :classification, :strategy, :tokens_used, :budget, :trace,
@@ -199,6 +201,16 @@ module Woods
     # @return [Pipeline]
     attr_reader :pipeline, :mode, :default_budget
 
+    # Diagnose the currently served semantic stores without provider or remote
+    # store requests. Capture one pipeline so a reload cannot mix store counts.
+    # @return [Hash, nil] local corpus statistics, or nil for lexical retrieval
+    def corpus_status(include_types: true)
+      return unless mode == :semantic
+
+      current = @pipeline
+      Retrieval::CorpusStatus.build(current.vector_store, current.metadata_store, include_types: include_types)
+    end
+
     # Optional callback invoked with the pipeline struct the moment
     # {#retrieve} resolves it, before any pipeline work runs. Nil in
     # production. Observability seam for the reload transaction's old-or-new
@@ -642,9 +654,9 @@ module Woods
     #
     # +top_of_type_global_rank+ is the 1-based position of the first
     # candidate of that type in the ranked list, or nil when no candidate
-    # of that type survived ranking. +total_of_type+ is the canonical
-    # count from the metadata store — answers "does this type exist in the
-    # index at all?" independent of query match. +source+ labels the bucket
+    # of that type survived ranking. +total_of_type+ counts retrieval metadata
+    # records, including chunk records and units without vectors. It does not
+    # measure structural unit coverage or embedded units. +source+ labels the bucket
     # the type landed in so the caller doesn't infer it from a nil rank;
     # see the RetrievalResult docstring for the four-value enum.
     #
@@ -727,7 +739,7 @@ module Woods
       return context if type_rank_context.empty?
 
       lines = ['', '### Type rank context', '',
-               '| Type | Source | Rank in unfiltered top-K | Global K | Total in index |',
+               '| Type | Source | Rank in unfiltered top-K | Global K | Retrieval metadata records |',
                '|------|--------|--------------------------|----------|----------------|']
       type_rank_context.each do |type, info|
         rank = info[:top_of_type_global_rank] || '—'

data/lib/woods/storage/local_corpus_stats.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Woods
+  module Storage
+    # Counts local store entries without loading vectors or source bodies.
+    # Counts describe records (including chunks), not canonical unit coverage.
+    module LocalCorpusStats
+      module_function
+
+      # @param types [Array<String, Symbol, nil>] one type per live entry
+      # @return [Hash] total, known types, and entries without a usable type
+      def from_types(types)
+        from_counts(types.tally)
+      end
+
+      # @param counts [Hash] locally grouped type => entry counts
+      # @return [Hash] entry counts with unknown types kept separate
+      def from_counts(counts)
+        by_type = Hash.new(0)
+        untyped = 0
+        counts.each do |type, count|
+          if type.nil? || type.to_s.strip.empty?
+            untyped += count
+          else
+            by_type[type.to_s] += count
+          end
+        end
+        { count: by_type.values.sum + untyped, by_type: by_type.sort.to_h, untyped_count: untyped }
+      end
+    end
+  end
+end

data/lib/woods/storage/metadata_store.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'local_corpus_stats'
+
 require 'json'
 require 'fileutils'
 require 'time'
@@ -265,6 +267,14 @@ module Woods
           @data.size
         end
 
+        # Local diagnostic capability; counts records, not embedded units.
+        # @return [Hash] metadata entry counts, including source-empty records
+        def local_corpus_stats(include_types: true)
+          return { count: count, by_type: nil, untyped_count: nil } unless include_types
+
+          LocalCorpusStats.from_types(@data.values.map { |record| record['type'] })
+        end
+
         # Iterate over every stored entry, yielding +(id, metadata)+ pairs.
         #
         # Persistence seam for {Snapshotter::Metadata}. Yields the raw internal
@@ -465,6 +475,16 @@ module Woods
           @db.get_first_value('SELECT COUNT(*) FROM units')
         end
 
+        # Reads only grouped type counts from the local SQLite database.
+        # @return [Hash] metadata entry counts, independent of vector coverage
+        def local_corpus_stats(include_types: true)
+          return { count: count, by_type: nil, untyped_count: nil } unless include_types
+
+          counts = @db.execute('SELECT type, COUNT(*) AS entry_count FROM units GROUP BY type')
+                      .to_h { |row| [row['type'], row['entry_count']] }
+          LocalCorpusStats.from_counts(counts)
+        end
+
         private
 
         # Bounded retry around a contended write, mirroring

data/lib/woods/storage/vector_store.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'set'
+require_relative 'local_corpus_stats'
 
 module Woods
   module Storage
@@ -263,6 +264,15 @@ module Woods
           @ids.size - @tombstones.size
         end
 
+        # Local diagnostic capability; does not load or copy vector values.
+        # @return [Hash] vector entry counts, including chunk entries
+        def local_corpus_stats(include_types: true)
+          return { count: count, by_type: nil, untyped_count: nil } unless include_types
+
+          types = @id_to_index.values.map { |idx| metadata_value(@metadata[idx], :type) }
+          LocalCorpusStats.from_types(types)
+        end
+
         private
 
         # Match a filter value against a metadata value. Arrays are

data/lib/woods/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Woods
-  VERSION = '2.0.0.beta4'
+  VERSION = '2.0.1'
 end

data/lib/woods/watch/child_environment.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Woods
+  module Watch
+    # Restore Bundler's original environment while preserving the selected
+    # application Gemfile. Each child must resolve the current lockfile anew.
+    module ChildEnvironment
+      NIL_VALUE = 'BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL'
+      ACTIVATION_KEYS = %w[BUNDLE_BIN_PATH BUNDLER_VERSION BUNDLE_LOCKFILE].freeze
+
+      # @param env [Hash] complete intended application environment
+      # @param root [String] application working directory
+      # @return [Hash] independent environment for exec with unsetenv_others
+      def self.build(env, root:)
+        result = env.dup
+        gemfile = result['BUNDLE_GEMFILE']
+        ACTIVATION_KEYS.each { |key| result.delete(key) }
+        env.each do |key, value|
+          next unless key.start_with?('BUNDLER_ORIG_')
+
+          original = key.delete_prefix('BUNDLER_ORIG_')
+          value == NIL_VALUE ? result.delete(original) : result[original] = value
+          result.delete(key)
+        end
+        result['BUNDLE_GEMFILE'] = File.expand_path(gemfile, root) if gemfile && !gemfile.empty?
+        result
+      end
+    end
+  end
+end

data/lib/woods/watch/cli.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require_relative 'supervisor'
+
+module Woods
+  module Watch
+    # Parses the foreground launcher contract without booting Rails.
+    class CLI
+      # @param output [#puts] diagnostic stream
+      def initialize(output: $stderr)
+        @output = output
+      end
+
+      # @param argv [Array<String>] launcher options and explicit child arguments
+      # @return [Integer] process exit status
+      def run(argv)
+        options, command = parse(argv.dup)
+        return 0 if @help
+
+        validate_platform!
+        validate_command!(command, options[:root])
+        supervisor = Supervisor.new(command: command, logger: @output, **options)
+        with_signals(supervisor) { supervisor.run }
+      rescue OptionParser::ParseError, ArgumentError, SystemCallError => e
+        @output.puts("[woods-watch] #{e.message}")
+        2
+      end
+
+      private
+
+      def parse(argv)
+        @help = false
+        options = { root: Dir.pwd, boot_timeout: 300, shutdown_timeout: 10 }
+        parser = OptionParser.new do |flags|
+          flags.banner = 'Usage: woods-watch [options] -- [bin/rails woods:watch]'
+          flags.on('--root PATH', 'Application root (default: current directory)') do |path|
+            options[:root] = File.realpath(path)
+          end
+          flags.on('--boot-timeout SECONDS', 'Boot/identity deadline (default: 300)') do |value|
+            options[:boot_timeout] = positive_seconds(value)
+          end
+          flags.on('--shutdown-timeout SECONDS', 'Graceful shutdown limit (default: 10)') do |value|
+            options[:shutdown_timeout] = positive_seconds(value)
+          end
+          flags.on('-h', '--help', 'Show help without booting Rails') { @help = true }
+        end
+        parser.order!(argv)
+        @output.puts(parser) if @help
+        [options, argv.empty? ? ['bin/rails', 'woods:watch'] : argv]
+      end
+
+      def positive_seconds(value)
+        number = Float(value)
+        return number if number.finite? && number.positive?
+
+        raise ArgumentError, 'timeout must be a positive finite number of seconds'
+      rescue ArgumentError, TypeError
+        raise ArgumentError, 'timeout must be a positive finite number of seconds'
+      end
+
+      def validate_platform!
+        return if Process.respond_to?(:fork) && !RUBY_PLATFORM.match?(/mswin|mingw|java/)
+
+        raise ArgumentError,
+              'managed watching requires POSIX process groups and fork; use an external raw-task supervisor'
+      end
+
+      def validate_command!(command, root)
+        executable = command.first
+        paths = if executable.include?(File::SEPARATOR)
+                  [File.expand_path(executable, root)]
+                else
+                  ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).map do |entry|
+                    File.expand_path(File.join(entry, executable), root)
+                  end
+                end
+        return if paths.any? { |path| File.file?(path) && File.executable?(path) }
+
+        raise ArgumentError, "child executable not found or not executable: #{executable}"
+      end
+
+      def with_signals(supervisor)
+        handlers = %w[INT TERM].to_h { |signal| [signal, Signal.trap(signal) { supervisor.stop }] }
+        yield
+      ensure
+        handlers&.each { |signal, handler| Signal.trap(signal, handler) }
+      end
+    end
+  end
+end

data/lib/woods/watch/daemon.rb

@@ -167,6 +167,8 @@ module Woods
       # @param force_polling [Boolean] never use the `listen` backend — the
       #   right choice across a container bind mount, where native FS events
       #   do not propagate
+      # @param lifecycle [#call, nil] private managed-child boundary reporter
+      # @param conservative_claims [Boolean] refuse unknown/foreign managed ownership
       # @param logger [#info, #warn, #error]
       # rubocop:disable-next Metrics/ParameterLists -- every collaborator is
       # injectable on purpose; that is what makes the daemon placement-agnostic
@@ -175,7 +177,8 @@ module Woods
                      policy: ReloadPolicy.new, debounce: DEFAULT_DEBOUNCE,
                      full_extraction_threshold: DEFAULT_FULL_EXTRACTION_THRESHOLD,
                      idle_timeout: nil, lock: nil, catch_up: true, force_polling: false,
-                     boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil)
+                     boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil,
+                     lifecycle: nil, conservative_claims: false)
         @poll_interval = validated_poll_interval(poll_interval)
         @output_dir = output_dir.to_s
         @root = (root || (defined?(Rails) ? Rails.root : Dir.pwd)).to_s
@@ -190,6 +193,8 @@ module Woods
         @boot_snapshot = boot_snapshot
         @force_polling = force_polling
         @logger = logger || default_logger
+        @lifecycle = lifecycle
+        @conservative_claims = conservative_claims
         @generation = Generation.new(output_dir: @output_dir)
         @status = Status.new(output_dir: @output_dir)
         @lock = lock || default_lock
@@ -307,9 +312,7 @@ module Woods
             enqueue(paths)
             drain
           end
-          await_watcher_ready
-          catch_up unless @stop_reason
-          drain_cycles
+          reconcile_startup
           @last_event_at = monotonic_now
         end
         watcher_thread.join
@@ -320,6 +323,15 @@ module Woods
         shut_down(heartbeat, watcher_thread)
       end
 
+      def reconcile_startup
+        await_watcher_ready
+        @lifecycle&.call(:backend_ready)
+        catch_up unless @stop_reason
+        drain_cycles
+        @lifecycle_started = true
+        report_lifecycle
+      end
+
       # @param heartbeat [Thread, nil]
       # @param watcher_thread [Thread, nil]
       # @return [void]
@@ -499,6 +511,7 @@ module Woods
 
         begin
           drain_cycles
+          report_lifecycle if @lifecycle_started
         ensure
           # An extraction is work, not idleness. Stamping only on the event
           # would let a cycle longer than `idle_timeout` read as a quiet
@@ -532,6 +545,35 @@ module Woods
         @pending_mutex.synchronize { @pending.empty? }
       end
 
+      # Only a drained startup boundary (or a later recovery) can establish
+      # maintenance readiness. The early running heartbeat does not prove it.
+      def report_lifecycle
+        return unless @lifecycle
+        return if @stop_reason && @stop_reason != :restart_required
+
+        reason = lifecycle_reason
+        state = reason == 'reconciled' ? 'ready' : 'degraded'
+        return if @lifecycle_state == [state, reason]
+
+        @lifecycle.call(:startup, state: state, generation: @generation.current.number, reason: reason)
+        @lifecycle_state = [state, reason]
+      end
+
+      def lifecycle_reason
+        return 'restart_required' if @stop_reason == :restart_required
+        return 'startup_failed' if @degraded_reason
+        return 'pending_work' unless pending_empty?
+        return 'catch_up_disabled' unless @catch_up
+        return 'no_index' unless managed_index_published?
+
+        'reconciled'
+      end
+
+      def managed_index_published?
+        @generation.current.number.positive? && !dangling_payload_pointer? &&
+          @generation.payload_dir.join('manifest.json').file?
+      end
+
       # Carry the pending set across a restart.
       #
       # Within a run, carried paths survive; at shutdown they were dropped, and
@@ -1108,7 +1150,7 @@ module Woods
       #
       # @return [Boolean]
       def another_daemon_alive?
-        return false if ENV['WOODS_IGNORE_WATCH'] == '1'
+        return false if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
         return false unless @status.alive?
 
         record = @status.read
@@ -1144,7 +1186,7 @@ module Woods
       # @return [Boolean] true when this instance now holds the claim (or
       #   was told to skip claiming entirely)
       def claim_startup?
-        return true if ENV['WOODS_IGNORE_WATCH'] == '1'
+        return true if ENV['WOODS_IGNORE_WATCH'] == '1' && !@conservative_claims
 
         with_claim_lock do
           3.times do
@@ -1272,11 +1314,17 @@ module Woods
       #   live daemon's claim
       def stale_claim?
         record = JSON.parse(File.read(claim_path))
+        return false if @conservative_claims && !verifiable_managed_claim?(record)
         return true unless same_claim_host?(record['host'])
 
         !claim_pid_alive?(record['pid'])
       rescue JSON::ParserError, SystemCallError
-        true
+        !@conservative_claims
+      end
+
+      def verifiable_managed_claim?(record)
+        record.is_a?(Hash) && record['host'] == Status.host_identity &&
+          record['pid'].is_a?(Integer) && record['pid'].positive?
       end
 
       # Mirrors {Status#alive?}'s host check: a pid is only meaningful inside

data/lib/woods/watch/event_stream.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Woods
+  module Watch
+    # Bounded framing plus the guardian-issued child identity used for cleanup.
+    class EventStream
+      MAX_BYTES = 4096
+
+      def initialize(reader:, guardian:, launcher:, attempt:)
+        @reader = reader
+        @guardian = guardian
+        @identity = { 'launcher' => launcher, 'attempt' => attempt, 'pid' => guardian, 'event' => 'spawned' }
+        @buffer = +''
+      end
+
+      attr_reader :child_pid
+
+      def read
+        records = []
+        return records if @reader.closed?
+
+        16.times do
+          part = @reader.read_nonblock(MAX_BYTES, exception: false)
+          break if part == :wait_readable
+          return finish(records) if part.nil?
+
+          @buffer << part
+          drain_lines(records)
+        end
+        records
+      end
+
+      def eof?
+        @eof == true
+      end
+
+      private
+
+      def finish(records)
+        @eof = true
+        raise ArgumentError, 'truncated Woods lifecycle record' unless @buffer.empty?
+
+        records
+      end
+
+      def drain_lines(records)
+        while @buffer.include?("\n")
+          record = @buffer.slice!(0..@buffer.index("\n"))
+          raise ArgumentError, 'oversized Woods lifecycle record' if record.bytesize > MAX_BYTES
+
+          remember_child(record)
+          records << record
+        end
+        raise ArgumentError, 'oversized Woods lifecycle record' if @buffer.bytesize > MAX_BYTES
+      end
+
+      def remember_child(line)
+        record = JSON.parse(line)
+        return unless record.is_a?(Hash) && @identity.all? { |key, value| record[key] == value }
+
+        child = record['child_pid']
+        @child_pid = child if child.is_a?(Integer) && child.positive?
+      rescue JSON::ParserError
+        nil # The lifecycle validator reports malformed records.
+      end
+    end
+  end
+end

data/lib/woods/watch/guardian.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'io/wait'
+
+module Woods
+  module Watch
+    # Minimal pre-Bundler owner. EOF on its private parent pipe always stops
+    # the owned application group, even while an initializer is blocked.
+    class Guardian
+      # @param parent [IO] private parent-liveness reader
+      # @param events [IO] lifecycle writer
+      # @param config [Hash] argument vector and child environment
+      def initialize(parent:, events:, config:)
+        @parent = parent
+        @events = events
+        @config = config
+        @events.sync = true
+        @stopping = false
+      end
+
+      # @return [Integer] application exit status, or launch failure
+      def run
+        %w[INT TERM].each { |signal| Signal.trap(signal) { @stopping = true } }
+        report('hello')
+        spawn_child
+        monitor
+        report('exit', code: @status&.exitstatus, signal: @status&.termsig)
+        @status&.exitstatus || 1
+      rescue SystemCallError => e
+        report('spawn_error', reason: e.class.name.split('::').last)
+        127
+      ensure
+        terminate_group
+      end
+
+      private
+
+      def spawn_child
+        gate, release = IO.pipe
+        @child = fork do
+          release.close
+          @parent.close
+          Process.setpgrp
+          exit! 1 unless gate.read(1) == 'S'
+
+          gate.close
+          exec_child
+        end
+        gate.close
+        report('spawned', child_pid: @child)
+        release.write('S') if acknowledged?
+      ensure
+        release&.close unless release&.closed?
+      end
+
+      # Do not execute application code until its owner knows the child group.
+      # Guardian death before acknowledgment closes the exec gate with EOF.
+      def acknowledged?
+        until @stopping || Process.ppid != @config.fetch('owner_pid')
+          next unless @parent.wait_readable(0.05)
+
+          return @parent.read(1) == 'S'
+        end
+        false
+      end
+
+      def exec_child
+        options = { chdir: @config.fetch('root'), in: File::NULL, close_others: true, unsetenv_others: true }
+        options[3] = @events if @config['events']
+        command = @config.fetch('command')
+        Process.exec(@config.fetch('env'), [command.first, command.first], *command.drop(1), **options)
+      rescue SystemCallError => e
+        warn "[woods-watch] child executable could not start (#{e.class})"
+        exit! 127
+      end
+
+      def monitor
+        until @stopping
+          break if Process.ppid != @config.fetch('owner_pid')
+
+          result = Process.waitpid2(@child, Process::WNOHANG)
+          if result
+            @status = result.last
+            return
+          end
+          @stopping = true if @parent.wait_readable(0.05) && @parent.read_nonblock(1, exception: false).nil?
+        end
+        terminate_group
+      end
+
+      def terminate_group
+        return unless @child && !@terminated
+
+        signal_group('TERM')
+        await_child
+        signal_group('KILL')
+        @status ||= Process.waitpid2(@child).last
+        @terminated = true
+      rescue Errno::ECHILD
+        nil
+      end
+
+      def await_child
+        deadline = monotonic + @config.fetch('shutdown_timeout')
+        until @status || monotonic >= deadline
+          @status = Process.waitpid2(@child, Process::WNOHANG)&.last
+          sleep 0.02 unless @status
+        end
+      end
+
+      def signal_group(signal)
+        Process.kill(signal, -@child)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def report(event, **fields)
+        record = { version: 1, event: event, pid: Process.pid,
+                   launcher: @config['launcher'], attempt: @config['attempt'] }.merge(fields)
+        @events.write(JSON.generate(record) << "\n")
+      rescue Errno::EPIPE
+        @stopping = true
+      end
+    end
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  config_io = IO.for_fd(5)
+  config = JSON.parse(config_io.read)
+  config_io.close
+  parent = IO.for_fd(4)
+  events = IO.for_fd(3, 'w')
+  parent.close_on_exec = true
+  events.close_on_exec = true
+  exit Woods::Watch::Guardian.new(parent: parent, events: events, config: config).run
+end