woods 2.0.0.beta2 → 2.0.0.beta3

data/lib/woods/watch/daemon.rb

@@ -9,9 +9,11 @@ require_relative '../reload_policy'
 require_relative 'status'
 require_relative 'tree_scan'
 require_relative 'watcher'
+require_relative 'boot_snapshot'
 require 'json'
 require 'set'
 require 'securerandom'
+require 'timeout'
 
 module Woods
   module Watch
@@ -138,6 +140,9 @@ module Woods
       # which most daemon cycles are (a single-file cycle is milliseconds).
       HEARTBEAT_SHUTDOWN_TIMEOUT = 2
 
+      # Native registration or a polling baseline must finish before catch-up.
+      WATCHER_STARTUP_TIMEOUT = 30
+
       # @return [Woods::Generation]
       attr_reader :generation
 
@@ -156,6 +161,9 @@ module Woods
       # @param lock [Woods::Coordination::PipelineLock, nil] writer lock for
       #   this index; built from {LOCK_NAME} when nil
       # @param catch_up [Boolean] reconcile changes that predate startup
+      # @param poll_interval [Float] finite, positive seconds between polling scans
+      # @param boot_snapshot [BootSnapshot, nil] captured before environment
+      #   initialization; permits full reconciliation of unchanged startup inputs
       # @param force_polling [Boolean] never use the `listen` backend — the
       #   right choice across a container bind mount, where native FS events
       #   do not propagate
@@ -166,7 +174,9 @@ module Woods
       def initialize(output_dir:, root: nil, extractor_factory: nil, reloader: nil, watcher: nil,
                      policy: ReloadPolicy.new, debounce: DEFAULT_DEBOUNCE,
                      full_extraction_threshold: DEFAULT_FULL_EXTRACTION_THRESHOLD,
-                     idle_timeout: nil, lock: nil, catch_up: true, force_polling: false, logger: nil)
+                     idle_timeout: nil, lock: nil, catch_up: true, force_polling: false,
+                     boot_snapshot: nil, poll_interval: Watcher::DEFAULT_POLL_INTERVAL, logger: nil)
+        @poll_interval = validated_poll_interval(poll_interval)
         @output_dir = output_dir.to_s
         @root = (root || (defined?(Rails) ? Rails.root : Dir.pwd)).to_s
         @extractor_factory = extractor_factory || -> { Woods::Extractor.new(output_dir: @output_dir) }
@@ -177,6 +187,7 @@ module Woods
         @full_extraction_threshold = full_extraction_threshold
         @idle_timeout = idle_timeout
         @catch_up = catch_up
+        @boot_snapshot = boot_snapshot
         @force_polling = force_polling
         @logger = logger || default_logger
         @generation = Generation.new(output_dir: @output_dir)
@@ -231,11 +242,19 @@ module Woods
       # @return [Hash] `{ action:, state:, generation:, reason:, count:,
       #   duration_ms: }`
       def process(paths = [])
+        process_batch(paths)
+      end
+
+      private
+
+      # Only the resident loop can discharge work covered by its explicit
+      # environment-boot snapshot. Direct #process calls remain conservative.
+      def process_batch(paths = [], startup: false)
         change_set = ChangeSet.new(paths: drain_with(paths), root: @root)
         cycle_completed = false
 
         begin
-          result = case required_action(change_set)
+          result = case required_action(change_set, startup: startup)
                    when :ignore then nothing_to_do
                    when :restart then require_restart(change_set)
                    when :reload then attempt_reload ? extract(change_set) : degraded_reload(change_set)
@@ -249,13 +268,20 @@ module Woods
           # shutdown's heartbeat.kill lands on a thread doing real work) would
           # otherwise lose the batch #drain_with already popped from @pending.
           # Every branch above that finishes normally already carries its own
-          # paths forward on failure (or intentionally doesn't, e.g. :restart);
+          # paths forward on failure or restart;
           # this only fires for the abnormal case none of them can catch.
           carry_forward(change_set) unless cycle_completed
         end
       end
 
-      private
+      def validated_poll_interval(value)
+        interval = Float(value)
+        unless interval.finite? && interval.positive?
+          raise ArgumentError, 'poll_interval (WOODS_WATCH_POLL_INTERVAL) must be finite and greater than zero'
+        end
+
+        interval
+      end
 
       # The watch loop proper, split from {#run} so the shutdown `ensure` —
       # publish `stopped`, persist carried paths — can only ever fire for a
@@ -273,11 +299,19 @@ module Woods
         @last_event_at = monotonic_now
         heartbeat = start_heartbeat
 
-        watcher_thread = launch_watcher do |paths|
-          enqueue(paths)
-          drain
+        # Watch immediately, but establish startup obligations before any
+        # callback can drain them. Live callbacks enqueue under a separate mutex.
+        watcher_thread = nil
+        @drain_mutex.synchronize do
+          watcher_thread = launch_watcher do |paths|
+            enqueue(paths)
+            drain
+          end
+          await_watcher_ready
+          catch_up unless @stop_reason
+          drain_cycles
+          @last_event_at = monotonic_now
         end
-        catch_up
         watcher_thread.join
         raise @watcher_failure if @watcher_failure
 
@@ -308,7 +342,7 @@ module Woods
         # plausibly) before reaching that line, and it must still happen
         # before #persist_pending — otherwise a watcher thread still draining
         # a batch races the shutdown snapshot of @pending.
-        watcher_thread&.join
+        watcher_thread&.join(HEARTBEAT_SHUTDOWN_TIMEOUT) || watcher_thread&.kill&.join
         stop_heartbeat(heartbeat)
         # Before #persist_pending, for the same reason the watcher join is: a
         # retry drain still running would race the shutdown snapshot of
@@ -346,9 +380,18 @@ module Woods
           start_watching(&on_change)
         rescue StandardError => e
           @watcher_failure = e
+        ensure
+          @watcher_ready << :finished
         end
       end
 
+      def await_watcher_ready
+        Timeout.timeout(WATCHER_STARTUP_TIMEOUT) { @watcher_ready.pop }
+        raise @watcher_failure if @watcher_failure
+      rescue Timeout::Error
+        raise WatcherError, 'watcher did not establish its baseline before the startup timeout'
+      end
+
       # @param heartbeat [Thread, nil]
       # @return [void]
       def stop_heartbeat(heartbeat)
@@ -363,8 +406,12 @@ module Woods
       # `:reload` by restarting, since extracting against constants that no
       # longer match their source is the thing the classification exists to
       # prevent.
-      def required_action(change_set)
-        action = @policy.classify_all(change_set.relative_paths)
+      def required_action(change_set, startup: false)
+        paths = change_set.absolute_paths
+        paths = paths.reject { |path| startup_covered?(path) } if startup
+        relative = ChangeSet.new(paths: paths, root: @root).relative_paths
+        action = @policy.classify_all(relative)
+        action = :reextract if action == :ignore && @startup_full
         return :restart if action == :reload && !@reloader.enabled?
 
         action
@@ -386,10 +433,14 @@ module Woods
 
       def reset_cycle_state
         @pending = Set.new
+        @startup_paths = Set.new
+        @live_paths = Set.new
+        @startup_full = false
         @pending_mutex = Mutex.new
         @stop_reason = nil
         @drain_mutex = Mutex.new
         @retry_thread = nil
+        @watcher_ready = Queue.new
       end
 
       # Wait out the debounce window so events that land during it join the
@@ -405,9 +456,12 @@ module Woods
       end
 
       # Merge a watcher batch into the pending set without processing it.
-      def enqueue(paths)
+      def enqueue(paths, startup: false)
         absolute = ChangeSet.new(paths: paths, root: @root).absolute_paths
-        @pending_mutex.synchronize { @pending.merge(absolute) }
+        @pending_mutex.synchronize do
+          @pending.merge(absolute)
+          @live_paths.merge(absolute) unless startup
+        end
         @last_event_at = monotonic_now
       end
 
@@ -457,7 +511,7 @@ module Woods
       def drain_cycles
         until pending_empty? || @stop_reason
           settle
-          result = process
+          result = process_batch(startup: true)
 
           if result[:action] == :restart
             # @watcher, not a captured local — start_watching's polling
@@ -530,6 +584,10 @@ module Woods
 
         carried = restore_pending
         paths = (uncovered_paths + carried).uniq
+        boot_changes = @boot_snapshot ? @boot_snapshot.changed_paths : []
+        enqueue(boot_changes) unless boot_changes.empty?
+        prepare_startup_reconciliation(paths)
+        paths |= boot_changes
         if paths.empty?
           return reconcile_deletions if stale_deletions?
 
@@ -537,10 +595,28 @@ module Woods
         end
 
         @logger.info("[Woods] watch: #{paths.size} path(s) changed before startup — catching up")
-        enqueue(paths)
+        enqueue(paths, startup: true)
         drain
       end
 
+      def prepare_startup_reconciliation(paths)
+        return unless @boot_snapshot
+
+        restart_paths = paths.select do |path|
+          set = ChangeSet.new(paths: [path], root: @root)
+          required_action(set) == :restart && @boot_snapshot.covers?(path)
+        end
+        @pending_mutex.synchronize do
+          @startup_paths.merge(restart_paths)
+          @startup_full = restart_paths.any?
+        end
+      end
+
+      def startup_covered?(path)
+        eligible = @pending_mutex.synchronize { @startup_paths.include?(path) && !@live_paths.include?(path) }
+        eligible && @boot_snapshot.covers?(path)
+      end
+
       # A file deleted while nothing was watching leaves no mtime for the scan
       # to see — {TreeScan} only walks files that exist — so a deletion-only
       # downtime would log "index is current" while ghost units survive. The
@@ -759,7 +835,7 @@ module Woods
       # start at all (inotify exhaustion being the usual reason). A daemon that
       # costs some CPU beats one that silently never fires.
       def start_watching(&on_change)
-        @watcher.start(&on_change)
+        start_backend(&on_change)
       rescue WatcherError => e
         raise if @polling_fallback
 
@@ -769,8 +845,20 @@ module Woods
         # the output-directory feedback loop {#ignored_directories} exists to
         # break, on precisely the path a large tree reaches.
         @watcher = Watcher.build(
-          root: @root, ignored: ignored_directories, logger: @logger, force_polling: true
+          root: @root, ignored: ignored_directories, logger: @logger, force_polling: true,
+          poll_interval: @poll_interval
         )
+        start_backend(&on_change)
+      end
+
+      def start_backend(&on_change)
+        if @watcher.respond_to?(:ready_callback=)
+          @watcher.ready_callback = -> { @watcher_ready << :ready }
+        else
+          # Existing injected watchers own their startup contract. Built-in
+          # backends signal only after the actual baseline/registration.
+          @watcher_ready << :ready
+        end
         @watcher.start(&on_change)
       end
 
@@ -800,6 +888,7 @@ module Woods
       end
 
       def require_restart(change_set)
+        carry_forward(change_set)
         triggers = @policy.paths_requiring(change_set.relative_paths, :restart)
         reason = "restart required: #{triggers.first(5).join(', ')}"
         @logger.warn("[Woods] watch: #{reason}")
@@ -864,13 +953,7 @@ module Woods
       end
 
       def run_extraction(change_set, started)
-        # Count only paths that imply extraction work. Sixty edited markdown
-        # files plus one model is a one-model change, and reading it as a storm
-        # would trade a millisecond cycle for a full extraction.
-        actionable = actionable_count(change_set)
-        full = actionable > @full_extraction_threshold
-        log_storm(actionable) if full
-
+        full = full_extraction?(change_set)
         before = @generation.current.number
         extractor = @extractor_factory.call
         touched = if full
@@ -883,7 +966,7 @@ module Woods
         action = full ? :full : :incremental
         return unpublished(action, change_set, started) if wrote_without_publishing?(touched, before)
 
-        publish(action, change_set, touched, started)
+        finish_extraction(action, change_set, touched, started)
       rescue ScriptError, StandardError => e
         # Extraction failed, so nothing landed — the generation stays where it
         # was and the index keeps serving its last good state.
@@ -896,6 +979,28 @@ module Woods
                                      duration_ms: elapsed_ms(started))
       end
 
+      def full_extraction?(change_set)
+        if @startup_full
+          @logger.info('[Woods] watch: environment boot covers startup changes — full extraction')
+          return true
+        end
+
+        # Ignorable paths do not make a storm out of a single model edit.
+        actionable = actionable_count(change_set)
+        full = actionable > @full_extraction_threshold
+        log_storm(actionable) if full
+        full
+      end
+
+      def finish_extraction(action, change_set, touched, started)
+        result = publish(action, change_set, touched, started)
+        if action == :full
+          @startup_full = false
+          @pending_mutex.synchronize { @startup_paths.clear }
+        end
+        result
+      end
+
       def actionable_count(change_set)
         change_set.relative_paths.count { |path| @policy.classify(path) != :ignore }
       end
@@ -992,8 +1097,10 @@ module Woods
         return false if ENV['WOODS_IGNORE_WATCH'] == '1'
         return false unless @status.alive?
 
-        other = @status.read['pid']
-        return false if other.nil? || other.to_i == Process.pid
+        record = @status.read
+        other = record['pid']
+        return false if other.nil?
+        return false if same_claim_host?(record['host']) && other.to_i == Process.pid
 
         @logger.warn(
           "[Woods] a watch daemon (pid #{other}) is already maintaining #{@output_dir} — standing down. " \
@@ -1225,7 +1332,8 @@ module Woods
 
       def build_watcher
         @watcher = Watcher.build(
-          root: @root, ignored: ignored_directories, logger: @logger, force_polling: @force_polling
+          root: @root, ignored: ignored_directories, logger: @logger, force_polling: @force_polling,
+          poll_interval: @poll_interval
         )
       end
 

data/lib/woods/watch/listen_watcher.rb

@@ -15,6 +15,9 @@ module Woods
     # silent while files change under it. Hosts in that position should force
     # polling rather than trust this backend.
     class ListenWatcher
+      # Optional daemon handshake; called once detection is established.
+      attr_writer :ready_callback
+
       # @param root [String, Pathname] directory to watch
       # @param ignored [Array<String>] directory names/prefixes to skip
       # @param listen_class [Class] injectable for specs
@@ -50,6 +53,7 @@ module Woods
             on_change.call(batch) if batch.any?
           end
           @listener.start
+          @ready_callback&.call
         rescue StandardError => e
           # inotify watch exhaustion (ENOSPC) lands here, and it is the most
           # likely failure on a large tree. {Watcher.build}'s caller falls back

data/lib/woods/watch/polling_watcher.rb

@@ -18,11 +18,14 @@ module Woods
     # is what keeps that bounded: skipping `.git`, `node_modules`, `tmp` and
     # friends takes a Rails app from "everything" to "the source tree".
     class PollingWatcher
+      # Optional daemon handshake; called once detection is established.
+      attr_writer :ready_callback
+
       # @param root [String, Pathname] directory to watch
       # @param ignored [Array<String>] directory names/prefixes to skip
       # @param interval [Float] seconds between scans
       # @param sleeper [#call] injected for specs, so they need not pass time
-      def initialize(root:, ignored: Watcher::DEFAULT_IGNORED_DIRECTORIES, interval: 1.0,
+      def initialize(root:, ignored: Watcher::DEFAULT_IGNORED_DIRECTORIES, interval: Watcher::DEFAULT_POLL_INTERVAL,
                      sleeper: ->(seconds) { sleep(seconds) })
         @root = root.to_s
         @ignored = ignored
@@ -47,6 +50,7 @@ module Woods
 
         @running = true
         primed_now?
+        @ready_callback&.call
 
         while @running
           @sleeper.call(@interval)

data/lib/woods/watch/status.rb

@@ -31,9 +31,10 @@ module Woods
       STATES = %i[running degraded stopped].freeze
 
       # A status record older than this is not believed, however healthy it
-      # claims to be. Generous relative to a debounce window, because an idle
-      # daemon only rewrites its status when something happens.
+      # claims to be. The daemon heartbeats every five minutes, including
+      # while idle, so this allows two missed heartbeats.
       STALE_AFTER = 900 # 15 minutes
+      MAX_FUTURE_SKEW = 30 # seconds of clock difference tolerated across hosts
 
       # @param output_dir [String, Pathname] index directory
       # @param clock [#call] returns the ISO8601 stamp for a write
@@ -86,31 +87,34 @@ module Woods
 
       # Is a daemon currently maintaining this index?
       #
-      # Three things have to hold, and each rules out a different way the
-      # status file lies: the state has to be one a live daemon writes, the
-      # recorded pid has to still exist (a `kill -9` leaves the file behind),
-      # and the record has to be recent (a machine that lost power leaves a
-      # `running` record with a pid some unrelated process now owns).
+      # Require a live state, a positive pid, and a recent timestamp. A local
+      # pid must still exist; foreign-host trust explicitly substitutes bounded
+      # heartbeat freshness for that uncheckable process evidence. A crashed
+      # foreign daemon can therefore remain believable until the record ages out.
       #
       # Callers use this to decide whether to do the work themselves — a
       # session-start hook that would otherwise run `woods:incremental` can
       # skip it when a daemon is already on the job.
       #
       # @param max_age [Numeric] seconds after which a record is disbelieved
+      # @param trust_foreign_host [Boolean] trust a fresh foreign-host record
+      #   without a local pid check; defaults to WOODS_WATCH_TRUST_FOREIGN_HOST=1
       # @return [Boolean]
-      def alive?(max_age: STALE_AFTER)
+      def alive?(max_age: STALE_AFTER, trust_foreign_host: ENV['WOODS_WATCH_TRUST_FOREIGN_HOST'] == '1')
         record = read
         return false unless %w[running degraded].include?(record['state'])
-        return false unless same_host?(record['host'])
-        return false unless process_alive?(record['pid'])
+        return false unless record['pid'].is_a?(Integer) && record['pid'].positive?
+        return false unless recent?(record['updated_at'], max_age)
+        return trust_foreign_host unless same_host?(record['host'])
 
-        recent?(record['updated_at'], max_age)
+        process_alive?(record['pid'])
       end
 
       # The identity a pid is only meaningful within.
       #
       # In a container the hostname defaults to the container id, so this
-      # changes exactly when the pid namespace does.
+      # usually changes with the pid namespace. Custom or reused hostnames
+      # cannot establish namespace identity.
       #
       # @return [String]
       def self.host_identity
@@ -128,8 +132,8 @@ module Woods
       # deployment. A container pid like 47 almost always exists on the host, so
       # a host hook would read `running` plus a live-looking pid plus a fresh
       # timestamp and stand down while nothing was covering it. Comparing the
-      # recorded host means a cross-namespace reader disbelieves the record
-      # rather than misreading it.
+      # recorded host keeps a cross-namespace reader from checking an unrelated
+      # local pid. Foreign records are rejected unless freshness trust is enabled.
       #
       # Records written before this field existed have no host; treat them as
       # same-host so an in-place upgrade does not declare a live daemon dead.
@@ -160,7 +164,8 @@ module Woods
       def recent?(iso8601, max_age)
         return false if iso8601.nil?
 
-        Time.parse(@clock.call) - Time.parse(iso8601) <= max_age
+        age = Time.iso8601(@clock.call) - Time.iso8601(iso8601)
+        age.between?(-MAX_FUTURE_SKEW, max_age)
       rescue ArgumentError, TypeError
         false
       end

data/lib/woods/watch/tree_scan.rb

@@ -50,14 +50,14 @@ module Woods
       def each_file(root:, ignored:, visited: nil, &block)
         base = root.to_s
         prefix = "#{base}/"
-        visited ||= Set.new
+        visited = (visited || Set.new).dup
         visited << real_dir(base)
 
         Find.find(base) do |path|
           next if path == base
 
           skip = skip?(path.delete_prefix(prefix), ignored)
-          Find.prune if visit_entry(path, skip, ignored, visited, &block) == :prune
+          Find.prune if visit_entry(path, skip, ignored, visited, root: base, &block) == :prune
         end
       rescue Errno::ENOENT
         # The root vanished mid-walk (a worktree removed under us). Nothing to
@@ -68,7 +68,7 @@ module Woods
       # Classify one entry and act on it.
       #
       # @return [Symbol, nil] `:prune` when the caller should not descend
-      def visit_entry(path, skip, ignored, visited, &block)
+      def visit_entry(path, skip, ignored, visited, root:, &block)
         # `Find` stats with `lstat`, so it neither descends a symlinked
         # directory nor reports one as a file — the entry would just vanish. A
         # full extraction globs, and `Dir.glob` *does* follow them, so leaving
@@ -76,7 +76,7 @@ module Woods
         # symlinked `app/models`, or a monorepo linking shared code in, simply
         # never produced an event.
         if symlinked_directory?(path)
-          descend_symlink(path, ignored, visited, &block) unless skip
+          descend_symlink(path, ignored, visited | directory_ancestors(path, root), &block) unless skip
           nil
         elsif File.directory?(path)
           skip ? :prune : nil
@@ -86,19 +86,27 @@ module Woods
         end
       end
 
-      # Walk a symlinked directory, once.
-      #
-      # The `visited` set is keyed on the *resolved* path, which is what makes
-      # this terminate: a link pointing at its own ancestor is an infinite tree
-      # to any walker that follows links naively, and two links pointing at one
-      # directory would otherwise yield every file under it twice — enough to
-      # make a change look like two changes to the caller diffing walks.
+      # Detect cycles along this logical traversal branch, not across siblings.
+      # Two aliases of one target are distinct extraction paths: suppressing
+      # one can leave only an ignored alias and hide the relevant app path.
+      # Find does not expose exit events for ordinary directories, so collect
+      # their ancestors at each symlink rather than retaining a global set.
+      def directory_ancestors(path, root)
+        ancestors = Set.new
+        directory = File.dirname(path)
+        loop do
+          ancestors << real_dir(directory)
+          break if directory == root || File.dirname(directory) == directory
+
+          directory = File.dirname(directory)
+        end
+        ancestors
+      end
+
       def descend_symlink(path, ignored, visited, &block)
         target = real_dir(path)
         return if target.nil? || visited.include?(target)
 
-        visited << target
-
         # Walk the *resolved* directory — `Find.find` lstats even the root it is
         # handed, so pointing it at the link itself yields the link and stops —
         # then rewrite each result back under the link. Callers compare these

data/lib/woods/watch/watcher.rb

@@ -27,6 +27,9 @@ module Woods
     # Every backend implements the same two methods: `start(&on_change)`,
     # which yields an array of absolute paths, and `stop`.
     module Watcher
+      # Seconds of sleep between polling scans.
+      DEFAULT_POLL_INTERVAL = 1.0
+
       # Directory names never worth watching. Scanning them is what makes a
       # polling watcher expensive, and a change inside one is never extraction
       # input.
@@ -52,7 +55,7 @@ module Woods
       # @param force_polling [Boolean] skip the listen backend entirely
       # @param logger [#info, #warn] where backend selection is reported
       # @return [#start, #stop]
-      def build(root:, ignored: DEFAULT_IGNORED_DIRECTORIES, poll_interval: 1.0,
+      def build(root:, ignored: DEFAULT_IGNORED_DIRECTORIES, poll_interval: DEFAULT_POLL_INTERVAL,
                 force_polling: false, logger: nil)
         force_polling ||= containerized?