woods 1.6.1 → 2.0.0.beta2

data/lib/woods/watch/listen_watcher.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Woods
+  module Watch
+    # Native filesystem events via the `listen` gem.
+    #
+    # Preferred when available: no idle CPU and sub-second latency. Woods does
+    # not depend on `listen`, so {Watcher.build} only reaches for this backend
+    # when the host already has it (Rails apps usually do — it's what
+    # `ActiveSupport::EventedFileUpdateChecker` uses).
+    #
+    # Note the caveat that motivates keeping {PollingWatcher} around: native
+    # events do not propagate reliably across container bind mounts, so a
+    # daemon inside a dev container watching a mounted host directory can sit
+    # silent while files change under it. Hosts in that position should force
+    # polling rather than trust this backend.
+    class ListenWatcher
+      # @param root [String, Pathname] directory to watch
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @param listen_class [Class] injectable for specs
+      def initialize(root:, ignored: Watcher::DEFAULT_IGNORED_DIRECTORIES, listen_class: nil)
+        @root = root.to_s
+        @ignored = ignored
+        @listen_class = listen_class || ::Listen
+        # Set here rather than in #sleep_until_stopped: a `stop` arriving during
+        # startup — the daemon's signal handler racing its watcher thread, which
+        # they do by construction — used to be overwritten by the parking loop's
+        # own initialization, leaving a watcher nothing could stop.
+        @stopped = false
+      end
+
+      # Begin delivering events until {#stop}.
+      #
+      # `listen` reports modified, added and removed paths as three separate
+      # arrays; the daemon treats them uniformly (a vanished path is resolved
+      # by the extraction layer, which keys deletion on the file being gone),
+      # so they are merged into one batch.
+      #
+      # @yieldparam changed [Array<String>] absolute paths
+      # @return [void]
+      def start(&on_change)
+        # Only the setup is wrapped. `sleep_until_stopped` parks for the whole
+        # life of the daemon, so folding it into this rescue labelled every
+        # later failure — including one raised by `on_change`, i.e. by the
+        # extraction itself — as "failed to start", and sent it up as a
+        # WatcherError the daemon then treated as a dead backend.
+        begin
+          @listener = @listen_class.to(@root, ignore: ignore_patterns) do |modified, added, removed|
+            batch = (modified + added + removed).uniq
+            on_change.call(batch) if batch.any?
+          end
+          @listener.start
+        rescue StandardError => e
+          # inotify watch exhaustion (ENOSPC) lands here, and it is the most
+          # likely failure on a large tree. {Watcher.build}'s caller falls back
+          # to polling on this, which works where listen just gave up.
+          raise WatcherError, "listen backend failed to start: #{e.class}: #{e.message}"
+        end
+
+        sleep_until_stopped
+      end
+
+      # @return [void]
+      def stop
+        @stopped = true
+        @listener&.stop
+      end
+
+      private
+
+      def ignore_patterns
+        @ignored.map { |dir| %r{\A#{Regexp.escape(dir)}(/|\z)} }
+      end
+
+      # `listen` runs its own thread, so the caller's thread has to park.
+      def sleep_until_stopped
+        sleep(0.2) until @stopped
+      end
+    end
+  end
+end

data/lib/woods/watch/polling_watcher.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'set'
+
+require_relative 'tree_scan'
+
+module Woods
+  module Watch
+    # Detects changes by comparing a directory tree's mtimes between scans.
+    #
+    # The dependency-free backend, and the one that works where native FS
+    # events don't — most importantly across container bind mounts, which is
+    # the common Woods deployment. It reports creations, modifications and
+    # deletions alike, because it diffs two snapshots rather than listening
+    # for individual events.
+    #
+    # Cost is one `File.mtime` per watched file per interval. The ignore list
+    # is what keeps that bounded: skipping `.git`, `node_modules`, `tmp` and
+    # friends takes a Rails app from "everything" to "the source tree".
+    class PollingWatcher
+      # @param root [String, Pathname] directory to watch
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @param interval [Float] seconds between scans
+      # @param sleeper [#call] injected for specs, so they need not pass time
+      def initialize(root:, ignored: Watcher::DEFAULT_IGNORED_DIRECTORIES, interval: 1.0,
+                     sleeper: ->(seconds) { sleep(seconds) })
+        @root = root.to_s
+        @ignored = ignored
+        @interval = interval
+        @sleeper = sleeper
+        @running = false
+        @stop_requested = false
+        @snapshot = nil
+      end
+
+      # Scan until {#stop}, yielding each batch of changed absolute paths.
+      #
+      # A {#stop} that arrives before or during startup is honoured rather than
+      # overwritten: the daemon's signal handler and its watcher thread race by
+      # construction, and a `SIGINT` during boot used to be swallowed, leaving a
+      # watcher nobody could stop.
+      #
+      # @yieldparam changed [Array<String>] absolute paths
+      # @return [void]
+      def start(&on_change)
+        return if @stop_requested
+
+        @running = true
+        primed_now?
+
+        while @running
+          @sleeper.call(@interval)
+          break unless @running
+
+          changed = poll
+          on_change.call(changed) if changed.any?
+        end
+      end
+
+      # Stop after the current interval.
+      #
+      # @return [void]
+      def stop
+        @stop_requested = true
+        @running = false
+      end
+
+      # Take the baseline the next {#poll} compares against, if it hasn't been
+      # taken already.
+      #
+      # Separate from {#poll} so that a watcher's first cycle reports nothing:
+      # announcing every file in the tree on startup would trip the daemon's
+      # storm threshold on every boot.
+      #
+      # @return [Boolean] true when this call took the baseline
+      def primed_now?
+        return false unless @snapshot.nil?
+
+        @snapshot = scan
+        true
+      end
+
+      # Run one comparison and return what moved. Public so a caller can drive
+      # the watcher synchronously — which is exactly what the specs do, and
+      # what an embedded host that owns its own loop would do.
+      #
+      # The first call primes the baseline and reports nothing.
+      #
+      # @return [Array<String>] changed absolute paths
+      def poll
+        return [] if primed_now?
+
+        previous = @snapshot
+        @snapshot = scan
+
+        added = @snapshot.keys - previous.keys
+        removed = previous.keys - @snapshot.keys
+        modified = (@snapshot.keys & previous.keys).reject { |path| @snapshot[path] == previous[path] }
+
+        (added + removed + modified).sort
+      end
+
+      private
+
+      # Full-resolution mtime *and* size, because whole-second mtimes lose real
+      # modifications. Truncating to `to_i` means a write at T+0.1s recorded by
+      # a poll at T+0.5s and a second write at T+0.9s produce the same stamp:
+      # the next poll sees no change and no later event ever mentions the file
+      # again. Save-then-formatter inside one second is entirely ordinary at the
+      # default 1s interval, and ext4/APFS/XFS all carry sub-second mtimes.
+      #
+      # Size is the tiebreaker for the residual case — a filesystem that really
+      # only offers 1s granularity (some network and container mounts), where a
+      # same-second rewrite that changes length is still caught.
+      #
+      # @return [Hash{String => Array}] path => [mtime, size], per watched file
+      def scan
+        snapshot = {}
+
+        each_watched_path do |path|
+          stat = File.stat(path)
+          snapshot[path] = [stat.mtime.to_f, stat.size]
+        rescue SystemCallError
+          # Vanished between the glob and the stat — the next scan sees it as
+          # removed, which is the same answer one interval later.
+          next
+        end
+
+        snapshot
+      end
+
+      def each_watched_path(&block)
+        TreeScan.each_file(root: @root, ignored: @ignored, &block)
+      end
+    end
+  end
+end

data/lib/woods/watch/status.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'socket'
+
+require_relative '../atomic_file'
+
+module Woods
+  module Watch
+    # The daemon's own liveness and health, written where a reader can find it.
+    #
+    # A stale index is only dangerous when nothing says so. An agent asking
+    # `woods_status` needs to distinguish three situations that look identical
+    # from the index alone:
+    #
+    # * **running** — the index is current, or will be within a debounce window.
+    # * **degraded** — the daemon is alive but *cannot* update: a syntax error
+    #   mid-edit made the reload fail, the watcher died, extraction raised. The
+    #   index is intact and frozen at a known generation; the reason says why.
+    # * **stopped** — nothing is maintaining this index. Fall back to whatever
+    #   the last explicit run left.
+    #
+    # Degraded is the important one. #164's failure posture is that a daemon
+    # never crash-loops and never publishes a partial write, which means it
+    # spends real time in a state where the answer it can give is out of date.
+    # Saying so is the difference between a stale answer and a wrong one.
+    class Status
+      FILENAME = 'watch_status.json'
+
+      STATES = %i[running degraded stopped].freeze
+
+      # A status record older than this is not believed, however healthy it
+      # claims to be. Generous relative to a debounce window, because an idle
+      # daemon only rewrites its status when something happens.
+      STALE_AFTER = 900 # 15 minutes
+
+      # @param output_dir [String, Pathname] index directory
+      # @param clock [#call] returns the ISO8601 stamp for a write
+      def initialize(output_dir:, clock: -> { Time.now.utc.iso8601 })
+        @path = File.join(output_dir.to_s, FILENAME)
+        @clock = clock
+      end
+
+      # @return [String] absolute path to the status file
+      attr_reader :path
+
+      # Record the daemon's current state.
+      #
+      # @param state [Symbol] one of {STATES}
+      # @param generation [Integer, nil] the generation the index is at
+      # @param reason [String, nil] required in spirit for `:degraded`
+      # @param details [Hash] extra fields (pid, last batch size, timings)
+      # @return [Hash] the record as written
+      def write(state:, generation: nil, reason: nil, **details)
+        raise ArgumentError, "Unknown watch state #{state.inspect}" unless STATES.include?(state)
+
+        record = {
+          'state' => state.to_s,
+          'reason' => reason,
+          'generation' => generation,
+          'pid' => Process.pid,
+          'host' => self.class.host_identity,
+          'updated_at' => @clock.call
+        }.merge(details.transform_keys(&:to_s))
+
+        # 0644 on purpose (O1): host-side worktree hooks read this file
+        # through a bind mount (the documented deployment, see Daemon's
+        # daemon-deference check), making it the one artifact with a
+        # cross-boundary consumer. Everything else Woods writes keeps
+        # AtomicFile's restrictive 0600 default.
+        AtomicFile.write(@path, JSON.generate(record), mode: 0o644)
+        record
+      end
+
+      # The last recorded state, or a stopped record when there is none.
+      #
+      # @return [Hash] string-keyed status record
+      def read
+        return { 'state' => 'stopped', 'reason' => 'no daemon has run' } unless File.exist?(@path)
+
+        JSON.parse(AtomicFile.read(@path))
+      rescue JSON::ParserError, SystemCallError, EncodingError => e
+        { 'state' => 'stopped', 'reason' => "unreadable status file: #{e.message}" }
+      end
+
+      # Is a daemon currently maintaining this index?
+      #
+      # Three things have to hold, and each rules out a different way the
+      # status file lies: the state has to be one a live daemon writes, the
+      # recorded pid has to still exist (a `kill -9` leaves the file behind),
+      # and the record has to be recent (a machine that lost power leaves a
+      # `running` record with a pid some unrelated process now owns).
+      #
+      # Callers use this to decide whether to do the work themselves — a
+      # session-start hook that would otherwise run `woods:incremental` can
+      # skip it when a daemon is already on the job.
+      #
+      # @param max_age [Numeric] seconds after which a record is disbelieved
+      # @return [Boolean]
+      def alive?(max_age: STALE_AFTER)
+        record = read
+        return false unless %w[running degraded].include?(record['state'])
+        return false unless same_host?(record['host'])
+        return false unless process_alive?(record['pid'])
+
+        recent?(record['updated_at'], max_age)
+      end
+
+      # The identity a pid is only meaningful within.
+      #
+      # In a container the hostname defaults to the container id, so this
+      # changes exactly when the pid namespace does.
+      #
+      # @return [String]
+      def self.host_identity
+        @host_identity ||= Socket.gethostname
+      rescue StandardError
+        'unknown'
+      end
+
+      private
+
+      # A pid is only meaningful inside the namespace that issued it.
+      #
+      # The daemon runs in a dev container while the status file is read by
+      # host-side worktree hooks through a bind mount — the documented
+      # deployment. A container pid like 47 almost always exists on the host, so
+      # a host hook would read `running` plus a live-looking pid plus a fresh
+      # timestamp and stand down while nothing was covering it. Comparing the
+      # recorded host means a cross-namespace reader disbelieves the record
+      # rather than misreading it.
+      #
+      # Records written before this field existed have no host; treat them as
+      # same-host so an in-place upgrade does not declare a live daemon dead.
+      def same_host?(host)
+        host.nil? || host == self.class.host_identity
+      end
+
+      # Signal 0 asks "could I signal this process?" without sending anything.
+      # EPERM means it exists but belongs to someone else — still alive.
+      def process_alive?(pid)
+        return false unless pid.is_a?(Integer) && pid.positive?
+
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+
+      # Both sides of this comparison must come from the same clock.
+      #
+      # `@clock` is injected so a spec can drive staleness without sleeping for
+      # a quarter of an hour; reading the left-hand side from `Time.now`
+      # regardless made that injection a no-op for the one thing it exists to
+      # test — a stubbed clock could write an `updated_at` far in the past or
+      # future and `recent?` would still answer from the wall clock.
+      def recent?(iso8601, max_age)
+        return false if iso8601.nil?
+
+        Time.parse(@clock.call) - Time.parse(iso8601) <= max_age
+      rescue ArgumentError, TypeError
+        false
+      end
+    end
+  end
+end

data/lib/woods/watch/tree_scan.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'find'
+require 'set'
+
+module Woods
+  module Watch
+    # Walks a watched tree once, applying the ignore list.
+    #
+    # Extracted because two callers need exactly the same notion of "the files
+    # under this root that could be extraction input": {PollingWatcher}, which
+    # diffs two walks, and {Daemon}'s startup catch-up, which compares one walk
+    # against the index's watermark. Keeping them on one implementation means
+    # the daemon can never catch up over a different file set than the watcher
+    # subsequently watches.
+    #
+    # `Find.find` with `Find.prune`, not `Dir.glob`, and the difference is
+    # operational rather than stylistic. A glob enumerates the whole tree and
+    # filters afterwards, so it descends into `.git` and `node_modules` and
+    # stats everything inside them before discarding the results. On a monolith
+    # across a virtiofs or gRPC-FUSE bind mount — the exact deployment polling
+    # exists to serve — that alone can take longer than the poll interval,
+    # producing sustained IO and change latency measured in tens of seconds.
+    # Pruning means an ignored subtree is never entered at all.
+    #
+    # `Find` also takes the root as a plain path, which sidesteps the other
+    # trap: a root containing glob metacharacters (`[`, `{`, `*` — all legal in
+    # a directory name, and `[` shows up in generated worktree paths) makes an
+    # interpolated glob pattern match nothing, silently watching an empty tree.
+    module TreeScan
+      # Files that look like source but are editor or VCS bookkeeping. Dotfiles
+      # are skipped wholesale except where Woods genuinely reads one.
+      NOT_IGNORED_DOTFILES = ['.ruby-version'].freeze
+
+      # Dotfile *prefixes* Woods still has to see. `.env`, `.env.local`,
+      # `.env.development` and friends are boot-captured configuration
+      # ({ReloadPolicy} classifies them `:restart`), so filtering them out here
+      # would make that classification unreachable — the daemon would never
+      # learn the file changed at all.
+      NOT_IGNORED_DOTFILE_PREFIXES = ['.env'].freeze
+
+      module_function
+
+      # Yield every watched file under a root as an absolute path.
+      #
+      # @param root [String, Pathname] directory to walk
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @yieldparam path [String] absolute path to a regular file
+      # @return [void]
+      def each_file(root:, ignored:, visited: nil, &block)
+        base = root.to_s
+        prefix = "#{base}/"
+        visited ||= Set.new
+        visited << real_dir(base)
+
+        Find.find(base) do |path|
+          next if path == base
+
+          skip = skip?(path.delete_prefix(prefix), ignored)
+          Find.prune if visit_entry(path, skip, ignored, visited, &block) == :prune
+        end
+      rescue Errno::ENOENT
+        # The root vanished mid-walk (a worktree removed under us). Nothing to
+        # report; the caller's next cycle sees it gone.
+        nil
+      end
+
+      # Classify one entry and act on it.
+      #
+      # @return [Symbol, nil] `:prune` when the caller should not descend
+      def visit_entry(path, skip, ignored, visited, &block)
+        # `Find` stats with `lstat`, so it neither descends a symlinked
+        # directory nor reports one as a file — the entry would just vanish. A
+        # full extraction globs, and `Dir.glob` *does* follow them, so leaving
+        # this alone meant the daemon was blind to a tree the oracle indexes: a
+        # symlinked `app/models`, or a monorepo linking shared code in, simply
+        # never produced an event.
+        if symlinked_directory?(path)
+          descend_symlink(path, ignored, visited, &block) unless skip
+          nil
+        elsif File.directory?(path)
+          skip ? :prune : nil
+        else
+          block.call(path) unless skip
+          nil
+        end
+      end
+
+      # Walk a symlinked directory, once.
+      #
+      # The `visited` set is keyed on the *resolved* path, which is what makes
+      # this terminate: a link pointing at its own ancestor is an infinite tree
+      # to any walker that follows links naively, and two links pointing at one
+      # directory would otherwise yield every file under it twice — enough to
+      # make a change look like two changes to the caller diffing walks.
+      def descend_symlink(path, ignored, visited, &block)
+        target = real_dir(path)
+        return if target.nil? || visited.include?(target)
+
+        visited << target
+
+        # Walk the *resolved* directory — `Find.find` lstats even the root it is
+        # handed, so pointing it at the link itself yields the link and stops —
+        # then rewrite each result back under the link. Callers compare these
+        # paths against change sets and the graph's registered paths, so they
+        # have to read as the tree looks, not as it resolves.
+        each_file(root: target, ignored: ignored, visited: visited) do |file|
+          block.call(File.join(path, file.delete_prefix("#{target}/")))
+        end
+      end
+
+      # @return [Boolean] true for a symlink that resolves to a directory
+      def symlinked_directory?(path)
+        File.symlink?(path) && File.directory?(path)
+      rescue SystemCallError
+        false
+      end
+
+      # @return [String, nil] resolved path, or nil for a broken/looping link
+      def real_dir(path)
+        File.realpath(path)
+      rescue SystemCallError
+        nil
+      end
+
+      # @return [Boolean] whether this entry is neither watched nor worth
+      #   descending into
+      def skip?(relative, ignored)
+        hidden?(relative) || ignored?(relative, ignored)
+      end
+
+      # @param root [String, Pathname] directory to walk
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @return [Array<String>] absolute paths of every watched file
+      def files(root:, ignored:)
+        [].tap { |paths| each_file(root: root, ignored: ignored) { |path| paths << path } }
+      end
+
+      # @param relative [String] path relative to the watched root
+      # @return [Boolean] whether any segment is a dotfile Woods does not read
+      def hidden?(relative)
+        relative.split(File::SEPARATOR).any? { |segment| hidden_segment?(segment) }
+      end
+
+      # @param segment [String] one path component
+      # @return [Boolean]
+      def hidden_segment?(segment)
+        return false unless segment.start_with?('.')
+        return false if ['.', '..'].include?(segment)
+        return false if NOT_IGNORED_DOTFILES.include?(segment)
+
+        NOT_IGNORED_DOTFILE_PREFIXES.none? { |prefix| segment.start_with?(prefix) }
+      end
+
+      # @param relative [String] path relative to the watched root
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @return [Boolean]
+      def ignored?(relative, ignored)
+        ignored.any? { |dir| relative == dir || relative.start_with?("#{dir}/") }
+      end
+    end
+  end
+end

data/lib/woods/watch/watcher.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'woods'
+
+module Woods
+  module Watch
+    # Raised when a watcher backend cannot be started.
+    class WatcherError < Woods::Error; end
+
+    # File-change sources for the daemon.
+    #
+    # Two backends, and the choice is operational rather than aesthetic:
+    #
+    # * {ListenWatcher} wraps the `listen` gem, which uses native FS events
+    #   (inotify, FSEvents) — low latency, no idle CPU. Requires the gem,
+    #   which Woods does not depend on.
+    # * {PollingWatcher} scans mtimes on an interval. Slower and does real
+    #   work while idle, but it needs nothing and, critically, it *works
+    #   across container bind mounts* — where native events are famously
+    #   unreliable (documented by `listen` itself; macOS Docker VMs are the
+    #   usual casualty). Extraction typically runs inside a dev container with
+    #   the source bind-mounted, so this is not an edge case.
+    #
+    # {.build} prefers listen and falls back, because a daemon that silently
+    # never fires is worse than one that costs a little CPU.
+    #
+    # Every backend implements the same two methods: `start(&on_change)`,
+    # which yields an array of absolute paths, and `stop`.
+    module Watcher
+      # Directory names never worth watching. Scanning them is what makes a
+      # polling watcher expensive, and a change inside one is never extraction
+      # input.
+      DEFAULT_IGNORED_DIRECTORIES = %w[
+        .git
+        node_modules
+        tmp
+        log
+        coverage
+        vendor/bundle
+        public/assets
+        public/packs
+        storage
+      ].freeze
+
+      module_function
+
+      # Build the best available backend for a root.
+      #
+      # @param root [String, Pathname] directory to watch
+      # @param ignored [Array<String>] directory names/prefixes to skip
+      # @param poll_interval [Float] seconds between scans (polling backend)
+      # @param force_polling [Boolean] skip the listen backend entirely
+      # @param logger [#info, #warn] where backend selection is reported
+      # @return [#start, #stop]
+      def build(root:, ignored: DEFAULT_IGNORED_DIRECTORIES, poll_interval: 1.0,
+                force_polling: false, logger: nil)
+        force_polling ||= containerized?
+
+        unless force_polling
+          begin
+            require 'listen'
+            logger&.info('[Woods] watch: using native FS events (listen)')
+            return ListenWatcher.new(root: root, ignored: ignored)
+          rescue LoadError
+            logger&.info('[Woods] watch: listen gem not available, polling instead')
+          end
+        end
+
+        logger&.info("[Woods] watch: polling every #{poll_interval}s")
+        PollingWatcher.new(root: root, ignored: ignored, interval: poll_interval)
+      end
+
+      # Are we probably inside a container?
+      #
+      # The failure this guards is silent and total: on a runtime where FS
+      # events do not cross the bind mount, auto-preferring `listen` produces a
+      # daemon that reports `running`, never fires, and keeps the index frozen
+      # while claiming to maintain it. Polling on a runtime that would have
+      # propagated events costs some CPU. Those are not symmetric, so the
+      # ambiguous case defaults to the recoverable one.
+      #
+      # Deliberately a heuristic and deliberately overridable — event
+      # propagation is runtime-specific (OrbStack propagates; Docker Desktop's
+      # VM is the case `listen` itself documents as unreliable), so a host that
+      # knows better sets `WOODS_WATCH_POLL=0`.
+      #
+      # @return [Boolean]
+      def containerized?
+        return ENV['WOODS_WATCH_POLL'] != '0' if ENV.key?('WOODS_WATCH_POLL')
+
+        File.exist?('/.dockerenv') || File.exist?('/run/.containerenv')
+      rescue StandardError
+        false
+      end
+    end
+  end
+end
+
+require_relative 'polling_watcher'
+require_relative 'listen_watcher'