woods 1.6.1 → 2.0.0.beta2

data/lib/tasks/woods.rake

@@ -15,19 +15,380 @@
 #   bundle exec rake woods:self_analyze      # Analyze gem's own source
 #   bundle exec rake woods:flow[EntryPoint]  # Generate execution flow
 
+# Reading Woods JSON artifacts with a bare File.read tags the result with the
+# process's default external encoding — US-ASCII under LANG=C, the common
+# case in CI and Docker — so a non-ASCII byte (e.g. a git branch name in the
+# manifest) raises Encoding::InvalidByteSequenceError. AtomicFile.write is
+# binmode, so the read side must go through AtomicFile.read to match.
+require 'woods/atomic_file'
+require 'woods/generation'
+require 'woods/git_command'
+
 namespace :woods do
+  # ── Multi-instance helpers (#164 phase 4) ────────────────────────────────
+  #
+  # Worktrees are disjoint by construction (each has its own Rails.root and
+  # its own output dir), so these only ever mediate writers against the *same*
+  # index: a manual rake run, a hook-triggered sync, and the watch daemon.
+
+  # Run a block holding the extraction lock, waiting for another writer to
+  # finish.
+  #
+  # This used to proceed *without* the lock after 30s, on the reasoning that a
+  # daemon cycle is milliseconds so a longer wait meant something unusual. That
+  # reasoning was wrong in the case that matters: a cycle includes a
+  # storm-triggered `extract_all`, which on a large host app runs for minutes.
+  # Proceeding then means two writers load `dependency_graph.json`, mutate
+  # divergent copies, and the last one silently discards the other's work — then
+  # bumps the generation, marking the clobbered graph fresh. Per-file atomic
+  # writes do not help, because the file *set* is not atomic.
+  #
+  # So the wait is now generous and the failure explicit. `WOODS_LOCK_WAIT`
+  # overrides it; exceeding it exits non-zero rather than corrupting the index,
+  # which is the outcome a CI job or a developer can actually act on.
+  # Fail the task when a run reported per-item errors (INF-4 / EXP-4).
+  #
+  # A printed-but-green run is invisible in the CI/cron pipelines these tasks
+  # target: a revoked API key, a full vector store, or a 401 on every page
+  # prints `Errors: N` and the job stays green while the index or the export
+  # target rots. `woods:unblocked_sync` and `woods:obsidian` already exit 1 on
+  # the same shape, and #270 fixed it for the extraction family; the embedding
+  # and Notion tasks carry identical CI exposure.
+  #
+  # Partial progress is deliberately *not* a carve-out here. The unblocked
+  # exporter has one (budget exhaustion with progress is the expected
+  # cold-start shape and converges on the next run); an embedding or Notion
+  # error is a genuine failure per unit and does not self-heal, so any
+  # non-zero count fails.
+  #
+  # @param errors [Integer, Array, nil] the run's error count or error list
+  # @return [void]
+  def woods_exit_on_reported_errors(errors)
+    # The embed indexer reports a count; the exporters report a list.
+    # (`Integer#size` exists and answers 8, so this must branch on the type.)
+    count = errors.is_a?(Numeric) ? errors.to_i : Array(errors).size
+    return if count.zero?
+
+    puts
+    puts 'Run completed with errors — failing so CI surfaces it.'
+    exit 1
+  end
+
+  def woods_with_extraction_lock(output_dir, wait: nil, &block)
+    # Requires first. The default wait reads a constant from the daemon, so
+    # resolving it above these lines NameError'd every write task — the same
+    # load-order bug as the missing require in `woods:watch`, reintroduced one
+    # method over by the fix for it.
+    require 'woods/coordination/pipeline_lock'
+    require 'woods/coordination/lock_heartbeat'
+    require 'woods/watch/daemon'
+
+    wait ||= Float(ENV.fetch('WOODS_LOCK_WAIT', Woods::Watch::Daemon::LOCK_STALE_TIMEOUT))
+
+    lock = Woods::Coordination::PipelineLock.new(
+      lock_dir: output_dir.to_s,
+      name: Woods::Watch::Daemon::LOCK_NAME,
+      stale_timeout: Woods::Watch::Daemon::LOCK_STALE_TIMEOUT
+    )
+
+    woods_abort_on_lock_timeout(wait) unless woods_acquire_within(lock, wait)
+
+    begin
+      Woods::Coordination::LockHeartbeat.run(lock, &block)
+    ensure
+      lock.release
+    end
+  end
+
+  # Poll for the lock until `wait` seconds have elapsed.
+  #
+  # Monotonic, so a clock adjustment mid-wait cannot cut the window short or
+  # extend it indefinitely.
+  #
+  # @return [Boolean] whether the lock was acquired
+  def woods_acquire_within(lock, wait)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wait
+    acquired = lock.acquire
+    until acquired || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+      sleep 0.25
+      acquired = lock.acquire
+    end
+    acquired
+  end
+
+  def woods_abort_on_lock_timeout(wait)
+    warn "ERROR: another writer has held the extraction lock for #{wait.round}s."
+    warn 'Refusing to write: two concurrent writers rewrite the dependency graph from divergent'
+    warn 'copies, and the loser\'s work is discarded under a generation that says "fresh".'
+    warn 'Set WOODS_LOCK_WAIT to wait longer, or stop the other writer.'
+    exit 1
+  end
+
+  # Is a watch daemon already maintaining this index?
+  #
+  # A session-start or worktree hook that fires `woods:incremental` on a tree
+  # a daemon is already watching is pure duplicated work — and it contends for
+  # the lock the daemon needs. Set WOODS_IGNORE_WATCH=1 to run anyway.
+  #
+  # Liveness alone is not coverage, and the difference matters. A `:running`
+  # daemon reconciles everything modified since the index's last successful
+  # publish when it starts (Daemon#catch_up), so changes that predate it are
+  # covered whether or not it witnessed them — that is what makes standing down
+  # safe. A `:degraded` daemon is alive but *cannot* currently update, so
+  # standing down for it would report success over work nothing is doing.
+  #
+  # @return [Symbol] `:none`, `:running`, or `:degraded`
+  def woods_daemon_coverage(output_dir)
+    return :none if ENV['WOODS_IGNORE_WATCH'] == '1'
+
+    require 'woods/watch/status'
+    status = Woods::Watch::Status.new(output_dir: output_dir)
+    return :none unless status.alive?
+
+    status.read['state'] == 'degraded' ? :degraded : :running
+  rescue StandardError
+    :none
+  end
+
+  # Delete the index without yanking it out from under another writer (#170).
+  #
+  # The old `woods:clean` body was a bare rm_rf. Run mid-daemon-cycle it
+  # deleted the daemon's own extraction lock, so the "writers serialize on
+  # PipelineLock" invariant evaporated at the exact moment a writer was
+  # mid-graph-rewrite. Now: refuse while a daemon is alive (same stand-down
+  # check as `woods:incremental`; WOODS_IGNORE_WATCH=1 overrides), then take
+  # the extraction lock like every other writer, delete everything EXCEPT the
+  # lock file itself, and let release remove the lock last.
+  #
+  # The directory the published generation's payload lives in.
+  #
+  # Read-side tasks resolve through this rather than the output root: an index
+  # publishing per-generation payloads keeps only `generation.json`, `dumps/`,
+  # `payloads/` and the lock files at the root. A flat index resolves to the
+  # root unchanged.
+  #
+  # @param output_dir [Pathname, String] index directory
+  # @return [Pathname]
+  def woods_payload_dir(output_dir)
+    Woods::Generation.new(output_dir: output_dir).payload_dir
+  end
+
+  # @param output_dir [Pathname, String] index directory
+  # @param wait [Numeric, nil] seconds to wait for the lock (default: the
+  #   shared writer wait; injectable so a spec need not sit out the window)
+  # @return [Symbol] `:cleaned`, or `:refused` when a live daemon is
+  #   maintaining this index
+  def woods_clean_index(output_dir, wait: nil)
+    require 'woods/watch/daemon'
+
+    output_dir = Pathname.new(output_dir)
+
+    unless woods_daemon_coverage(output_dir) == :none
+      warn 'ERROR: a watch daemon is maintaining this index — refusing to delete it out from under it.'
+      warn 'Stop the daemon first, or set WOODS_IGNORE_WATCH=1 to clean anyway.'
+      return :refused
+    end
+
+    lock_name = Woods::Watch::Daemon::LOCK_NAME
+    woods_with_extraction_lock(output_dir, wait: wait) do
+      woods_sweep_index_dir(output_dir, lock_name)
+    end
+
+    # The lock was released (and its file removed) above; drop the guard so
+    # the directory can empty, then remove the now-empty directory. Another
+    # writer may legitimately have recreated content between release and here
+    # — a non-empty directory is left alone rather than forced.
+    woods_remove_if_empty(output_dir, lock_name)
+    :cleaned
+  end
+
+  # Delete every index artifact except the lock file and its transaction
+  # guard. The guard lives inside the lock directory and a contender may hold
+  # its flock right now — deleting it mid-sweep would split the flock across
+  # two inodes and defeat the mutual exclusion it provides.
+  def woods_sweep_index_dir(output_dir, lock_name)
+    preserved = [
+      output_dir.join("#{lock_name}.lock"),
+      output_dir.join(Woods::Coordination::PipelineLock.guard_filename(lock_name))
+    ]
+    output_dir.children.each do |entry|
+      FileUtils.rm_rf(entry) unless preserved.include?(entry)
+    end
+  end
+
+  def woods_remove_if_empty(output_dir, lock_name)
+    FileUtils.rm_f(output_dir.join(Woods::Coordination::PipelineLock.guard_filename(lock_name)))
+    Dir.rmdir(output_dir)
+  rescue SystemCallError
+    nil
+  end
+
+  # The root containing the Rakefile that loaded this task file.
+  #
+  # `woods:watch_status` intentionally avoids Rails boot, so it cannot ask
+  # `Rails.root` for the conventional output path. `Dir.pwd` is not a stable
+  # substitute: `rake -f /app/Rakefile` and worktree launchers may invoke the
+  # task from somewhere else. Rake retains the selected Rakefile even when it
+  # does not chdir, which gives the same application root without loading the
+  # environment.
+  #
+  # @return [String] absolute directory containing the active Rakefile
+  def woods_task_root
+    rakefile = Rake.application.rakefile
+    return Rake.application.original_dir if rakefile.nil? || rakefile.empty?
+
+    # Rake may keep this relative (`Rakefile`) after searching upward from a
+    # nested invocation. At task execution its cwd is the directory it loaded
+    # that relative file from. An explicit absolute `-f` is already complete.
+    File.dirname(File.expand_path(rakefile))
+  end
+
+  # Changed paths across a git range, for `woods:incremental`'s CI branches.
+  #
+  # `git diff --name-only` split on lines corrupted three things at once: a
+  # path containing a newline split into two entries; a non-ASCII path came
+  # back octal-escaped inside quotes under git's default `core.quotePath`,
+  # which the dispatcher then can't match to any rule; and a rename reported
+  # only the new path, so the old path's unit was never pruned. `-z` +
+  # `--name-status` + `--no-renames` fixes all three: NUL-delimited records,
+  # `core.quotePath=false` unescaped, and a rename decomposed by git itself
+  # into a separate `A <new>` and `D <old>` record rather than one `R` record
+  # naming both.
+  #
+  # The diff is rooted at the extracted application (`git -C Rails.root`),
+  # consistent with {Woods::GitProvenance} (#262): extraction launched from
+  # another checkout must not diff that checkout's history.
+  #
+  # The child status is carried out, not discarded (M1): `Open3.capture2`
+  # turned an unresolvable range — a GitLab zero-SHA, an unfetched GitHub base
+  # ref, garbage — into an empty change set the caller could not tell apart
+  # from "nothing changed", and `woods:incremental` exited 0 over a sync that
+  # never ran.
+  #
+  # @param range [String] a git diff range/revision expression
+  # @param root [Pathname, String] repository root the diff runs against
+  # @return [Array(Array<String>, String, nil)] the changed paths (both halves
+  #   of any rename included), or nil paths plus a human-readable failure when
+  #   the range could not be resolved
+  def woods_changed_paths_for_range(range, root: Rails.root)
+    require 'open3'
+    output, error, status = Open3.capture3(
+      *Woods::GitCommand.argv(
+        root, '-c', 'core.quotePath=false',
+        'diff', '--name-status', '-z', '--no-renames', range
+      )
+    )
+    return [woods_parse_git_diff_name_status(output), nil] if status.success?
+
+    [nil, "#{error.strip} (git exited #{status.exitstatus})"]
+  rescue SystemCallError => e
+    # No git binary at all (a slim container image, git removed after
+    # checkout). Errno::ENOENT out of `Open3.capture3` used to kill the task
+    # with a backtrace: non-zero, so safe, but it bypassed the decision below
+    # — the `:running`-daemon stand-down branch was unreachable, and the
+    # operator got a stack trace instead of the remediation text. An absent
+    # binary is an unresolvable range like any other (INF-12).
+    [nil, "git unavailable: #{e.message}"]
+  end
+
+  # The changed-path set `woods:incremental` will process, or a stand-down.
+  #
+  # An explicit `CHANGED_FILES` list bypasses git entirely. Otherwise the
+  # range comes from the CI environment (GitLab's before-SHA, GitHub's base
+  # ref) or defaults to the last commit.
+  #
+  # A failed range is a decision, not a skip (M1): the changed-file set is
+  # unknown, so silently extracting nothing would exit 0 over work that never
+  # happened and leave CI drift unbounded. When a `:running` daemon maintains
+  # the index, its start-up catch-up covers whatever changed, so standing down
+  # with a printed reason is safe; a degraded daemon covers nothing, so the
+  # run fails like any other uncovered case. The decision runs BEFORE the
+  # task's empty-range exit — a failed range must never be mistaken for an
+  # empty one.
+  #
+  # @param output_dir [Pathname, String] index directory, for daemon coverage
+  # @return [Array<String>] changed paths
+  def woods_incremental_changed_paths(output_dir)
+    explicit = ENV.fetch('CHANGED_FILES', nil)
+    return explicit.split(',').map(&:strip) if explicit
+
+    range = woods_incremental_range
+    changed_files, failure = woods_changed_paths_for_range(range)
+    return changed_files unless failure
+
+    if woods_daemon_coverage(output_dir) == :running
+      puts "Could not resolve the git diff range #{range.inspect} (#{failure})."
+      puts 'A watch daemon is maintaining this index, so its catch-up covers the changed paths — standing down.'
+      puts 'To extract now anyway, repair or provide the range (check the CI env refs),'
+      puts 'set CHANGED_FILES explicitly, or run a full woods:extract.'
+      exit 0
+    end
+
+    warn "ERROR: could not resolve the git diff range #{range.inspect}: #{failure}"
+    warn 'The changed-file set is unknown, so incremental extraction would silently index nothing.'
+    warn 'Fix the range (a zero SHA or an unfetched base ref resolve to nothing), or run a full woods:extract.'
+    exit 1
+  end
+
+  # The git range `woods:incremental` diffs, from the CI environment or the
+  # last-commit default.
+  #
+  # @return [String]
+  def woods_incremental_range
+    if ENV['CI_COMMIT_BEFORE_SHA']
+      # GitLab CI
+      "#{ENV['CI_COMMIT_BEFORE_SHA']}..#{ENV.fetch('CI_COMMIT_SHA', nil)}"
+    elsif ENV['GITHUB_BASE_REF']
+      # GitHub Actions PR
+      "origin/#{ENV['GITHUB_BASE_REF']}...HEAD"
+    else
+      # Default: changes since last commit
+      'HEAD~1'
+    end
+  end
+
+  # Parse NUL-delimited `git diff --name-status -z --no-renames` output.
+  #
+  # Each record is `<status>\0<path>\0` — `--no-renames` guarantees a single
+  # path per record, since it is what stops git emitting a two-path `R`/`C`
+  # record in the first place. Status letters are not inspected beyond "did
+  # git report anything at all"; a deleted path still needs to reach the
+  # change set so its unit can be pruned.
+  #
+  # A bare `Open3.capture2` read is tagged with the process's default
+  # external encoding — US-ASCII under `LANG=C`, this daemon's usual
+  # environment (see `AtomicFile.read`'s gotcha) — so a UTF-8 path is a
+  # US-ASCII string containing invalid bytes until re-tagged.
+  #
+  # @param output [String] raw NUL-delimited git output
+  # @return [Array<String>] changed paths
+  def woods_parse_git_diff_name_status(output)
+    fields = output.dup.force_encoding(Encoding::UTF_8).split("\x00")
+    paths = []
+    fields.each_slice(2) do |status, path|
+      break if path.nil?
+
+      paths << path unless status.nil? || status.empty?
+    end
+    paths
+  end
+
   desc 'Full extraction of codebase for indexing'
   task extract: :environment do
     require 'woods/extractor'
 
-    output_dir = ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods'))
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
 
     puts 'Starting full codebase extraction...'
     puts "Output directory: #{output_dir}"
     puts
 
     extractor = Woods::Extractor.new(output_dir: output_dir)
-    results = extractor.extract_all
+    results = woods_with_extraction_lock(output_dir) do
+      extracted = extractor.extract_all
+      extractor.raise_on_publication_failure!
+      extracted
+    end
 
     puts
     puts 'Extraction complete!'
@@ -48,70 +409,48 @@ namespace :woods do
   task incremental: :environment do
     require 'woods/extractor'
 
-    output_dir = ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods'))
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
 
-    # Determine changed files from CI environment or git
-    require 'open3'
+    # Determine changed files from CI environment or git. A failed range is
+    # decided here too — stand down under a covering daemon, or fail loudly —
+    # BEFORE the empty-range exit below can mistake it for "nothing changed"
+    # (M1).
+    changed_files = woods_incremental_changed_paths(output_dir)
 
-    changed_files = if ENV['CHANGED_FILES']
-                      # Explicit list from CI
-                      ENV['CHANGED_FILES'].split(',').map(&:strip)
-                    elsif ENV['CI_COMMIT_BEFORE_SHA']
-                      # GitLab CI
-                      output, = Open3.capture2('git', 'diff', '--name-only',
-                                               "#{ENV['CI_COMMIT_BEFORE_SHA']}..#{ENV.fetch('CI_COMMIT_SHA', nil)}")
-                      output.lines.map(&:strip)
-                    elsif ENV['GITHUB_BASE_REF']
-                      # GitHub Actions PR
-                      output, = Open3.capture2('git', 'diff', '--name-only',
-                                               "origin/#{ENV['GITHUB_BASE_REF']}...HEAD")
-                      output.lines.map(&:strip)
-                    else
-                      # Default: changes since last commit
-                      output, = Open3.capture2('git', 'diff', '--name-only', 'HEAD~1')
-                      output.lines.map(&:strip)
-                    end
-
-    # Filter to relevant files
-    relevant_patterns = [
-      %r{^app/models/},
-      %r{^app/controllers/},
-      %r{^app/services/},
-      %r{^app/components/},
-      %r{^app/views/components/},
-      %r{^app/views/.*\.rb$},  # Phlex views
-      %r{^app/interactors/},
-      %r{^app/operations/},
-      %r{^app/commands/},
-      %r{^app/use_cases/},
-      %r{^app/jobs/},
-      %r{^app/workers/},       # Sidekiq workers
-      %r{^app/mailers/},
-      %r{^app/graphql/}, # GraphQL types/mutations/resolvers
-      %r{^app/serializers/},
-      %r{^app/decorators/},
-      %r{^app/blueprinters/},
-      %r{^db/migrate/},
-      %r{^db/schema\.rb$}, # Schema changes affect model metadata
-      %r{^config/routes\.rb$},
-      /^Gemfile\.lock$/ # Dependency changes trigger framework re-index
-    ]
-
-    changed_files = changed_files.select do |f|
-      relevant_patterns.any? { |p| f.match?(p) }
-    end
+    # Filter to paths that imply extraction work. The rule set lives in
+    # PathDispatcher alongside the dispatch itself — a second hand-maintained
+    # pattern list here would drift, and a path this filter drops never
+    # reaches the index however good the dispatch behind it is (#164).
+    dispatcher = Woods::PathDispatcher.new
+    changed_files = changed_files.reject(&:empty?).select { |f| dispatcher.relevant?(f) }
 
     if changed_files.empty?
       puts 'No relevant files changed. Skipping extraction.'
       exit 0
     end
 
+    case woods_daemon_coverage(output_dir)
+    when :running
+      puts 'A watch daemon is maintaining this index — skipping.'
+      puts 'It reconciles anything changed since the last publish when it starts, so these are covered.'
+      puts 'Set WOODS_IGNORE_WATCH=1 to extract anyway.'
+      exit 0
+    when :degraded
+      # Alive but unable to update. Standing down here would exit 0 over work
+      # nothing is actually doing.
+      puts 'Warning: a watch daemon is alive but degraded — extracting anyway rather than assuming coverage.'
+    end
+
     puts "Incremental extraction for #{changed_files.size} changed files..."
     changed_files.each { |f| puts "  - #{f}" }
     puts
 
     extractor = Woods::Extractor.new(output_dir: output_dir)
-    affected = extractor.extract_changed(changed_files)
+    affected = woods_with_extraction_lock(output_dir) do
+      extracted = extractor.extract_changed(changed_files)
+      extractor.raise_on_publication_failure!
+      extracted
+    end
 
     puts
     puts "Re-extracted #{affected.size} affected units."
@@ -120,99 +459,190 @@ namespace :woods do
   desc 'Tend the garden — incremental extraction (alias for incremental)'
   task tend: :incremental
 
+  desc 'Watch the app and keep the index current (resident daemon)'
+  task watch: :environment do
+    # Both, and the extractor is not optional. The daemon's default
+    # extractor_factory names Woods::Extractor lazily, so omitting this require
+    # loaded and started cleanly and then NameError'd on the first real cycle —
+    # which the failure posture turns into a permanently degraded daemon rather
+    # than a crash. Every spec pre-requires the extractor in its own setup, so
+    # the suite stayed green over a broken entry point.
+    require 'woods/extractor'
+    require 'woods/watch/daemon'
+
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
+
+    daemon = Woods::Watch::Daemon.new(
+      output_dir: output_dir,
+      root: Rails.root,
+      debounce: Float(ENV.fetch('WOODS_WATCH_DEBOUNCE', Woods::Watch::Daemon::DEFAULT_DEBOUNCE)),
+      full_extraction_threshold: Integer(
+        ENV.fetch('WOODS_WATCH_FULL_THRESHOLD', Woods::Watch::Daemon::DEFAULT_FULL_EXTRACTION_THRESHOLD)
+      ),
+      # The documented fix for a container watching a bind mount, where native
+      # FS events do not propagate and the daemon would sit silent. Nothing
+      # exposed it before, which made the advice unfollowable.
+      force_polling: ENV['WOODS_WATCH_POLL'] == '1', # container autodetect also applies; see Watcher.containerized?
+      idle_timeout: ENV.fetch('WOODS_WATCH_IDLE_TIMEOUT', nil) && Float(ENV.fetch('WOODS_WATCH_IDLE_TIMEOUT')),
+      catch_up: ENV['WOODS_WATCH_CATCH_UP'] != '0',
+      logger: Rails.logger
+    )
+
+    # The trap sets a flag and nothing else. `daemon.stop` reaches
+    # `Listen::Listener#stop`, which drives a state machine behind mutexes —
+    # and taking a mutex in trap context raises ThreadError on some Ruby
+    # versions, turning Ctrl-C into a crash instead of a clean shutdown. A tiny
+    # supervisor thread does the real work outside trap context.
+    stop_requested = Queue.new
+    %w[INT TERM].each { |sig| Signal.trap(sig) { stop_requested.push(sig) } }
+    Thread.new do
+      stop_requested.pop
+      daemon.stop
+    end
+
+    puts "Watching #{Rails.root} — index at #{output_dir}"
+    puts 'Ctrl-C to stop.'
+    puts
+
+    reason = daemon.run
+
+    if reason == :restart_required
+      # Boot-captured state changed; Rails cannot reload it. Exit non-zero so
+      # a supervisor (foreman, systemd, `docker compose` restart policy)
+      # brings the process back with the new configuration.
+      warn 'Restart required — boot-captured configuration changed. Exiting for a supervisor to restart.'
+      exit 75 # EX_TEMPFAIL
+    end
+
+    puts 'Watcher stopped.'
+  end
+
+  desc 'Keep watch over the woods — resident index daemon (alias for watch)'
+  task guard: :watch
+
+  desc 'Report whether a watch daemon is maintaining this index (exit 0 if alive)'
+  # Deliberately not `=> :environment`. This reads one small JSON file, and the
+  # whole point is that a worktree hook can call it before deciding whether to
+  # do real work — paying a full Rails boot to find out would cost more than the
+  # sync it is trying to avoid. WOODS_OUTPUT covers the non-default layout;
+  # otherwise the conventional path is derived without booting.
+  task :watch_status do
+    require 'woods/watch/status'
+    require 'json'
+
+    output_dir = ENV.fetch('WOODS_OUTPUT') { File.join(woods_task_root, 'tmp/woods') }
+    status = Woods::Watch::Status.new(output_dir: output_dir)
+
+    puts JSON.pretty_generate(status.read)
+    # Exit status is the point: a worktree hook can `rake woods:watch_status ||
+    # start_daemon` without parsing anything.
+    exit(status.alive? ? 0 : 1)
+  end
+
+  desc 'Re-run named extractors wholesale, e.g. woods:refresh[routes,middleware]'
+  task :refresh, [:extractor] => :environment do |_task, args|
+    require 'woods/extractor'
+
+    keys = [args[:extractor], *args.extras].compact.map(&:strip).reject(&:empty?)
+
+    if keys.empty?
+      puts 'Usage: rake "woods:refresh[routes]"  (comma-separate for several)'
+      puts
+      puts 'Whole-app extractors — no per-file entry point, so these are the'
+      puts 'ones a targeted refresh is normally for:'
+      puts "  #{Woods::Extractor::WHOLE_APP_EXTRACTORS.keys.sort.join(', ')}"
+      puts
+      puts 'Any extractor key is accepted:'
+      puts "  #{Woods::Extractor::EXTRACTORS.keys.sort.join(', ')}"
+      exit 1
+    end
+
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
+    extractor = Woods::Extractor.new(output_dir: output_dir)
+
+    # A refresh is a fourth writer against this index, and it rewrites the whole
+    # dependency graph. Two writers loading the persisted graph, mutating
+    # divergent copies and writing back means the last one silently discards the
+    # other's work — and then bumps the generation, telling readers the
+    # clobbered state is fresh. Atomic writes do not help: each write is
+    # individually intact, the *set* is not. So it serializes like the others.
+    begin
+      result = woods_with_extraction_lock(output_dir) do
+        refreshed = extractor.refresh(*keys)
+        extractor.raise_on_publication_failure!
+        refreshed
+      end
+    rescue ArgumentError => e
+      puts "ERROR: #{e.message}"
+      puts "Known extractors: #{Woods::Extractor::EXTRACTORS.keys.sort.join(', ')}"
+      exit 1
+    end
+
+    puts "Refreshed: #{result[:types].join(', ')}"
+    puts "Warning: ignored unknown extractor(s): #{result[:unknown].join(', ')}" if result[:unknown].any?
+    puts "#{result[:touched].size} unit(s) written or removed."
+  end
+
   desc 'Extract only Rails/gem framework sources (run when dependencies change)'
   task extract_framework: :environment do
-    require 'woods/extractors/rails_source_extractor'
+    require 'woods/extractor'
 
-    output_dir = ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods'))
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
 
     puts 'Extracting Rails and gem framework sources...'
     puts "Rails version: #{Rails.version}"
     puts
 
-    extractor = Woods::Extractors::RailsSourceExtractor.new
-    units = extractor.extract_all
-
-    # Write output
-    framework_dir = Pathname.new(output_dir).join('rails_source')
-    FileUtils.mkdir_p(framework_dir)
-
-    units.each do |unit|
-      file_name = "#{unit.identifier.gsub('/', '__').gsub('::', '__')}.json"
-      File.write(
-        framework_dir.join(file_name),
-        JSON.pretty_generate(unit.to_h)
-      )
+    # Back-compat alias for `woods:refresh[rails_source]` (#169). The old body
+    # hand-wrote unit JSON with a bare File.write — no AtomicFile, no path
+    # normalization, no _index.json, no manifest counts, no PipelineLock, no
+    # generation bump — so the units it produced were invisible to
+    # generation-keyed readers and went stale forever. Routing through
+    # Extractor#refresh sends framework sources through the same write
+    # pipeline (and the same writer lock) as every other unit type. This runs
+    # regardless of `include_framework_sources` — an explicit invocation is
+    # the escape hatch the knob deliberately leaves open.
+    extractor = Woods::Extractor.new(output_dir: output_dir)
+    result = woods_with_extraction_lock(output_dir) do
+      refreshed = extractor.refresh(:rails_source)
+      extractor.raise_on_publication_failure!
+      refreshed
     end
 
-    puts "Extracted #{units.size} framework source units."
-    puts "Output: #{framework_dir}"
+    puts "Extracted #{result[:touched].size} framework source unit(s)."
+    puts "Output: #{Pathname.new(output_dir).join('rails_source')}"
   end
 
   desc 'Validate extracted index integrity'
   task validate: :environment do
-    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods')))
+    require 'woods/resilience/index_validator'
+
+    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir))
 
     unless output_dir.exist?
       puts "ERROR: Index directory does not exist: #{output_dir}"
       exit 1
     end
 
-    manifest_path = output_dir.join('manifest.json')
+    manifest_path = woods_payload_dir(output_dir).join('manifest.json')
     unless manifest_path.exist?
       puts 'ERROR: Manifest not found. Run extraction first.'
       exit 1
     end
 
-    manifest = JSON.parse(File.read(manifest_path))
+    manifest = JSON.parse(Woods::AtomicFile.read(manifest_path))
 
     puts 'Validating index...'
     puts "  Extracted at: #{manifest['extracted_at']}"
     puts "  Git SHA: #{manifest['git_sha']}"
     puts
 
-    errors = []
-    warnings = []
-
-    # Check each type directory
-    manifest['counts'].each do |type, expected_count|
-      type_dir = output_dir.join(type)
-      unless type_dir.exist?
-        errors << "Missing directory: #{type}"
-        next
-      end
-
-      actual_count = Dir[type_dir.join('*.json')].reject { |f| f.end_with?('_index.json') }.size
-
-      warnings << "#{type}: expected #{expected_count}, found #{actual_count}" if actual_count != expected_count
+    # One implementation of the check (B-128): the class the worktree
+    # integration spec asserts through is the one this task runs.
+    report = Woods::Resilience::IndexValidator.new(index_dir: output_dir.to_s, app_root: Rails.root.to_s).validate
+    errors = report.errors
+    warnings = report.warnings
 
-      # Validate each unit file is valid JSON
-      Dir[type_dir.join('*.json')].each do |file|
-        next if file.end_with?('_index.json')
-
-        begin
-          data = JSON.parse(File.read(file))
-          errors << "#{file}: missing identifier" unless data['identifier']
-          errors << "#{file}: missing source_code" unless data['source_code']
-        rescue JSON::ParserError => e
-          errors << "#{file}: invalid JSON - #{e.message}"
-        end
-      end
-    end
-
-    # Check dependency graph
-    graph_path = output_dir.join('dependency_graph.json')
-    if graph_path.exist?
-      begin
-        JSON.parse(File.read(graph_path))
-      rescue JSON::ParserError
-        errors << 'dependency_graph.json: invalid JSON'
-      end
-    else
-      errors << 'Missing dependency_graph.json'
-    end
-
-    # Report
     if errors.any?
       puts 'ERRORS:'
       errors.each { |e| puts "  ✗ #{e}" }
@@ -238,15 +668,16 @@ namespace :woods do
 
   desc 'Show index statistics'
   task stats: :environment do
-    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods')))
+    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir))
 
     unless output_dir.exist?
       puts 'Index directory does not exist. Run extraction first.'
       exit 1
     end
 
-    manifest_path = output_dir.join('manifest.json')
-    manifest = manifest_path.exist? ? JSON.parse(File.read(manifest_path)) : {}
+    payload_dir = woods_payload_dir(output_dir)
+    manifest_path = payload_dir.join('manifest.json')
+    manifest = manifest_path.exist? ? JSON.parse(Woods::AtomicFile.read(manifest_path)) : {}
 
     puts 'Woods Index Statistics'
     puts '=' * 50
@@ -265,7 +696,7 @@ namespace :woods do
     total_chunks = 0
 
     (manifest['counts'] || {}).each do |type, count|
-      type_dir = output_dir.join(type)
+      type_dir = payload_dir.join(type)
       next unless type_dir.exist?
 
       type_size = Dir[type_dir.join('*.json')].sum { |f| File.size(f) }
@@ -276,7 +707,7 @@ namespace :woods do
       index_path = type_dir.join('_index.json')
       type_chunks = 0
       if index_path.exist?
-        index = JSON.parse(File.read(index_path))
+        index = JSON.parse(Woods::AtomicFile.read(index_path))
         type_chunks = index.sum { |u| u['chunk_count'] || 0 }
         total_chunks += type_chunks
       end
@@ -289,9 +720,9 @@ namespace :woods do
     puts
 
     # Dependency graph stats
-    graph_path = output_dir.join('dependency_graph.json')
+    graph_path = payload_dir.join('dependency_graph.json')
     if graph_path.exist?
-      graph = JSON.parse(File.read(graph_path))
+      graph = JSON.parse(Woods::AtomicFile.read(graph_path))
       stats = graph['stats'] || {}
       puts 'Dependency Graph'
       puts '-' * 50
@@ -305,11 +736,11 @@ namespace :woods do
 
   desc 'Clean extracted index'
   task clean: :environment do
-    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods')))
+    output_dir = Pathname.new(ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir))
 
     if output_dir.exist?
       puts "Removing #{output_dir}..."
-      FileUtils.rm_rf(output_dir)
+      exit 1 unless woods_clean_index(output_dir) == :cleaned
       puts 'Done.'
     else
       puts 'Index directory does not exist.'
@@ -319,36 +750,38 @@ namespace :woods do
   desc 'Clear the brush — remove index (alias for clean)'
   task clear: :clean
 
-  # Internal debugging tool — hidden from `rails -T`
-  task :retrieve, [:query] => :environment do |_t, args|
-    query = args[:query] || raise('Usage: rake woods:retrieve[query]')
-
+  # Resolve the configured retrieval stack and run one ad-hoc query (#178).
+  #
+  # Every backend — embedding provider, vector store, metadata store, graph
+  # store — is resolved through Woods::Builder from Woods.configuration,
+  # the same wiring `woods:embed` writes through
+  # (Woods::Tasks.build_embed_indexer). The old task body hardcoded
+  # Ollama + InMemory + SQLite + Memory, so on any other configured stack it
+  # queried backends the embed run never wrote to and silently returned
+  # nothing.
+  #
+  # In-memory stores start empty in a fresh process; hosts on the :local /
+  # :shared_filesystem presets should query through woods-mcp, which
+  # hydrates them from the dumps on disk.
+  #
+  # @param query [String] natural-language retrieval query
+  # @return [String] human-formatted retrieval output
+  def woods_run_retrieval(query)
     require 'woods'
-    require 'woods/retriever'
-    require 'woods/embedding/provider'
-    require 'woods/storage/vector_store'
-    require 'woods/storage/metadata_store'
-    require 'woods/storage/graph_store'
     require 'woods/formatting/human_adapter'
 
     config = Woods.configuration
+    retriever = Woods::Builder.new(config).build_retriever
+    result = retriever.retrieve(query, budget: config.max_context_tokens)
 
-    provider = Woods::Embedding::Provider::Ollama.new
-    vector_store = Woods::Storage::VectorStore::InMemory.new
-    metadata_store = Woods::Storage::MetadataStore::SQLite.new
-    graph_store = Woods::Storage::GraphStore::Memory.new
-
-    retriever = Woods::Retriever.new(
-      vector_store: vector_store,
-      metadata_store: metadata_store,
-      graph_store: graph_store,
-      embedding_provider: provider
-    )
+    Woods::Formatting::HumanAdapter.new.format(result)
+  end
 
-    result = retriever.retrieve(query, budget: config.max_context_tokens)
+  # Internal debugging tool — hidden from `rails -T`
+  task :retrieve, [:query] => :environment do |_t, args|
+    query = args[:query] || raise('Usage: rake woods:retrieve[query]')
 
-    formatter = Woods::Formatting::HumanAdapter.new
-    puts formatter.format(result)
+    puts woods_run_retrieval(query)
   end
 
   desc 'Embed all extracted units'
@@ -356,9 +789,20 @@ namespace :woods do
     require 'woods'
     require 'woods/tasks'
 
+    # Embedding writes dumps/, checkpoint.json and woods.json under the same
+    # output dir the extraction writers own — and `woods:clean` deletes them —
+    # so it serializes on the same PipelineLock (#170). One lock domain per
+    # index, deliberately coarse: an embed does block an extract it doesn't
+    # byte-conflict with, but the failure the lock prevents is a concurrent
+    # clean/extract silently clobbering a dump mid-promotion, and a second
+    # lock name would reintroduce exactly the unlocked-writer gap.
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
+
     indexer = Woods::Tasks.build_embed_indexer
     puts 'Embedding all extracted units...'
-    Woods::Tasks.print_embed_stats(indexer.index_all, mode: :full)
+    stats = woods_with_extraction_lock(output_dir) { indexer.index_all }
+    Woods::Tasks.print_embed_stats(stats, mode: :full)
+    woods_exit_on_reported_errors(stats[:errors])
   end
 
   desc 'Nest the data — embed all units (alias for embed)'
@@ -369,9 +813,14 @@ namespace :woods do
     require 'woods'
     require 'woods/tasks'
 
+    # Same lock domain as woods:embed — see the comment there (#170).
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
+
     indexer = Woods::Tasks.build_embed_indexer
     puts 'Embedding changed units (incremental)...'
-    Woods::Tasks.print_embed_stats(indexer.index_incremental, mode: :incremental)
+    stats = woods_with_extraction_lock(output_dir) { indexer.index_incremental }
+    Woods::Tasks.print_embed_stats(stats, mode: :incremental)
+    woods_exit_on_reported_errors(stats[:errors])
   end
 
   desc 'Hone the blade — incremental embedding (alias for embed_incremental)'
@@ -452,31 +901,31 @@ namespace :woods do
     FileUtils.mkdir_p(docs_dir)
     renderer = Woods::RubyAnalyzer::MermaidRenderer.new
 
+    # architecture.md embeds the call-graph, dependency-map, and dataflow
+    # renders as sections; emitting them as standalone files too duplicated
+    # ~13k generated lines in the docs tree for no reader benefit.
     File.write(
       File.join(docs_dir, 'architecture.md'),
       renderer.render_architecture(units, graph_data, analysis)
     )
 
-    File.write(
-      File.join(docs_dir, 'call-graph.md'),
-      "# Call Graph\n\n```mermaid\n#{renderer.render_call_graph(units)}\n```\n"
-    )
-
-    File.write(
-      File.join(docs_dir, 'dependency-map.md'),
-      "# Dependency Map\n\n```mermaid\n#{renderer.render_dependency_map(graph_data)}\n```\n"
-    )
-
-    File.write(
-      File.join(docs_dir, 'dataflow.md'),
-      "# Data Flow\n\n```mermaid\n#{renderer.render_dataflow(units)}\n```\n"
-    )
-
     puts "  JSON output: #{json_dir}"
     puts "  Mermaid docs: #{docs_dir}"
     puts 'Self-analysis complete.'
   end
 
+  # Internal developer tool. Unlike self_analyze, this publishes an ordinary
+  # index generation that the packaged Index MCP server can read. It is
+  # intentionally Woods-only and static: it does not boot Rails.
+  task :self_map, [:output_dir] do |_task, args|
+    require 'woods/gem_mapper'
+
+    gem_root = File.expand_path('../..', __dir__)
+    output_dir = args[:output_dir] || ENV.fetch('WOODS_SELF_MAP_OUTPUT', File.join(gem_root, 'tmp', 'woods_self_map'))
+    result = Woods::GemMapper.new(root: gem_root, output_dir: output_dir).map!
+    puts "Woods self-map #{result[:status]} (generation #{result[:generation]}) at #{output_dir}"
+  end
+
   desc 'Generate execution flow document for a Rails entry point'
   task :flow, [:entry_point] => :environment do |_t, args|
     require 'json'
@@ -489,8 +938,8 @@ namespace :woods do
       exit 1
     end
 
-    output_dir = ENV.fetch('WOODS_OUTPUT', Rails.root.join('tmp/woods'))
-    graph_path = File.join(output_dir, 'dependency_graph.json')
+    output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
+    graph_path = File.join(woods_payload_dir(output_dir).to_s, 'dependency_graph.json')
 
     unless File.exist?(graph_path)
       puts "ERROR: Dependency graph not found at #{graph_path}"
@@ -498,11 +947,11 @@ namespace :woods do
       exit 1
     end
 
-    graph_data = JSON.parse(File.read(graph_path))
+    graph_data = JSON.parse(Woods::AtomicFile.read(graph_path))
     graph = Woods::DependencyGraph.from_h(graph_data)
 
     max_depth = ENV.fetch('MAX_DEPTH', 5).to_i
-    assembler = Woods::FlowAssembler.new(graph: graph, extracted_dir: output_dir)
+    assembler = Woods::FlowAssembler.new(graph: graph, extracted_dir: woods_payload_dir(output_dir).to_s)
     flow = assembler.assemble(entry_point, max_depth: max_depth)
 
     format = ENV.fetch('FORMAT', 'markdown').downcase
@@ -571,6 +1020,7 @@ namespace :woods do
       puts "  Errors:      #{stats[:errors].size}"
       stats[:errors].first(5).each { |e| puts "    - #{e}" }
       puts "    ... and #{stats[:errors].size - 5} more" if stats[:errors].size > 5
+      woods_exit_on_reported_errors(stats[:errors])
     end
   end
 
@@ -640,7 +1090,11 @@ namespace :woods do
       # post-merge pipelines (a dead token would otherwise stay green forever).
       # Exception: budget exhaustion *with* partial progress is the expected
       # cold-start shape; it converges on the next run.
-      budget_only = stats[:errors].all? { |e| e.include?('daily budget exhausted') }
+      # Matched on the message because errors reach here as strings, not
+      # exceptions. `budget exhausted` is the stable part of
+      # BudgetExhaustedError's message — spec/unblocked/rate_limiter_spec.rb
+      # pins it so this branch cannot be silently broken by a rewording.
+      budget_only = stats[:errors].all? { |e| e.include?('budget exhausted') }
       unless budget_only && stats[:synced].positive?
         puts
         puts 'Sync completed with errors — failing so CI surfaces it.'