woods 1.6.1 → 2.0.0.beta2

data/lib/woods/mcp/index_reader.rb

@@ -7,6 +7,8 @@ require 'json'
 require 'pathname'
 require 'set'
 
+require_relative '../generation'
+
 module Woods
   module MCP
     # Reads extraction output from disk for the MCP server.
@@ -29,7 +31,7 @@ module Woods
         engines view_templates migrations action_cable_channels
         scheduled_jobs rake_tasks state_machines events decorators
         database_views caching factories test_mappings rails_source
-        poros libs
+        poros libs packages ruby_classes ruby_modules ruby_methods ruby_files
       ].freeze
 
       # Singular type name for each directory (used in search filtering).
@@ -42,15 +44,200 @@ module Woods
       MAX_UNIT_CACHE = 50
 
       # @param index_dir [String] Path to extraction output directory
+      # @param auto_refresh [Boolean] re-read the index when its published
+      #   generation moves. On by default; specs that assert caching behaviour
+      #   turn it off.
       # @raise [ArgumentError] if directory doesn't exist or has no manifest.json
-      def initialize(index_dir)
+      def initialize(index_dir, auto_refresh: true)
         @index_dir = Pathname.new(index_dir)
         raise ArgumentError, "Index directory does not exist: #{index_dir}" unless @index_dir.directory?
-        raise ArgumentError, "No manifest.json found in: #{index_dir}" unless @index_dir.join('manifest.json').file?
+        raise ArgumentError, "No manifest.json found in: #{index_dir}" unless manifest_present?
 
         @unit_cache = {}
+        @unit_cache_signatures = {}
         @unit_cache_order = []
         @identifier_map = nil
+        @auto_refresh = auto_refresh
+        @pin_depth = 0
+        @pin_owners = Hash.new(0)
+        @exclusive_waiters = 0
+        @exclusive_owner = nil
+        # Pins waiting for the held pins to drain so a moved generation can be
+        # adopted (MCP-5). Gates NEW pin entries exactly as @exclusive_waiters
+        # does, so sustained overlapping traffic cannot pin the reader on a
+        # retired generation indefinitely.
+        @refresh_waiters = 0
+        @freshness_mutex = Mutex.new
+        @freshness_condition = ConditionVariable.new
+        # Separate from @freshness_mutex: the LRU bookkeeping must not
+        # serialize behind a generation check, and a reader that took the
+        # freshness lock must be able to touch the caches without deadlocking.
+        @cache_mutex = Mutex.new
+        @generation = Woods::Generation.new(output_dir: @index_dir)
+        @loaded_generation = nil
+        @loaded_token = nil
+        @generation_signature = nil
+        @payload_retention_pin = nil
+        # The directory the loaded generation's payload lives in. nil until a
+        # generation naming a payload is loaded; {#payload_dir} then falls back
+        # to the index root, which is every flat/pre-pointer index.
+        @payload_dir = nil
+      end
+
+      # The generation this reader's caches were populated from.
+      #
+      # @return [Integer, nil] nil until something has been read
+      attr_reader :loaded_generation
+
+      # Drop caches if the index has been rewritten since they were populated.
+      #
+      # This is what makes the MCP `reload` tool an optimization rather than a
+      # correctness requirement. A long-lived server used to hold whatever it
+      # read at boot until someone thought to call `reload` — so an agent
+      # working alongside a running extraction got answers describing the tree
+      # as of the last time the server happened to start.
+      #
+      # Called at the top of every public read. The cost is one `File.stat` of
+      # a ~100-byte file; the file is only parsed when its mtime or size moved,
+      # and the caches are only dropped when the generation number actually
+      # advanced. An index with no generation file (written before generations
+      # existed, or by a third party) never refreshes — same behaviour as
+      # before.
+      #
+      # The check-and-reload is a mutex-guarded critical section. The stdio
+      # transport serializes tool calls, but `woods-mcp-http` runs handlers on
+      # its Rack server's request threads — and unguarded, two concurrent reads
+      # race the signature bookkeeping: duplicate `reload!`s, or one request's
+      # freshly populated caches dropped mid-sequence by the other's refresh.
+      #
+      # @return [Integer, nil] the generation now loaded, or nil when the
+      #   caches were already current
+      def ensure_fresh!
+        return nil unless @auto_refresh
+
+        thread = Thread.current
+        @freshness_mutex.synchronize do
+          wait_for_generation_access(thread)
+          refresh_if_stale
+        end
+      end
+
+      # Suppress cache invalidation for the duration of a block.
+      #
+      # {#ensure_fresh!} runs per read, which bounds staleness but does not
+      # make a *sequence* of reads consistent: a caller that reads the manifest
+      # and then a unit can straddle two generations if a write lands between
+      # them. Both halves describe a real state of the tree, but not the same
+      # one.
+      #
+      # Wrapping a multi-read operation checks freshness once, up front, and
+      # then holds it — so nothing already cached is dropped and re-read at a
+      # newer generation partway through.
+      #
+      # **What this does not do on a flat index:** an artifact that has never
+      # been read is loaded from whatever is on disk when the block reaches it.
+      # Guaranteeing more would mean materializing the whole index on entry,
+      # which is what {#warmup!} costs, per request. When the loaded generation
+      # names an immutable payload directory (its {Woods::Generation} carries a
+      # +payload+ pointer), that gap is closed for free: every read in the
+      # block resolves through {#payload_dir}, which is fixed at pin entry and
+      # names one immutable snapshot, so a never-read artifact still comes from
+      # the pinned generation even as a newer payload is published. A flat
+      # index carries no pointer and keeps the old behaviour — see
+      # docs/WATCH_DAEMON.md.
+      #
+      # Pins are *refcounted*, not a boolean. Under a threaded transport two
+      # requests can hold pins at once, and with a boolean the first to finish
+      # unpinned the reader while the second still relied on it — its remaining
+      # reads could then be invalidated mid-sequence, the exact tear this
+      # method exists to prevent. Freshness is checked when the depth goes
+      # 0 → 1; nested pins and the reads of a pin already held ride the
+      # generation it entered on, and invalidation resumes when the last pin
+      # releases.
+      #
+      # **Bounded staleness under sustained overlap (MCP-5).** Refresh used to
+      # be attempted ONLY at depth 0 → 1, so on a threaded transport with
+      # enough concurrent traffic — several agents against one shared
+      # `woods-mcp-http`, the deployment stateless mode exists for — the depth
+      # never reached 0 and a retired generation was served indefinitely, with
+      # nothing in the response saying so. A pin that arrives after the
+      # generation signature moved while pins are held now registers as a
+      # refresh waiter: like {#with_exclusive_reload}'s waiters it gates NEW
+      # pin entries, waits for the held pins to drain, refreshes, and admits
+      # the queue. Nested pins and existing holders are exempt, so the drain
+      # can always complete. The cost is one drain per published generation.
+      #
+      # The Index MCP server applies this around reader-backed handlers. Direct
+      # IndexReader callers remain responsible for pinning any multi-read
+      # operation that must not refresh between accessors.
+      #
+      # @example
+      #   reader.with_pinned_generation { [reader.manifest, reader.find_unit("Post")] }
+      #
+      # @yield the block to run against a single generation
+      # @return [Object] the block's value
+      def with_pinned_generation
+        thread = Thread.current
+        exclusive_owner = false
+        @freshness_mutex.synchronize do
+          if @exclusive_owner.equal?(thread)
+            exclusive_owner = true
+          else
+            wait_for_generation_access(thread)
+            drain_for_pending_refresh(thread)
+            refresh_if_stale if @auto_refresh && @pin_depth.zero?
+            acquire_payload_retention_pin if @pin_depth.zero?
+            @pin_depth += 1
+            @pin_owners[thread] += 1
+          end
+        end
+
+        begin
+          yield
+        ensure
+          release_generation_pin(thread) unless exclusive_owner
+        end
+      end
+
+      # Reload all cached index state while excluding pinned readers.
+      #
+      # Existing pins may finish and nest normally. Once an exclusive reload is
+      # waiting, new pins wait until the reload block has assembled its result.
+      #
+      # @yieldparam manifest [Hash] the manifest loaded after caches are cleared
+      # @return [Object] the block's value
+      # @raise [ThreadError] when upgrading a pin or nesting an exclusive reload
+      def with_exclusive_reload
+        with_exclusive_generation do
+          reload!
+          yield manifest
+        end
+      end
+
+      # Hold the exclusive generation lock WITHOUT refreshing anything.
+      #
+      # This is the swap lock the reload transaction (M7) acquires after its
+      # candidate stores are built off-side. Inside the block the caller
+      # rechecks the generation and then swaps atomically; the reader's own
+      # caches are reloaded by the caller ONLY after the recheck passes, so a
+      # failed recheck leaves the previous fully aligned generation untouched.
+      # {#with_exclusive_reload} is this lock plus the unconditional reload.
+      #
+      # Waiters follow the same rules as {#with_exclusive_reload}: in-flight
+      # pins finish before the block runs, and new pins wait while it is held.
+      #
+      # @yield no arguments
+      # @return [Object] the block's value
+      # @raise [ThreadError] when upgrading a pin or nesting an exclusive lock
+      def with_exclusive_generation
+        thread = Thread.current
+        acquire_exclusive_generation(thread)
+
+        begin
+          yield
+        ensure
+          release_exclusive_generation(thread)
+        end
       end
 
       # Pre-populate cached state so the first MCP tool call doesn't pay
@@ -66,27 +253,18 @@ module Woods
       #
       # @return [Hash] Per-step outcome: `{step => true | Exception}`
       def warmup!
-        steps = {
-          manifest: -> { manifest },
-          summary: -> { summary },
-          dependency_graph: -> { dependency_graph },
-          graph_analysis: -> { graph_analysis },
-          identifier_map: -> { identifier_map }
-        }
-        steps.each_with_object({}) do |(step, runner), result|
-          runner.call
-          result[step] = true
-        rescue StandardError => e
-          result[step] = e
-        end
+        with_pinned_generation { warmup_steps }
       end
 
       # Clear all cached state so the next access re-reads from disk.
       #
       # @return [void]
       def reload!
-        @unit_cache = {}
-        @unit_cache_order = []
+        @cache_mutex.synchronize do
+          @unit_cache = {}
+          @unit_cache_signatures = {}
+          @unit_cache_order = []
+        end
         @identifier_map = nil
         @index_cache = {}
         @manifest = nil
@@ -95,10 +273,31 @@ module Woods
         @graph_analysis = nil
         @raw_graph_data = nil
         @normalized_graph_edges = nil
+        @graph_node_types = nil
+        remove_instance_variable(:@multi_database_graph) if defined?(@multi_database_graph)
+      end
+
+      # The directory the current read resolves payload artifacts against —
+      # manifest, summary, graph, per-type indexes, unit files and precomputed
+      # flows. It is the immutable payload directory the loaded generation
+      # names, or the index root for a flat/pre-pointer index.
+      #
+      # Fixed for the life of a pin (only {#refresh_if_stale} moves it, and
+      # pins suppress refresh), so every artifact of a pinned read resolves
+      # through one generation. Public because callers that read payload files
+      # without going through this reader — the flow assembler, the
+      # precomputed-flow loader — have to resolve to the same generation this
+      # reader is serving, not to whatever is published when they look.
+      #
+      # @return [Pathname]
+      def payload_dir
+        ensure_fresh!
+        current_payload_dir
       end
 
       # @return [Hash] Parsed manifest.json
       def manifest
+        ensure_fresh!
         @manifest ||= parse_json('manifest.json')
       end
 
@@ -115,22 +314,22 @@ module Woods
 
       # @return [String, nil] SUMMARY.md content, or nil if not present
       def summary
+        ensure_fresh!
         @summary ||= begin
-          path = @index_dir.join('SUMMARY.md')
-          path.file? ? path.read : nil
+          path = current_payload_dir.join('SUMMARY.md')
+          path.file? ? read_utf8(path) : nil
         end
       end
 
       # @return [Woods::DependencyGraph] Graph loaded from disk
       def dependency_graph
-        @dependency_graph ||= begin
-          data = parse_json('dependency_graph.json')
-          Woods::DependencyGraph.from_h(data)
-        end
+        ensure_fresh!
+        @dependency_graph ||= Woods::DependencyGraph.from_h(raw_graph_data)
       end
 
       # @return [Hash] Parsed graph_analysis.json
       def graph_analysis
+        ensure_fresh!
         @graph_analysis ||= parse_json('graph_analysis.json')
       end
 
@@ -139,6 +338,7 @@ module Woods
       # @param identifier [String] Unit identifier (e.g. "Post", "Api::V1::HealthController")
       # @return [Hash, nil] Full unit data or nil if not found
       def find_unit(identifier)
+        ensure_fresh!
         location = identifier_map[identifier]
         return nil unless location
 
@@ -150,6 +350,7 @@ module Woods
       # @param type [String, nil] Singular type name (e.g. "model", "controller")
       # @return [Array<Hash>] Index entries for matching units
       def list_units(type: nil)
+        ensure_fresh!
         dirs = if type
                  dir = TYPE_TO_DIR[type]
                  dir ? [dir] : []
@@ -164,6 +365,14 @@ module Woods
       # Override with WOODS_SEARCH_MAX_SCAN env var.
       DEFAULT_SEARCH_MAX_SCAN = 500
 
+      # Wall-clock budget for a single match against the user-supplied search
+      # pattern (audit P5). The query is compiled as a raw Ruby regex; a
+      # pattern with catastrophic backtracking would otherwise stall the
+      # dispatch thread indefinitely. Ruby 3.2+ can enforce a per-match limit
+      # at the engine level (Regexp.new(timeout:)); on older Rubies the
+      # pattern compiles without one and this constant is unused.
+      SEARCH_PATTERN_TIMEOUT = 1.0
+
       # Search units by case-insensitive pattern.
       #
       # Phase 1: match identifiers from index files (cheap).
@@ -194,6 +403,20 @@ module Woods
       # @return [Hash] { results: Array<Hash>, note: String|nil, partial: Boolean }
       # @raise [ArgumentError] when all of query, exact_prefix, and exact_suffix are blank
       def search(query = nil, types: nil, fields: %w[identifier], limit: 20, exact_prefix: nil, exact_suffix: nil)
+        # Pinned, not merely checked-once. This walks the identifier map and
+        # then loads units for the hits; each nested `find_unit` re-checks
+        # freshness, so a publish landing mid-walk rebuilt the caches while the
+        # result list still held identifiers from the previous generation — one
+        # response describing two indexes.
+        with_pinned_generation do
+          search_within_pin(query, types: types, fields: fields, limit: limit,
+                                   exact_prefix: exact_prefix, exact_suffix: exact_suffix)
+        end
+      end
+
+      # @api private
+      def search_within_pin(query = nil, types: nil, fields: %w[identifier], limit: 20,
+                            exact_prefix: nil, exact_suffix: nil)
         prefix = exact_prefix.blank? ? nil : exact_prefix.downcase
         suffix = exact_suffix.blank? ? nil : exact_suffix.downcase
         if query.blank? && !prefix && !suffix
@@ -212,85 +435,92 @@ module Woods
         phase2_scanned = 0
         partial = false
 
-        dirs = if types
-                 types.filter_map { |t| TYPE_TO_DIR[t] }
-               else
-                 TYPE_DIRS
-               end
-
-        # Phase 2 candidates are collected per-dir and then scanned in
-        # round-robin across dirs. Exhausting the per-run scan cap linearly
-        # down TYPE_DIRS order would starve later types (`concerns` at pos
-        # 13, `test_mappings` at pos 31) on any codebase where the earlier
-        # dirs together exceed max_scan entries. Interleaving guarantees
-        # every type contributes to the scanned set.
-        phase2_queues = {}
-
-        dirs.each do |dir|
-          type_name = DIR_TO_TYPE[dir]
-          entries = read_index(dir)
-
-          # Broad-match detection: warn when pattern matches >50% of dir entries
-          if entries.size > 1
-            matching_count = entries.count do |e|
-              identifier_passes_filters?(e['identifier'], pattern, prefix, suffix)
-            end
-            if matching_count > entries.size / 2.0
-              notes << "broad pattern matched #{matching_count}/#{entries.size} entries in #{dir}"
+        begin
+          dirs = if types
+                   types.filter_map { |t| TYPE_TO_DIR[t] }
+                 else
+                   TYPE_DIRS
+                 end
+
+          # Phase 2 candidates are collected per-dir and then scanned in
+          # round-robin across dirs. Exhausting the per-run scan cap linearly
+          # down TYPE_DIRS order would starve later types (`concerns` at pos
+          # 13, `test_mappings` at pos 31) on any codebase where the earlier
+          # dirs together exceed max_scan entries. Interleaving guarantees
+          # every type contributes to the scanned set.
+          phase2_queues = {}
+
+          dirs.each do |dir|
+            type_name = DIR_TO_TYPE[dir]
+            entries = read_index(dir)
+
+            # Broad-match detection: warn when pattern matches >50% of dir entries
+            if entries.size > 1
+              matching_count = entries.count do |e|
+                identifier_passes_filters?(e['identifier'], pattern, prefix, suffix)
+              end
+              if matching_count > entries.size / 2.0
+                notes << "broad pattern matched #{matching_count}/#{entries.size} entries in #{dir}"
+              end
             end
-          end
 
-          entries.each do |entry|
-            id = entry['identifier']
-            next unless identifier_passes_prefix_suffix?(id, prefix, suffix)
+            entries.each do |entry|
+              id = entry['identifier']
+              next unless identifier_passes_prefix_suffix?(id, prefix, suffix)
 
-            # Phase 1: identifier matching (still in-order per dir)
-            if fields.include?('identifier') && pattern.match?(id)
-              next if results.size >= limit
+              # Phase 1: identifier matching (still in-order per dir)
+              if fields.include?('identifier') && pattern.match?(id)
+                next if results.size >= limit
 
-              results << { identifier: id, type: type_name, match_field: 'identifier' }
-              next
-            end
+                results << { identifier: id, type: type_name, match_field: 'identifier' }
+                next
+              end
 
-            # Phase 2 is only reached when the caller opted into deeper fields.
-            next unless fields.include?('metadata') || fields.include?('source_code')
+              # Phase 2 is only reached when the caller opted into deeper fields.
+              next unless fields.include?('metadata') || fields.include?('source_code')
 
-            (phase2_queues[dir] ||= []) << [type_name, id]
+              (phase2_queues[dir] ||= []) << [type_name, id]
+            end
           end
-        end
 
-        if results.size < limit && phase2_queues.any?
-          queues = phase2_queues.values.map(&:dup)
-          catch(:phase2_done) do
-            loop do
-              progressed = false
-              queues.each do |queue|
-                next if queue.empty?
+          if results.size < limit && phase2_queues.any?
+            queues = phase2_queues.values.map(&:dup)
+            catch(:phase2_done) do
+              loop do
+                progressed = false
+                queues.each do |queue|
+                  next if queue.empty?
 
-                throw :phase2_done if results.size >= limit
+                  throw :phase2_done if results.size >= limit
 
-                if phase2_scanned >= max_scan
-                  partial = true
-                  throw :phase2_done
-                end
+                  if phase2_scanned >= max_scan
+                    partial = true
+                    throw :phase2_done
+                  end
 
-                type_name, id = queue.shift
-                progressed = true
+                  type_name, id = queue.shift
+                  progressed = true
 
-                unit = find_unit(id)
-                next unless unit
+                  unit = find_unit(id)
+                  next unless unit
 
-                phase2_scanned += 1
+                  phase2_scanned += 1
 
-                if fields.include?('source_code') && unit['source_code'] && pattern.match?(unit['source_code'])
-                  results << { identifier: id, type: type_name, match_field: 'source_code' }
-                elsif fields.include?('metadata') && unit['metadata'] && pattern.match?(unit['metadata'].to_json)
-                  results << { identifier: id, type: type_name, match_field: 'metadata' }
+                  if fields.include?('source_code') && unit['source_code'] && pattern.match?(unit['source_code'])
+                    results << { identifier: id, type: type_name, match_field: 'source_code' }
+                  elsif fields.include?('metadata') && unit['metadata'] && pattern.match?(unit['metadata'].to_json)
+                    results << { identifier: id, type: type_name, match_field: 'metadata' }
+                  end
                 end
+                break unless progressed
               end
-              break unless progressed
             end
           end
+        rescue StandardError => e
+          raise unless regexp_timeout_error?(e)
+
+          notes << "search aborted: the pattern exceeded the #{SEARCH_PATTERN_TIMEOUT}s per-match limit"
+          partial = true
         end
 
         response = { results: results.first(limit) }
@@ -330,6 +560,11 @@ module Woods
       # @param limit [Integer] Maximum results to return
       # @return [Array<Hash>] Matching rails_source unit summaries
       def framework_sources(keyword, limit: 20)
+        with_pinned_generation { framework_sources_within_pin(keyword, limit: limit) }
+      end
+
+      # @api private
+      def framework_sources_within_pin(keyword, limit: 20)
         # Multi-word keywords ("ActiveRecord callbacks") are split on
         # whitespace and ANDed. Single-word queries behave as before.
         tokens = keyword.to_s.strip.split(/\s+/)
@@ -375,6 +610,11 @@ module Woods
       # @param types [Array<String>, nil] Filter to these singular type names
       # @return [Array<Hash>] Units sorted by last_modified descending
       def recent_changes(limit: 10, types: nil)
+        with_pinned_generation { recent_changes_within_pin(limit: limit, types: types) }
+      end
+
+      # @api private
+      def recent_changes_within_pin(limit: 10, types: nil)
         dirs = if types
                  types.filter_map { |t| TYPE_TO_DIR[t] }
                else
@@ -410,24 +650,362 @@ module Woods
       end
 
       # @return [Hash] Raw dependency graph data from JSON
+      #
+      # The single parse point for dependency_graph.json per generation
+      # (audit P6): {#dependency_graph} builds the typed graph from this
+      # same hash (DependencyGraph.from_h does not mutate its input), so a
+      # generation that touches both accessors holds one parsed copy.
+      #
+      # The parsed structure is deep-frozen before the memo publishes it.
+      # It is shared state for the whole generation — the typed graph, the
+      # memoized edge normalization, traversal, and the server graph tools
+      # all read from it — so a caller mutation of a nested edge or node
+      # would reach the internal copy and corrupt every later read. Frozen
+      # at parse time, not lazily, so even the first accessor's returned
+      # value is immutable.
       def raw_graph_data
-        @raw_graph_data ||= parse_json('dependency_graph.json')
+        ensure_fresh!
+        @raw_graph_data ||= deep_freeze_json(parse_json('dependency_graph.json'))
       end
 
       private
 
+      def wait_for_generation_access(thread)
+        @freshness_condition.wait(@freshness_mutex) while generation_access_blocked?(thread)
+      end
+
+      def generation_access_blocked?(thread)
+        return false if @exclusive_owner.equal?(thread) || @pin_owners.key?(thread)
+
+        !@exclusive_owner.nil? || @exclusive_waiters.positive? || @refresh_waiters.positive?
+      end
+
+      # Wait for held pins to drain when the generation moved while they were
+      # held, so the pin about to be taken adopts the new generation instead
+      # of riding the retired one (MCP-5).
+      #
+      # Exemptions keep the drain from deadlocking against itself:
+      #   - a thread that already holds a pin (a NESTED pin) never waits — the
+      #     drain it would wait for can only happen after that thread returns,
+      #     and its nested read must stay on its owner's generation anyway;
+      #   - the reads of an existing pin go through {#ensure_fresh!}, not
+      #     here, and stay pinned as before.
+      #
+      # Registering as a waiter blocks other new pins ({#generation_access_blocked?})
+      # for the same reason an exclusive reload does: without it, a steady
+      # arrival rate keeps the depth off zero forever. The waiter itself holds
+      # no pin, so it never blocks the drain it is waiting on.
+      #
+      # Callers hold `@freshness_mutex`.
+      #
+      # @param thread [Thread] the thread taking the pin
+      # @return [void]
+      def drain_for_pending_refresh(thread)
+        return unless @auto_refresh
+        return if @pin_depth.zero?
+        return if @pin_owners.key?(thread)
+        return unless generation_moved?
+
+        @refresh_waiters += 1
+        begin
+          @freshness_condition.wait(@freshness_mutex) until @pin_depth.zero? && @exclusive_owner.nil?
+        ensure
+          @refresh_waiters -= 1
+          @freshness_condition.broadcast
+        end
+      end
+
+      # Has the published generation moved since the caches were populated?
+      # Signature-only (one `File.stat`) — the same cheap probe
+      # {#refresh_if_stale} guards its parse with, without consuming it.
+      #
+      # @return [Boolean]
+      def generation_moved?
+        signature = generation_signature
+        !signature.nil? && signature != @generation_signature
+      end
+
+      def release_generation_pin(thread)
+        @freshness_mutex.synchronize do
+          @pin_depth -= 1
+          @pin_owners[thread] -= 1
+          @pin_owners.delete(thread) if @pin_owners[thread].zero?
+          if @pin_depth.zero?
+            release_payload_retention_pin
+            @freshness_condition.broadcast
+          end
+        end
+      end
+
+      # Keep the immutable payload backing a pinned read on disk across
+      # processes. The in-memory pin counters above coordinate threads that
+      # share this reader, but a writer in another process cannot see them and
+      # retention could otherwise remove the loaded generation before a later
+      # artifact in the same request was opened.
+      #
+      # The reader opens the generation's existing manifest read-only and
+      # takes a shared advisory lock. PayloadStore takes an exclusive,
+      # non-blocking lock on the same inode before pruning. Re-reading the
+      # generation after acquiring closes both races around the lock: if a
+      # publish landed first we adopt it and retry, and if pruning landed first
+      # the manifest open (or the directory check) fails and we retry from the
+      # new published pointer.
+      #
+      # The retry is bounded (CORE-4). A payload directory that exists without
+      # its `manifest.json` — only reachable by tampering, since every
+      # published payload carries one — leaves the pointer unchanged, so every
+      # iteration would take the identical path with no sleep and no cap: a
+      # 100%-CPU hang of the request thread instead of a degraded read. A
+      # second consecutive ENOENT against the SAME expected directory means
+      # retrying cannot help, so the read proceeds unpinned, exactly as it
+      # does for a flat index.
+      #
+      # @return [void]
+      def acquire_payload_retention_pin
+        missing_manifest_dir = nil
+        saw_missing_manifest = false
+        loop do
+          expected_dir = current_payload_dir
+          return if expected_dir == @index_dir
+
+          # Intentionally outlives this method; the last overlapping pin
+          # closes it in #release_payload_retention_pin.
+          file = File.open( # rubocop:disable Style/FileOpen
+            expected_dir.join('manifest.json').to_s, File::RDONLY
+          )
+          file.flock(File::LOCK_SH)
+          marker = @generation.current
+          resolved = resolve_payload_dir(marker)
+          marker_still_matches = marker.number.zero? || (same_generation?(marker) && resolved == expected_dir)
+          if marker_still_matches && expected_dir.directory?
+            @payload_retention_pin = file
+            return
+          end
+
+          file.flock(File::LOCK_UN)
+          file.close
+          load_generation(marker) unless marker.number.zero?
+        rescue Errno::ENOENT
+          file&.close unless file&.closed?
+          return if saw_missing_manifest && missing_manifest_dir == expected_dir
+
+          saw_missing_manifest = true
+          missing_manifest_dir = expected_dir
+          marker = @generation.current
+          load_generation(marker) unless marker.number.zero?
+        end
+      end
+
+      # @return [void]
+      def release_payload_retention_pin
+        @payload_retention_pin&.flock(File::LOCK_UN)
+        @payload_retention_pin&.close
+        @payload_retention_pin = nil
+      rescue IOError, SystemCallError
+        @payload_retention_pin = nil
+      end
+
+      def acquire_exclusive_generation(thread)
+        @freshness_mutex.synchronize do
+          raise ThreadError, 'Cannot start an exclusive reload from a pinned generation' if @pin_owners.key?(thread)
+          raise ThreadError, 'This thread already owns an exclusive reload' if @exclusive_owner.equal?(thread)
+
+          @exclusive_waiters += 1
+          begin
+            @freshness_condition.wait(@freshness_mutex) until @pin_depth.zero? && @exclusive_owner.nil?
+            @exclusive_owner = thread
+          ensure
+            @exclusive_waiters -= 1
+            @freshness_condition.broadcast
+          end
+        end
+      end
+
+      def release_exclusive_generation(thread)
+        @freshness_mutex.synchronize do
+          @exclusive_owner = nil if @exclusive_owner.equal?(thread)
+          @freshness_condition.broadcast
+        end
+      end
+
+      # The body of {#ensure_fresh!}. Callers must hold `@freshness_mutex`; it
+      # is split out so {#with_pinned_generation} can run it inside the same
+      # critical section that increments the pin depth.
+      #
+      # @return [Integer, nil] the generation now loaded, or nil when the
+      #   caches were already current
+      def refresh_if_stale
+        return nil if @pin_depth.positive?
+
+        signature = generation_signature
+        return nil if signature.nil? || signature == @generation_signature
+
+        @generation_signature = signature
+        marker = @generation.current
+        return nil if marker.number.zero? || same_generation?(marker)
+
+        load_generation(marker)
+      end
+
+      # Adopt one already-read marker as the reader's loaded generation.
+      # Callers hold @freshness_mutex.
+      #
+      # @param marker [Woods::Generation::Marker]
+      # @return [void]
+      def load_generation(marker)
+        reload!
+        @payload_dir = resolve_payload_dir(marker)
+        @loaded_token = marker.token
+        @loaded_generation = marker.number
+      end
+
+      # Is there an index here at all?
+      #
+      # A flat index answers with the manifest at the root. An index that
+      # publishes per-generation payloads has no manifest there — every payload
+      # artifact lives in the directory the published generation names — so the
+      # pointer is followed before concluding the directory is not an index.
+      #
+      # @return [Boolean]
+      def manifest_present?
+        return true if @index_dir.join('manifest.json').file?
+
+        marker = Woods::Generation.new(output_dir: @index_dir).current
+        resolve_payload_dir(marker).join('manifest.json').file?
+      end
+
+      # The loaded generation's payload directory, without a freshness check.
+      #
+      # Internal reads call this rather than {#payload_dir}: they are already
+      # inside a public read that checked freshness once, and re-checking per
+      # artifact would take the freshness mutex and stat generation.json for
+      # every unit file a request touches.
+      #
+      # @return [Pathname]
+      def current_payload_dir
+        @payload_dir || @index_dir
+      end
+
+      # Resolve a generation's payload pointer to an on-disk directory.
+      #
+      # Delegates to {Woods::Generation#payload_dir} so every reader of a
+      # payload artifact — the exporters, the validator, the daemon and this
+      # reader — follows the pointer by exactly the same rules.
+      #
+      # @param marker [Woods::Generation::Marker]
+      # @return [Pathname]
+      def resolve_payload_dir(marker)
+        Woods::Generation.new(output_dir: @index_dir).payload_dir(marker)
+      end
+
+      # Compare the *token*, not the number.
+      #
+      # `bump!` is a read-modify-write, so two writers that overlap can both
+      # publish the same number — which the design permits, since a manual rake
+      # run proceeds after waiting for the lock. A reader already loaded at N+1
+      # would then see `published == loaded`, conclude it was current, and hold
+      # caches describing the *other* writer's N+1 indefinitely: a stale index
+      # that believes it is fresh, which is the one failure generations exist to
+      # prevent.
+      #
+      # The token is a fresh random value per publish, so it distinguishes two
+      # collapsed bumps that the counter cannot. Falls back to the number for
+      # generation files written before tokens existed.
+      def same_generation?(marker)
+        return @loaded_generation == marker.number if marker.token.nil?
+
+        @loaded_token == marker.token
+      end
+
+      # Touch every lazy accessor, recording rather than raising per-step
+      # failures so a missing optional artefact (graph_analysis.json on an index
+      # written before it existed) never blocks the rest.
+      #
+      # @return [Hash] `{step => true | Exception}`
+      def warmup_steps
+        steps = {
+          manifest: -> { manifest },
+          summary: -> { summary },
+          dependency_graph: -> { dependency_graph },
+          graph_analysis: -> { graph_analysis },
+          identifier_map: -> { identifier_map }
+        }
+        steps.each_with_object({}) do |(step, runner), result|
+          runner.call
+          result[step] = true
+        rescue StandardError => e
+          result[step] = e
+        end
+      end
+
       # Compile a case-insensitive regex from a query string.
       #
       # Treats the query as a raw Ruby regex pattern. Falls back to an escaped
       # literal match (with a :note field added by callers) when the pattern is
       # invalid.
       #
+      # The compiled pattern carries the {SEARCH_PATTERN_TIMEOUT} per-match
+      # wall-clock limit on Ruby 3.2+: a pattern with catastrophic
+      # backtracking raises Regexp::TimeoutError on its first overrun instead
+      # of stalling the dispatch thread indefinitely, which {#search_within_pin}
+      # converts into a partial response.
+      #
       # @param query [String] Raw regex pattern
       # @return [Regexp] Compiled case-insensitive pattern
       def compile_search_pattern(query)
-        Regexp.new(query, Regexp::IGNORECASE)
+        compile_case_insensitive_pattern(query)
       rescue RegexpError
-        Regexp.new(Regexp.escape(query), Regexp::IGNORECASE)
+        compile_case_insensitive_pattern(Regexp.escape(query))
+      end
+
+      # @param pattern [String] Regex source (already escaped when falling back)
+      # @return [Regexp]
+      def compile_case_insensitive_pattern(pattern)
+        if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2')
+          Regexp.new(pattern, Regexp::IGNORECASE, timeout: SEARCH_PATTERN_TIMEOUT)
+        else
+          Regexp.new(pattern, Regexp::IGNORECASE)
+        end
+      end
+
+      # Is this the engine-level per-match timeout, Regexp::TimeoutError?
+      #
+      # The constant does not exist before Ruby 3.2, and Ruby resolves
+      # rescue-class expressions lazily, when an exception occurs — naming
+      # the constant in a rescue clause turned any *other* in-block failure
+      # into `NameError: uninitialized constant Regexp::TimeoutError` on
+      # Ruby 3.0/3.1, masking the original error. Rescue StandardError and
+      # recognize the timeout through a defined? check instead; everything
+      # else is re-raised unchanged.
+      #
+      # @param error [StandardError]
+      # @return [Boolean]
+      def regexp_timeout_error?(error)
+        defined?(Regexp::TimeoutError) && error.is_a?(Regexp::TimeoutError)
+      end
+
+      # A cheap stand-in for "has the generation file been rewritten?", so the
+      # common case costs one `File.stat` of a ~100-byte file instead of a parse.
+      #
+      # The inode is load-bearing, not belt-and-braces. `[mtime, size]` alone
+      # misses a bump whenever two writes land in the same mtime tick with the
+      # same payload length — and equal length is the daemon's *steady state*:
+      # reason `"incremental"` every cycle, fixed-width number and timestamp.
+      # Coarse mtime granularity is not exotic either; it includes the
+      # volume-mounted Docker deployment the Index Server is documented for. The
+      # reader would then serve a stale index indefinitely, which is the exact
+      # silent staleness generations exist to end.
+      #
+      # {AtomicFile} renames a fresh Tempfile over the target on every write, so
+      # a new inode is guaranteed per publish and this costs nothing extra.
+      #
+      # @return [Array, nil] opaque signature, or nil when there is no
+      #   generation file
+      def generation_signature
+        stat = File.stat(@generation.path)
+        [stat.mtime.to_f, stat.size, stat.ino]
+      rescue SystemCallError
+        nil
       end
 
       # Case-insensitive literal prefix/suffix check on an identifier.
@@ -452,8 +1030,65 @@ module Woods
 
       # Memoized normalized edges — converts bare strings (old format) to hashes once.
       # Cleared by reload! alongside raw_graph_data.
+      #
+      # `edges` holds one type's edges per identifier. Where an identifier
+      # names units of several types the rest live in `variants`, and a
+      # traversal that read only the primary would report a unit as having no
+      # dependencies at all. They are unioned here so traversal follows the
+      # identifier's whole out-edge set.
       def normalized_graph_edges
-        @normalized_graph_edges ||= normalize_all_edges(raw_graph_data['edges'] || {})
+        @normalized_graph_edges ||= begin
+          edges = normalize_all_edges(raw_graph_data['edges'] || {})
+          variant_records.each do |record|
+            identifier = record['identifier']
+            next unless identifier
+
+            extra = normalize_all_edges(identifier => Array(record['edges']))
+            edges[identifier] = ((edges[identifier] || []) + extra[identifier]).uniq
+          end
+          edges
+        end
+      end
+
+      # Every unit type registered under each identifier, sorted.
+      #
+      # One entry for all but the handful of identifiers a codebase reuses
+      # across types; those get one per coexisting unit.
+      #
+      # @return [Hash{String => Array<String>}]
+      def graph_node_types
+        @graph_node_types ||= begin
+          types = (raw_graph_data['nodes'] || {}).transform_values { |node| [node['type']].compact }
+          variant_records.each do |record|
+            identifier = record['identifier']
+            next unless identifier && record['type']
+
+            (types[identifier] ||= []) << record['type']
+          end
+          types.transform_values { |list| list.uniq.sort }
+        end
+      end
+
+      # Does the graph span more than one database?
+      #
+      # Nodes carry `database` only from Rails 6.1 on, and only for models, so
+      # an app with one database (or none recorded) answers false and the
+      # traversal shape is unchanged for it.
+      #
+      # @return [Boolean]
+      def multi_database_graph?
+        return @multi_database_graph if defined?(@multi_database_graph)
+
+        databases = (raw_graph_data['nodes'] || {}).each_value.filter_map { |node| node['database'] }
+        @multi_database_graph = databases.uniq.size > 1
+      end
+
+      # @return [Array<Hash>] the graph's `variants` section, empty when the
+      #   graph has no identifier shared across types (and for every graph
+      #   written before the section existed)
+      def variant_records
+        records = raw_graph_data['variants']
+        records.is_a?(Array) ? records.grep(Hash) : []
       end
 
       # Build identifier → { type_dir, filename } map from all _index.json files.
@@ -480,42 +1115,118 @@ module Woods
       def read_index(dir)
         @index_cache ||= {}
         @index_cache[dir] ||= begin
-          path = @index_dir.join(dir, '_index.json')
-          path.file? ? JSON.parse(path.read) : []
+          path = current_payload_dir.join(dir, '_index.json')
+          path.file? ? JSON.parse(read_utf8(path)) : []
         end
       end
 
       # Load a unit JSON file with LRU cache eviction.
+      #
+      # The bookkeeping runs under {@cache_mutex}; the file read deliberately
+      # does not, so concurrent readers still parse different units in
+      # parallel. `woods-mcp-http` executes tool handlers on its Rack server's
+      # request threads, and unsynchronized bookkeeping drifts: two threads
+      # can both observe `size >= MAX_UNIT_CACHE` and both evict, or both push
+      # the same key so the order array accumulates a duplicate. A duplicate
+      # for a key never read again is never cleaned up (only a cache *hit*
+      # dedupes it), so `shift` starts evicting keys that are no longer in the
+      # cache and the cache creeps past its cap — a slow leak rather than a
+      # wrong answer, but a leak.
+      #
+      # Values are unaffected either way: both threads parse the same file.
       def load_unit(type_dir, filename)
         cache_key = "#{type_dir}/#{filename}"
+        path = current_payload_dir.join(type_dir, filename)
+        open_unit(path.to_s) do |file|
+          signature = unit_file_signature(file.stat)
+          cached = @cache_mutex.synchronize do
+            if @unit_cache.key?(cache_key) && @unit_cache_signatures[cache_key] == signature
+              # Move to end (most recently used)
+              @unit_cache_order.delete(cache_key)
+              @unit_cache_order.push(cache_key)
+              @unit_cache[cache_key]
+            elsif @unit_cache.key?(cache_key)
+              @unit_cache.delete(cache_key)
+              @unit_cache_signatures.delete(cache_key)
+              @unit_cache_order.delete(cache_key)
+              nil
+            end
+          end
+          return cached if cached
 
-        if @unit_cache.key?(cache_key)
-          # Move to end (most recently used)
-          @unit_cache_order.delete(cache_key)
-          @unit_cache_order.push(cache_key)
-          return @unit_cache[cache_key]
-        end
-
-        path = @index_dir.join(type_dir, filename)
-        return nil unless path.file?
+          data = JSON.parse(file.read)
 
-        data = JSON.parse(path.read)
+          @cache_mutex.synchronize do
+            # Evict oldest if at capacity
+            if @unit_cache.size >= MAX_UNIT_CACHE && !@unit_cache.key?(cache_key)
+              oldest = @unit_cache_order.shift
+              @unit_cache.delete(oldest)
+              @unit_cache_signatures.delete(oldest)
+            end
 
-        # Evict oldest if at capacity
-        if @unit_cache.size >= MAX_UNIT_CACHE
-          oldest = @unit_cache_order.shift
-          @unit_cache.delete(oldest)
+            @unit_cache[cache_key] = data
+            @unit_cache_signatures[cache_key] = signature
+            @unit_cache_order.delete(cache_key)
+            @unit_cache_order.push(cache_key)
+          end
+          data
         end
+      end
+
+      def unit_file_signature(stat)
+        [stat.dev, stat.ino, stat.size, stat.mtime.to_r, stat.ctime.to_r]
+      end
 
-        @unit_cache[cache_key] = data
-        @unit_cache_order.push(cache_key)
-        data
+      def open_unit(path, &block)
+        File.open(path, 'rb', &block)
       end
 
-      # Parse a JSON file relative to the index directory.
+      # Read an index artifact as UTF-8, whatever the host locale.
+      #
+      # Bare Pathname#read tags the bytes with Encoding.default_external. On
+      # a C/US-ASCII host that tag is US-ASCII, so a single non-ASCII byte in
+      # an artifact (a branch like "feature/café" in manifest.json, a unit
+      # identifier, prose in SUMMARY.md) makes JSON.parse raise
+      # Encoding::InvalidByteSequenceError and leaves bare string reads with
+      # an invalid encoding — search, lookup, dependencies, dependents,
+      # framework, and recent_changes then surface as misleading
+      # corrupt_artifact results. Reading binary (the mode {#load_unit} has
+      # always used) and forcing UTF-8 keeps the bytes byte-identical while
+      # giving them the encoding extraction writes artifacts in.
+      #
+      # @param path [Pathname] Artifact path inside the loaded generation
+      # @return [String] Contents, tagged UTF-8
+      def read_utf8(path)
+        path.binread.force_encoding(Encoding::UTF_8)
+      end
+
+      # Parse a JSON payload file (manifest, graph, analysis) resolved through
+      # the loaded generation's {#payload_dir}.
       def parse_json(filename)
-        path = @index_dir.join(filename)
-        JSON.parse(path.read)
+        path = current_payload_dir.join(filename)
+        JSON.parse(read_utf8(path))
+      end
+
+      # Deep-freeze a parsed JSON structure (hashes, arrays, and the strings
+      # and primitives inside them) so a shared parse result cannot be
+      # mutated through any reference.
+      #
+      # JSON.parse(..., freeze: true) would do this natively, but that
+      # keyword needs json >= 2.7.0 while the supported floor (Ruby 3.0)
+      # bundles json 2.5 — a host app on the floor would get ArgumentError
+      # from this path. A small recursive walk is version-independent.
+      #
+      # @param value [Object]
+      # @return [Object] the same structure, frozen
+      def deep_freeze_json(value)
+        case value
+        when Hash then value.each do |key, item|
+          deep_freeze_json(key)
+          deep_freeze_json(item)
+        end.freeze
+        when Array then value.each { |item| deep_freeze_json(item) }.freeze
+        else value.freeze
+        end
       end
 
       # BFS traversal in either direction.
@@ -554,12 +1265,12 @@ module Woods
                         resolve_reverse_neighbors(graph_data, normalized_edges, current, via_set)
                       end
 
-          # Filter by node type if requested
+          # Filter by node type if requested. An identifier naming units of
+          # several types matches when any of them does — excluding it because
+          # the type that happens to sort first is not the requested one would
+          # hide a unit the filter asked for.
           filtered = if type_set
-                       neighbors.select do |n|
-                         node_meta = nodes_data[n]
-                         node_meta && type_set.include?(node_meta['type'])
-                       end
+                       neighbors.select { |n| graph_node_types[n]&.any? { |t| type_set.include?(t) } }
                      else
                        neighbors
                      end
@@ -569,11 +1280,19 @@ module Woods
           # node's deps list already shows this node as a child.
           will_expand = current_depth < depth
           node_meta = nodes_data[current]
-          result_nodes[current] = {
+          entry = {
             type: node_meta&.dig('type'),
             depth: current_depth,
             deps: will_expand ? filtered : []
           }
+          # Only when the identifier is genuinely ambiguous, so the shape is
+          # unchanged for every node in an index with no shared identifiers.
+          node_types = graph_node_types[current] || []
+          entry[:types] = node_types if node_types.size > 1
+          # Likewise: a single-database app learns nothing from a column that
+          # answers the same thing on every row (B-183).
+          entry[:database] = node_meta&.dig('database') if multi_database_graph?
+          result_nodes[current] = entry
 
           next unless will_expand