woods 1.6.1 → 2.0.0.beta2

data/lib/woods/mcp/bootstrapper.rb

@@ -9,6 +9,7 @@ require_relative 'provider_probe'
 require_relative '../index_artifact'
 require_relative '../builder'
 require_relative '../resolved_config'
+require_relative '../generation'
 require_relative '../storage/snapshotter'
 require_relative '../storage/inapplicable_backend'
 
@@ -33,7 +34,7 @@ module Woods
           exit 1
         end
 
-        unless File.exist?(File.join(dir, 'manifest.json'))
+        unless manifest_present?(dir)
           warn "Error: No manifest.json found in: #{dir}"
           warn 'Run `bundle exec rake woods:extract` in your Rails app first.'
           exit 1
@@ -42,6 +43,26 @@ module Woods
         dir
       end
 
+      # Is there a manifest this index would resolve one from?
+      #
+      # A flat index (pre-payload, or never bumped) answers with
+      # +manifest.json+ directly under +dir+. A payload-born index (#164
+      # payloads) has none there — every artifact lives under the directory
+      # +generation.json+'s +payload+ pointer names — so the pointer is
+      # followed before concluding there is no index at all. Mirrors
+      # {Woods::MCP::IndexReader#manifest_present?} exactly; keep the two in
+      # agreement.
+      #
+      # @param dir [String] candidate index directory
+      # @return [Boolean]
+      def self.manifest_present?(dir)
+        return true if File.exist?(File.join(dir, 'manifest.json'))
+
+        generation = Woods::Generation.new(output_dir: dir)
+        generation.payload_dir(generation.current).join('manifest.json').file?
+      end
+      private_class_method :manifest_present?
+
       # Build a snapshot store for temporal tracking.
       #
       # Auto-enables when a SQLite database already exists in the index directory,
@@ -111,6 +132,10 @@ module Woods
         state.mark(:hydrating)
 
         artifact = build_artifact(index_dir)
+        if static_source_map_without_embeddings?(artifact)
+          state.mark(:degraded, reason: Woods::Error.new('static source map has no embedding artifact'))
+          return [nil, state]
+        end
         config, _source = ConfigResolver.resolve(Woods.configuration,
                                                  artifact: artifact,
                                                  ollama_probe: method(:ollama_reachable?))
@@ -128,104 +153,476 @@ module Woods
         # artifact to validate against.
         resolved = build_resolved_config(config)
         state.resolved_config = resolved
-        retriever = build_retriever_from_config(config, resolved, artifact)
+        retriever = build_retriever_from_config(config, resolved, artifact, state)
         probe_and_mark_state(config, state)
+        derive_state_from_store_health(state)
         warn "[woods-mcp] semantic search: #{state.status} (#{config.embedding_provider})"
 
         [retriever, state]
       end
 
-      # Backwards-compatible wrapper — existing callers (exe/woods-mcp and
-      # exe/woods-mcp-http) just want the retriever. They rescue typed
-      # BootstrapError at their own top level; we do not catch here.
-      def self.build_retriever_compat(index_dir: nil)
-        retriever, _state = build_retriever(index_dir: index_dir)
-        retriever
+      def self.static_source_map_without_embeddings?(artifact)
+        return false unless artifact
+
+        generation = Woods::Generation.new(output_dir: artifact.output_dir)
+        manifest_path = generation.payload_dir(generation.current).join('manifest.json')
+        return false unless manifest_path.file?
+
+        provenance = JSON.parse(Woods::AtomicFile.read(manifest_path))['provenance'] || {}
+        provenance['mode'] == 'woods_static_ruby_source' && provenance['embeddings'] == 'absent'
+      rescue JSON::ParserError, SystemCallError
+        false
       end
+      private_class_method :static_source_map_without_embeddings?
 
       # Refresh a live retriever's in-memory stores from the latest dumps on
       # disk. Used by the MCP +reload+ tool so agents can pick up a fresh embed
-      # run without restarting the process. The retriever instance is preserved
-      # (tool closures kept their reference) — only the stores are mutated.
+      # run without restarting the process.
+      #
+      # The transaction is build-then-swap (M7) — the old +clear!+ +
+      # +bulk_load+ on the LIVE stores left an empty or half-loaded window a
+      # concurrent reader could observe:
       #
-      # No-op when:
+      #   1. Capture ONCE, before any candidate work: the served generation
+      #      marker AND the promoted dump identity (+dumps/latest+). Every
+      #      candidate then hydrates EXCLUSIVELY from those captured
+      #      locations — config from the captured dump's embedded snapshot,
+      #      vector/metadata from the captured dump directory, the graph from
+      #      the captured marker's payload — never from whatever is on disk
+      #      mid-build.
+      #   2. If ANY candidate fails: the reader is not reloaded and no store
+      #      is swapped — the previous fully aligned generation stays served,
+      #      the old retriever keeps answering, and a DISTINCT reload-phase
+      #      degraded condition is recorded on +state+ (never the boot
+      #      degraded state, which describes the old stores). Raised as
+      #      {Woods::MCP::ReloadDegraded}. The same fail-closed rule applies
+      #      before any candidate work when the captured dump's embedded
+      #      config names store types the refreshable live target cannot
+      #      refresh (a re-embed switched stores): that divergence must never
+      #      surface as an empty success (M2).
+      #   3. Commit: acquire the SAME on-disk PipelineLock every writer uses
+      #      (bounded poll; a reload must not queue behind a long extraction),
+      #      THEN the reader's exclusive generation lock, THEN recheck BOTH
+      #      identities. A generation movement raises
+      #      {Woods::MCP::ReloadGenerationMoved}; a promoted-dump movement
+      #      raises {Woods::MCP::ReloadDumpMoved} (an embed promotes without
+      #      bumping the generation file, so the dump identity is rechecked
+      #      separately). The writer lock is held through reader alignment and
+      #      the single Pipeline assignment — no publication can interleave
+      #      between recheck and swap — and released AFTER the reader lock.
+      #   4. A successful reload clears the reload-failure condition.
+      #
+      # Deterministic-interleaving hooks (M7 specs): +hooks+ carries optional
+      # callables — +after_vector_candidate:+ (post-vector, pre-metadata
+      # hydration), +after_pipeline_lock:+ (writer lock held, reader lock not
+      # yet taken), +before_swap:+ (both rechecks passed, Pipeline assignment
+      # imminent), +after_swap:+ (bundle swapped). Production passes nil.
+      #
+      # A zero-count no-op when:
       #   - +retriever+ is nil (no embedding provider configured)
-      #   - stores are durable (pgvector / Qdrant auto-refresh externally)
+      #   - the live stores are durable (pgvector / Qdrant auto-refresh
+      #     externally) — a partial all-or-nothing swap would mix backends
       #   - +woods.json+ is absent (Shape-1 deployments don't use Snapshotter)
+      #   - no dump has ever been promoted (extraction ran, +woods:embed+
+      #     never did) — boot hydrates that shape with +load_or_empty+, so
+      #     reload must not fail closed over it (MCP-1)
+      #
+      # Every one of those returns still refreshes +reader+ first (MCP-2).
+      # There is no store work to do, but the JSON index may still have moved,
+      # and on a flat (pre-2.0) index the reader never self-refreshes — reload
+      # is its only freshness path, so a store no-op must not become an index
+      # no-op reported as +reloaded: true+ over the retired generation.
       #
-      # @param retriever [Woods::Retriever, nil]
+      # @param retriever [Woods::Retriever, Cache::CachedRetriever, nil]
       # @param index_dir [String, Pathname]
+      # @param reader [Woods::MCP::IndexReader, nil] when given, the commit
+      #   phase runs under the reader's exclusive generation lock and the
+      #   reader's caches are reloaded alongside the swap
+      # @param state [Woods::MCP::BootstrapState, nil] records the reload
+      #   -phase degraded condition and its recovery
+      # @param hooks [Hash, nil] optional deterministic-interleaving hooks
       # @return [Hash] Stats — +{ vectors:, metadata:, graph: }+ record counts
-      # @raise [Woods::MCP::BootstrapError] surfaced from ConfigResolver / Snapshotter
-      def self.reload_stores!(retriever, index_dir:)
-        return { vectors: 0, metadata: 0, graph: 0 } unless retriever
+      # @raise [Woods::MCP::ReloadDegraded] the transaction aborted; nothing
+      #   was swapped and the generation named by the error is still served
+      def self.reload_stores!(retriever, index_dir:, reader: nil, state: nil, hooks: nil)
+        zero_counts = { vectors: 0, metadata: 0, graph: 0 }
+        return refresh_reader_only(reader, zero_counts) unless retriever
+
+        target = swap_target(retriever)
+        return refresh_reader_only(reader, zero_counts) unless target
 
         artifact = build_artifact(index_dir)
-        config, _source = ConfigResolver.resolve(Woods.configuration,
-                                                 artifact: artifact,
-                                                 ollama_probe: method(:ollama_reachable?))
-        resolved = build_resolved_config(config)
+        return refresh_reader_only(reader, zero_counts) unless artifact
 
-        vectors_count = refill_in_memory_vector_store(retriever, config, resolved, artifact)
-        metadata_count = refill_in_memory_metadata_store(retriever, config, resolved, artifact)
-        graph_count = refill_in_memory_graph_store(retriever, config, artifact)
+        generation = Woods::Generation.new(output_dir: artifact.output_dir)
 
-        # Context-cache entries from the previous embed run no longer agree
-        # with the refreshed stores. Drop them so the next codebase_retrieve
-        # call goes through the full pipeline with the new data. Embedding
-        # caches (query → vector) survive — that mapping is deterministic
-        # for a given provider+model.
-        retriever.invalidate_context_cache! if retriever.respond_to?(:invalidate_context_cache!)
+        # CAPTURE ONCE — the identities every candidate and the commit recheck
+        # are anchored to. Nothing downstream re-reads a live pointer until
+        # the commit recheck itself.
+        served = generation.current
+        captured_dump = artifact.latest_dump_path
 
-        { vectors: vectors_count, metadata: metadata_count, graph: graph_count }
+        # Nothing has ever been promoted: extraction ran, `woods:embed` did
+        # not. Boot hydrates this with `load_or_empty` and serves empty stores
+        # happily, so reload has to agree — building candidates with
+        # `required: true` against a nil dump raises MissingArtifact, degrades
+        # the transaction, and sticks a reload_failure on `state` that only a
+        # SUCCESSFUL reload clears, which is impossible until an embed runs
+        # (MCP-1). A dump that IS promoted but incomplete stays fail-closed.
+        return refresh_reader_only(reader, zero_counts) if captured_dump.nil?
+
+        begin
+          config, _source = ConfigResolver.resolve(
+            Woods.configuration,
+            artifact: artifact,
+            ollama_probe: method(:ollama_reachable?),
+            stored_config: captured_stored_config(artifact, captured_dump)
+          )
+        rescue StandardError => e
+          raise ReloadDegraded.new(
+            'reload could not resolve the index configuration; the previous generation is still ' \
+            "being served: #{e.class}: #{e.message}",
+            generation: served.number, stores: %w[vector metadata graph], error: e
+          )
+        end
+        resolved = build_resolved_config(config)
+
+        return refresh_reader_only(reader, zero_counts) unless refreshable_stores?(target)
+
+        # A captured dump whose embedded config names store types the
+        # refreshable live target cannot refresh cannot be turned into
+        # candidates at all — fail closed with the reload-phase degraded
+        # condition instead of an empty success (M2).
+        assert_dump_stores_refreshable!(target, config, served)
+
+        # Phase 1: candidates off-side, exclusively from the captured locations.
+        candidates = build_reload_candidates(config, resolved, artifact, served, captured_dump, hooks)
+        return refresh_reader_only(reader, zero_counts) unless candidates
+
+        # Phase 2: writer lock, reader lock, recheck both identities, swap.
+        commit_reload!(target, candidates, artifact: artifact, generation: generation,
+                                           served: served, captured_dump: captured_dump, reader: reader,
+                                           retriever: retriever, state: state, hooks: hooks)
+      rescue Woods::MCP::ReloadDegraded => e
+        state&.record_reload_failure(generation: e.generation, stores: e.stores,
+                                     reason: "#{e.class}: #{e.message}")
+        raise
       end
 
-      # Retriever (and CachedRetriever) expose public +vector_store+ /
-      # +metadata_store+ / +graph_store+ readers so this helper never pokes
-      # private state. Durable backends don't implement +clear!+/+bulk_load+
-      # — they return 0 silently because they're already refreshed externally.
-      def self.refill_in_memory_vector_store(retriever, config, resolved, artifact)
-        vs = retriever.respond_to?(:vector_store) ? retriever.vector_store : nil
-        return 0 unless vs.respond_to?(:clear!) && vs.respond_to?(:bulk_load)
+      # Refresh the reader's cached index state and answer the zero-count
+      # no-op (MCP-2).
+      #
+      # A store no-op is not an index no-op: the caller invoked `reload`
+      # because something on disk moved, and on a flat (pre-2.0) index
+      # {IndexReader} never self-refreshes, so skipping this reports
+      # +reloaded: true+ over the retired generation. Runs under the reader's
+      # exclusive generation lock, the same gate the swap path uses, so pinned
+      # readers drain before the caches are cleared.
+      #
+      # @param reader [Woods::MCP::IndexReader, nil]
+      # @param zero_counts [Hash]
+      # @return [Hash] +zero_counts+, unchanged
+      def self.refresh_reader_only(reader, zero_counts)
+        reader&.with_exclusive_reload { nil }
+        zero_counts
+      end
+      private_class_method :refresh_reader_only
+
+      # The stored config snapshot anchored at the CAPTURED dump (M7): the
+      # embedded +woods.json+ inside that dump directory, falling back to the
+      # root file for dumps that predate embedded configs — the same
+      # preference order as {IndexArtifact#read_config}, but pinned to the
+      # captured identity instead of re-reading the +latest+ pointer. Nil when
+      # neither exists, which resolves the same way a missing snapshot does.
+      def self.captured_stored_config(artifact, captured_dump)
+        return nil unless captured_dump
+
+        embedded = artifact.dump_config_path(captured_dump)
+        if embedded.exist?
+          ResolvedConfig.from_hash(JSON.parse(embedded.read(encoding: Encoding::UTF_8)))
+        elsif artifact.config_path.exist?
+          ResolvedConfig.from_hash(JSON.parse(artifact.config_path.read(encoding: Encoding::UTF_8)))
+        end
+      end
+      private_class_method :captured_stored_config
+
+      # Reach the swappable retriever inside a (possibly cache-wrapped)
+      # retriever. {Cache::CachedRetriever} keeps the real one in +@retriever+
+      # and nothing else wraps today; the guarded ivar probe degrades to nil
+      # on an unknown shape rather than raising mid-reload. Same rationale as
+      # the retired +extract_ranker+ helper this replaces.
+      def self.swap_target(retriever)
+        return retriever if retriever.respond_to?(:swap_stores!)
+
+        inner = (retriever.instance_variable_get(:@retriever) if retriever.instance_variable_defined?(:@retriever))
+        inner if inner.respond_to?(:swap_stores!)
+      end
+      private_class_method :swap_target
+
+      # Can this retriever's vector AND metadata stores both be refreshed
+      # here? In-memory stores expose +clear!+/+bulk_load+; durable backends
+      # (pgvector, Qdrant) don't implement +clear!+ — they're refreshed
+      # externally. The transaction is all-or-nothing, so a shape that can't
+      # refresh both takes the no-op rather than a partial swap.
+      def self.refreshable_stores?(target)
+        vs = target.vector_store
+        ms = target.metadata_store
+        vs.respond_to?(:clear!) && vs.respond_to?(:bulk_load) &&
+          ms.respond_to?(:clear!) && ms.respond_to?(:bulk_load)
+      end
+      private_class_method :refreshable_stores?
+
+      # The captured dump's embedded config and the refreshable live target
+      # must agree on store types (M2). Each is valid alone: the live target
+      # is refreshable (in-memory), and the dump is complete and valid. But
+      # when a re-embed ran with a different store configuration and promoted
+      # over the dump this process hydrated from, the reload-time resolver
+      # adopts the dump's store types, every candidate builder returns nil,
+      # and the transaction used to answer +reloaded: true+ with zero counts
+      # while NOTHING was swapped and no degraded condition was recorded —
+      # each component fine, the combination a misaligned-state success.
+      # Zero counts with a genuine empty dump stays legitimate; store-type
+      # divergence is a degraded reload: the reload-phase condition names
+      # both sides of the mismatch and the honest state, nothing swaps, and
+      # the previous generation keeps being served.
+      def self.assert_dump_stores_refreshable!(target, config, served)
+        divergent = {
+          vector: config.vector_store,
+          metadata: config.metadata_store
+        }.reject { |_, dump_type| dump_type == :in_memory }
+        return if divergent.empty?
+
+        detail = divergent.map do |component, dump_type|
+          live_store = component == :vector ? target.vector_store : target.metadata_store
+          "the promoted dump declares #{component}_store=#{dump_type.inspect} but the live retriever " \
+            "serves a #{live_store.class} #{component} store"
+        end.join('; ')
+
+        raise ReloadDegraded.new(
+          "reload aborted: #{detail}; nothing was swapped and the previous generation is still " \
+          'being served — restart woods-mcp to adopt the dump\'s store configuration, or re-run ' \
+          'woods:embed with the store configuration this server was started with',
+          generation: served.number, stores: divergent.keys
+        )
+      end
+      private_class_method :assert_dump_stores_refreshable!
+
+      # Immutable candidate bundle built off-side. +graph_store+ is nil when
+      # the index carries no +dependency_graph.json+ (the live graph is kept).
+      ReloadCandidates = Struct.new(:vector_store, :metadata_store, :graph_store,
+                                    :vector_count, :metadata_count, :graph_count,
+                                    keyword_init: true)
+      private_constant :ReloadCandidates
+
+      # Build every candidate store against the CAPTURED identities. Any
+      # failure raises {Woods::MCP::ReloadDegraded} naming the failing
+      # component; nothing has been swapped at this point.
+      #
+      # Vector and metadata candidates must both exist (the transaction never
+      # half-swaps); a nil graph candidate means "keep the live graph".
+      def self.build_reload_candidates(config, resolved, artifact, served, captured_dump, hooks)
+        vector = reload_vector_candidate(config, resolved, captured_dump, served)
+        return nil unless vector
+
+        run_hook(hooks, :after_vector_candidate)
+
+        metadata = reload_metadata_candidate(config, resolved, captured_dump, served)
+        return nil unless metadata
+
+        graph = reload_graph_candidate(config, artifact, served)
+
+        # Back-fill the candidate vector store's per-entry metadata from the
+        # candidate metadata store, OFF-SIDE before the swap. The WVF1 dump
+        # persists id + floats only — without this, every type-filtered
+        # search returns nothing after a reload (the boot path back-fills;
+        # reload must too). Same contract as the boot-path call in
+        # {.build_retriever_from_config}.
+        populate_vector_metadata(vector, metadata)
+
+        ReloadCandidates.new(
+          vector_store: vector, metadata_store: metadata, graph_store: graph,
+          vector_count: vector.count, metadata_count: metadata.count,
+          graph_count: graph ? 1 : 0
+        )
+      end
+      private_class_method :build_reload_candidates
 
-        fresh = hydrated_vector_store(config, resolved, artifact)
-        return 0 unless fresh
+      def self.reload_vector_candidate(config, resolved, captured_dump, served)
+        return nil unless config.vector_store == :in_memory
 
-        vs.clear!
-        vs.bulk_load(fresh.each_entry.map { |id, vec, meta| { id: id, vector: vec, metadata: meta } })
-        vs.respond_to?(:count) ? vs.count : 0
+        Woods::Storage::Snapshotter::Vector.load_dump_dir(
+          captured_dump, resolved_config: resolved, required: true
+        )
+      rescue StandardError => e
+        raise ReloadDegraded.new(
+          "vector store refresh failed: #{e.class}: #{e.message}",
+          generation: served.number, stores: [:vector], error: e
+        )
       end
-      private_class_method :refill_in_memory_vector_store
+      private_class_method :reload_vector_candidate
 
-      def self.refill_in_memory_metadata_store(retriever, config, resolved, artifact)
-        ms = retriever.respond_to?(:metadata_store) ? retriever.metadata_store : nil
-        return 0 unless ms.respond_to?(:clear!) && ms.respond_to?(:bulk_load)
+      def self.reload_metadata_candidate(config, resolved, captured_dump, served)
+        return nil unless config.metadata_store == :in_memory
+
+        Woods::Storage::Snapshotter::Metadata.load_dump_dir(
+          captured_dump, resolved_config: resolved, required: true
+        )
+      rescue StandardError => e
+        raise ReloadDegraded.new(
+          "metadata store refresh failed: #{e.class}: #{e.message}",
+          generation: served.number, stores: [:metadata], error: e
+        )
+      end
+      private_class_method :reload_metadata_candidate
 
-        fresh = hydrated_metadata_store(config, resolved, artifact)
-        return 0 unless fresh
+      def self.reload_graph_candidate(config, artifact, served)
+        hydrated_graph_store(config, artifact, nil, strict: true, marker: served)
+      rescue StandardError => e
+        raise ReloadDegraded.new(
+          "graph store refresh failed: #{e.class}: #{e.message}",
+          generation: served.number, stores: [:graph], error: e
+        )
+      end
+      private_class_method :reload_graph_candidate
+
+      # Commit phase: acquire the SAME on-disk PipelineLock every extraction /
+      # embed writer uses (bounded poll — a reload must not queue behind a
+      # long extraction), THEN the reader's exclusive generation lock, THEN
+      # recheck BOTH captured identities. The writer lock is held through the
+      # reader alignment and the one-assignment Pipeline swap and released
+      # AFTER the reader lock, so no writer publication can interleave between
+      # recheck and swap.
+      def self.commit_reload!(target, candidates, artifact:, generation:, served:,
+                              captured_dump:, reader:, retriever:, state:, hooks:)
+        lock = reload_extraction_lock(artifact.output_dir)
+        unless acquire_writer_lock_briefly(lock)
+          raise ReloadDegraded.new(
+            'reload could not acquire the extraction writer lock; a writer is actively ' \
+            'publishing. Nothing was swapped and the previous generation is still being ' \
+            'served — invoke reload again once the writer finishes.',
+            generation: served.number, stores: %w[vector metadata graph]
+          )
+        end
+
+        begin
+          run_hook(hooks, :after_pipeline_lock)
+
+          body = lambda do
+            # Recheck BOTH captured identities under the writer lock. An embed
+            # promotes without bumping the generation file, so the dump
+            # identity needs its own check — the generation marker alone
+            # would bless a bundle whose halves hydrate from two dumps.
+            current = generation.current
+            unless same_generation_marker?(served, current)
+              raise ReloadGenerationMoved.new(
+                "index generation moved during reload (captured #{served.number}, now #{current.number}); " \
+                'nothing was swapped and the previous generation is still being served — invoke reload again',
+                generation: served.number, stores: %w[vector metadata graph]
+              )
+            end
+            current_dump = artifact.latest_dump_path
+            unless same_dump_identity?(captured_dump, current_dump)
+              raise ReloadDumpMoved.new(
+                "promoted dump changed during reload (captured #{captured_dump}, now #{current_dump}); " \
+                'nothing was swapped and the previous generation is still being served — invoke reload again',
+                generation: served.number, stores: %w[vector metadata graph]
+              )
+            end
+
+            run_hook(hooks, :before_swap)
+
+            target.swap_stores!(
+              vector_store: candidates.vector_store,
+              metadata_store: candidates.metadata_store,
+              graph_store: candidates.graph_store || target.graph_store
+            )
+
+            # Align the reader caches with the swapped bundle — the retired
+            # generation's unit caches, identifier map and graph must not leak
+            # into responses describing the new one.
+            reader&.reload!
+
+            # Context-cache entries from the previous embed run no longer agree
+            # with the refreshed stores. Drop them so the next codebase_retrieve
+            # call goes through the full pipeline with the new data. Embedding
+            # caches (query → vector) survive — that mapping is deterministic
+            # for a given provider+model. The Ranker needs no memo invalidation:
+            # the swapped pipeline carries a FRESH ranker with no memoized
+            # PageRank from the retired graph.
+            retriever.invalidate_context_cache! if retriever.respond_to?(:invalidate_context_cache!)
+
+            # Successful recovery clears the reload-phase degraded condition.
+            state&.clear_reload_failure!
+
+            run_hook(hooks, :after_swap)
+
+            { vectors: candidates.vector_count, metadata: candidates.metadata_count,
+              graph: candidates.graph_count }
+          end
+
+          reader ? reader.with_exclusive_generation(&body) : body.call
+        ensure
+          # Release order: the reader lock (the block above) first, then the
+          # writer lock.
+          lock.release
+        end
+      end
+      private_class_method :commit_reload!
+
+      # The same lock every other writer against this index uses — rake
+      # extract/incremental/embed and the watch daemon all build it from the
+      # daemon's constants (keep in agreement with
+      # {Woods::MCP::Server.build_extraction_lock} and the rake helper).
+      def self.reload_extraction_lock(output_dir)
+        require 'woods/coordination/pipeline_lock'
+        require 'woods/watch/daemon'
+
+        Woods::Coordination::PipelineLock.new(
+          lock_dir: output_dir.to_s,
+          name: Woods::Watch::Daemon::LOCK_NAME,
+          stale_timeout: Woods::Watch::Daemon::LOCK_STALE_TIMEOUT
+        )
+      end
+      private_class_method :reload_extraction_lock
+
+      # Poll for the writer lock briefly. Monotonic, so a clock adjustment
+      # mid-wait cannot stretch or shrink the window. Bounded because a reload
+      # must not queue behind a full extraction; a busy writer fails the
+      # attempt with a typed retryable error instead.
+      WRITER_LOCK_WAIT = 2.0 # seconds — mirrors Server::PIPELINE_LOCK_WAIT
+
+      def self.acquire_writer_lock_briefly(lock)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + WRITER_LOCK_WAIT
+        acquired = lock.acquire
+        until acquired || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+          sleep 0.1
+          acquired = lock.acquire
+        end
+        acquired
+      end
+      private_class_method :acquire_writer_lock_briefly
 
-        ms.clear!
-        ms.bulk_load(fresh.each_entry)
-        ms.respond_to?(:count) ? ms.count : 0
+      def self.run_hook(hooks, name)
+        hooks&.fetch(name, nil)&.call
       end
-      private_class_method :refill_in_memory_metadata_store
+      private_class_method :run_hook
 
-      # GraphStore::Memory doesn't expose a +clear!+/+bulk_load+ pair today
-      # — a fresh run hands it an entirely new DependencyGraph from disk.
-      # Swap the inner graph via +replace_graph+ so SearchExecutor / Ranker /
-      # MCP tools keep their references to the same wrapper and see the new
-      # graph (no closure references break).
-      def self.refill_in_memory_graph_store(retriever, config, artifact)
-        gs = retriever.respond_to?(:graph_store) ? retriever.graph_store : nil
-        return 0 unless gs.respond_to?(:replace_graph)
+      # Token comparison, number fallback — mirrors IndexReader's
+      # same_generation? rule (two collapsed bumps share a number but not a
+      # token; pre-token generation files are told apart by number only).
+      def self.same_generation_marker?(captured, current)
+        return captured.number == current.number if captured.token.nil? || current.token.nil?
 
-        fresh = hydrated_graph_store(config, artifact)
-        return 0 if fresh.nil?
+        captured.token == current.token
+      end
+      private_class_method :same_generation_marker?
 
-        gs.replace_graph(fresh.graph)
-        1
+      # The promoted-dump identity: the resolved +dumps/latest+ path captured
+      # before candidate construction, compared against a fresh pointer read.
+      def self.same_dump_identity?(captured_dump, current_dump)
+        captured_dump.to_s == current_dump.to_s
       end
-      private_class_method :refill_in_memory_graph_store
+      private_class_method :same_dump_identity?
 
       # Check whether Ollama is reachable at the configured base URL.
       #
@@ -261,10 +658,10 @@ module Woods
       end
       private_class_method :build_artifact
 
-      def self.build_retriever_from_config(config, resolved, artifact)
-        vector_store = hydrated_vector_store(config, resolved, artifact)
-        metadata_store = hydrated_metadata_store(config, resolved, artifact)
-        graph_store = hydrated_graph_store(config, artifact)
+      def self.build_retriever_from_config(config, resolved, artifact, state = nil)
+        vector_store = hydrated_vector_store(config, resolved, artifact, state)
+        metadata_store = hydrated_metadata_store(config, resolved, artifact, state)
+        graph_store = hydrated_graph_store(config, artifact, state)
 
         # Cross-populate the vector store's per-entry metadata cache from
         # the metadata store. The WVF1 binary format stores only id + float
@@ -306,15 +703,32 @@ module Woods
       # Builder falls back to a fresh empty store — the pre-fix behaviour for
       # hosts that haven't run an extraction yet.
       #
+      # Resolved through {Woods::Generation#payload_dir}, same as every other
+      # payload artifact reader: a payload-born index (#164 payloads) keeps
+      # +dependency_graph.json+ only under +payloads/gen-N/+, never at the
+      # index root, so reading the root path unconditionally hydrated an
+      # empty graph and silently no-op'd PageRank / graph expansion. Resolves
+      # to the root itself for a flat (pre-payload) index, so this is a no-op
+      # change for every index that predates payloads.
+      #
       # @param config [Woods::Configuration]
       # @param artifact [Woods::IndexArtifact, nil]
+      # @param strict [Boolean] when true (reload candidate path), a soft
+      #   failure RAISES instead of degrading to an empty store — a reload
+      #   must never swap a good live store for an empty candidate (M7)
+      # @param marker [Woods::Generation::Marker, nil] resolve the payload
+      #   through THIS generation marker instead of the currently published
+      #   one (M7: the reload transaction reads the payload of the marker it
+      #   captured, so a concurrent publish cannot mix a graph from another
+      #   generation into the candidate bundle)
       # @return [Woods::Storage::GraphStore::Memory, nil]
       # @raise [Woods::Storage::InapplicableBackend] if the configured
       #   graph_store reports +durable? => true+
-      def self.hydrated_graph_store(config, artifact)
+      def self.hydrated_graph_store(config, artifact, state = nil, strict: false, marker: nil)
         return nil unless artifact
 
-        graph_json = artifact.output_dir.join('dependency_graph.json')
+        generation = Woods::Generation.new(output_dir: artifact.output_dir)
+        graph_json = generation.payload_dir(marker || generation.current).join('dependency_graph.json')
         return nil unless graph_json.exist?
 
         require_relative '../dependency_graph'
@@ -328,12 +742,20 @@ module Woods
                 'stores. Populate the durable backend via the extraction write path instead.'
         end
 
-        graph = Woods::DependencyGraph.from_h(JSON.parse(graph_json.read))
+        # AtomicFile.read, never a bare Pathname#read (H1): a bare read tags
+        # content with Encoding.default_external, and under LANG=C a graph
+        # holding any non-ASCII identifier raised
+        # Encoding::InvalidByteSequenceError — a locale bug that the M6
+        # degraded-state honesty turned into a full retrieval outage.
+        graph = Woods::DependencyGraph.from_h(JSON.parse(Woods::AtomicFile.read(graph_json)))
         Woods::Storage::GraphStore::Memory.new(graph)
       rescue Woods::Storage::InapplicableBackend
         raise
       rescue StandardError => e
+        raise if strict
+
         warn "[woods-mcp] graph hydration failed (#{e.class}: #{e.message}); starting with empty store"
+        state&.record_hydration_failure(:graph, e)
         nil
       end
       private_class_method :hydrated_graph_store
@@ -351,9 +773,13 @@ module Woods
       # Back-fill the vector store's per-entry metadata hashes from the
       # metadata store. Only makes sense when both are in-memory — durable
       # backends return nil from the hydration helpers and never reach
-      # this path.
+      # this path. Both callers hand over stores they built themselves —
+      # boot's freshly hydrated pair and {.build_reload_candidates}'
+      # off-side candidates (the M7 transaction retired the old in-place
+      # variant that reached a LIVE store) — but the guard below still has
+      # to hold: either store can turn out to be a durable adapter.
       def self.populate_vector_metadata(vector_store, metadata_store)
-        return unless vector_store.respond_to?(:each_entry) && vector_store.respond_to?(:store)
+        return unless implements_own?(vector_store, :each_entry) && vector_store.respond_to?(:store)
         return unless metadata_store.respond_to?(:find)
 
         # Collect (id, vector) pairs in one pass; overwriting via #store
@@ -365,16 +791,76 @@ module Woods
           meta = metadata_store.find(id.to_s.sub(CHUNK_SUFFIX_PATTERN, ''))
           next if meta.nil? || (meta.respond_to?(:empty?) && meta.empty?)
 
-          vector_store.store(id, vec, meta)
+          vector_store.store(id, vec, vector_filter_metadata(meta))
         end
       end
       private_class_method :populate_vector_metadata
 
+      # Does +object+ define +method_name+ itself, rather than merely
+      # inheriting {Storage::VectorStore::Interface}'s default
+      # {NotImplementedError} stub?
+      #
+      # Never test for this with a bare +respond_to?+ — the interface module
+      # *defines* every method as a raising stub, so every adapter (including
+      # pgvector/Qdrant, which implement +each_id+ but not +each_entry+)
+      # answers +true+. That was B-108: {.populate_reloaded_vector_metadata}'s
+      # guard used to be +respond_to?(:each_entry)+, which crashed the +reload+
+      # tool the moment a durable backend was configured — +NotImplementedError+
+      # is a +ScriptError+, which escapes every +rescue StandardError+ between
+      # here and the stdio transport loop and killed the server. Mirrors
+      # {Woods::Embedding::Indexer#implements_own?}.
+      #
+      # @param object [Object]
+      # @param method_name [Symbol]
+      # @return [Boolean]
+      def self.implements_own?(object, method_name)
+        return false unless object.respond_to?(method_name)
+        return true unless defined?(Storage::VectorStore::Interface)
+
+        object.method(method_name).owner != Storage::VectorStore::Interface
+      end
+      private_class_method :implements_own?
+
+      # Reduce a metadata-store record to the SYMBOL-keyed subset the live
+      # embed path writes per vector — see
+      # {Embedding::Indexer#store_vectors}: +{ type:, identifier:, file_path: }+
+      # — plus +namespace:+, which only the backfill can supply (the store
+      # carries it per unit) and which namespace-filtered search after a
+      # dump/reload depends on.
+      #
+      # The metadata store returns STRING-keyed records on every real
+      # backend (SQLite round-trips through JSON.parse; InMemory stringifies
+      # on store), while {Retrieval::SearchExecutor#build_vector_filters}
+      # builds symbol-keyed filters and +VectorStore::InMemory#gather_candidates+
+      # probes +meta[:type]+ directly. Back-filling the raw string-keyed
+      # record therefore made every type-filtered +codebase_retrieve+ return
+      # EMPTY on a dump-hydrated server (#150 item 5) — the live embed path
+      # writes symbol keys, which is why in-process specs never saw it.
+      # Nil fields are dropped rather than stored.
+      #
+      # @param meta [Hash] Metadata record (string- or symbol-keyed)
+      # @return [Hash{Symbol => Object}] Symbol-keyed filter subset
+      def self.vector_filter_metadata(meta)
+        {
+          type: meta['type'] || meta[:type],
+          identifier: meta['identifier'] || meta[:identifier],
+          file_path: meta['file_path'] || meta[:file_path],
+          namespace: meta['namespace'] || meta[:namespace]
+        }.compact
+      end
+      private_class_method :vector_filter_metadata
+
       # Return a hydrated InMemory vector store when Shape 2 applies
       # (in-memory configured + artifact on disk + resolved config) —
       # otherwise nil, which tells Builder to construct a fresh one.
       # Durable backends (pgvector, Qdrant) never match this path.
-      def self.hydrated_vector_store(config, resolved, artifact)
+      #
+      # A soft failure (transient I/O, corrupt dump) records the error on
+      # +state+ so the boot status reflects store health instead of reporting
+      # :hydrated over empty stores (M6). With +strict:+ (reload candidate
+      # path) a soft failure raises instead — a reload must never swap a
+      # good live store for an empty candidate (M7).
+      def self.hydrated_vector_store(config, resolved, artifact, state = nil, strict: false)
         return nil unless artifact && resolved
         return nil unless config.vector_store == :in_memory
 
@@ -385,12 +871,15 @@ module Woods
         # not a transient I/O issue. Operators must see these.
         raise
       rescue StandardError => e
+        raise if strict
+
         warn "[woods-mcp] vector hydration failed (#{e.class}: #{e.message}); starting with empty store"
+        state&.record_hydration_failure(:vector, e)
         nil
       end
       private_class_method :hydrated_vector_store
 
-      def self.hydrated_metadata_store(config, resolved, artifact)
+      def self.hydrated_metadata_store(config, resolved, artifact, state = nil, strict: false)
         return nil unless artifact && resolved
         return nil unless config.metadata_store == :in_memory
 
@@ -398,7 +887,10 @@ module Woods
       rescue Woods::MCP::BootstrapError, ArgumentError
         raise
       rescue StandardError => e
+        raise if strict
+
         warn "[woods-mcp] metadata hydration failed (#{e.class}: #{e.message}); starting with empty store"
+        state&.record_hydration_failure(:metadata, e)
         nil
       end
       private_class_method :hydrated_metadata_store
@@ -413,6 +905,19 @@ module Woods
              'starting degraded — will retry on first query'
       end
       private_class_method :probe_and_mark_state
+
+      # Derive the final boot status from store health (M6). Reaching the
+      # provider is not enough to claim :hydrated: a hydration soft-failure
+      # left empty in-memory stores, and a "healthy" status on top of them
+      # presents a server that answers everything with nothing as fully
+      # operational. Provider-unreachable degradations keep their reason;
+      # only a false :hydrated is corrected.
+      def self.derive_state_from_store_health(state)
+        return unless state.hydration_failed? && state.status == :hydrated
+
+        state.mark(:degraded, reason: state.hydration_failures.values.first)
+      end
+      private_class_method :derive_state_from_store_health
     end
   end
 end