woods 1.6.1 → 2.0.0.beta2

data/lib/woods/retrieval/ranker.rb

@@ -30,6 +30,13 @@ module Woods
         diversity: 0.05
       }.freeze
 
+      # Chunked units embed as `Identifier#chunk_N`; metadata and the graph are
+      # keyed on the bare identifier. Mirrors the same constant in Retriever,
+      # Bootstrapper, ContextAssembler and Indexer — kept local to each, as
+      # they are, rather than introducing a shared home in a patch release.
+      CHUNK_SUFFIX_PATTERN = /#chunk_\d+\z/
+      private_constant :CHUNK_SUFFIX_PATTERN
+
       # RRF constant — balances rank position vs. absolute score.
       # Standard value from the original RRF paper (Cormack et al., 2009).
       RRF_K = 60
@@ -44,9 +51,16 @@ module Woods
 
       # Rank candidates by weighted signal scoring with diversity adjustment.
       #
+      # Returned candidates carry their final weighted score (0.0–1.0) in
+      # +score+ — NOT the raw retrieval score they arrived with. Downstream
+      # consumers ({ContextAssembler#assemble_section} re-sorts by +score+;
+      # source attributions report it) must agree with the ranked order, so
+      # the ranker rewrites the score the same way {#apply_rrf} always has.
+      #
       # @param candidates [Array<Candidate>] Search candidates from executor
       # @param classification [QueryClassifier::Classification] Query classification
-      # @return [Array<Candidate>] Re-ranked candidates (best first)
+      # @return [Array<Candidate>] Re-ranked candidates (best first), scores
+      #   rewritten to the final weighted score
       def rank(candidates, classification:)
         return [] if candidates.empty?
 
@@ -57,11 +71,53 @@ module Woods
         sorted = sorted_by_weighted_score(scored)
         apply_diversity_penalty(sorted)
 
-        sorted.map { |item| item[:candidate] }
+        finalize_ranked(sorted)
+      end
+
+      # Drop the memoized PageRank rank-percentile map so the next {#rank}
+      # call recomputes it from the graph store.
+      #
+      # The map is memoized per Ranker instance ({#pagerank_importance_map}),
+      # so a Ranker outlives any change to the graph it was built against.
+      # The Index MCP reload path does NOT call this: the M7 build-then-swap
+      # transaction ({Woods::MCP::Bootstrapper.commit_reload!}) installs a
+      # whole fresh Pipeline — including a fresh Ranker with no memo — rather
+      # than mutating a live graph in place, and must keep doing so.
+      #
+      # This is therefore an API for direct embedders only: code that holds a
+      # Ranker itself and repoints its graph store without rebuilding the
+      # Ranker has to invalidate the memo here, or the ranker keeps scoring
+      # importance from the retired graph.
+      #
+      # @return [void]
+      def invalidate_pagerank_cache!
+        @pagerank_importance_map = nil
       end
 
       private
 
+      # Rebuild each ranked candidate with its final weighted score.
+      #
+      # Mirrors {#rebuild_rrf_candidates}: a fresh Candidate is built rather
+      # than mutating the executor's structs in place, so callers holding
+      # the pre-rank candidate list (e.g. RetrievalTrace consumers) keep
+      # their original scores.
+      #
+      # @param sorted [Array<Hash>] Scored items sorted by weighted_score
+      # @return [Array<Candidate>]
+      def finalize_ranked(sorted)
+        sorted.map do |item|
+          candidate = item[:candidate]
+          build_candidate(
+            identifier: candidate.identifier,
+            score: item[:weighted_score],
+            source: candidate.source,
+            metadata: candidate.metadata,
+            matched_fields: candidate.matched_fields
+          )
+        end
+      end
+
       # Check if candidates come from multiple retrieval sources.
       #
       # @param candidates [Array<Candidate>]
@@ -76,66 +132,160 @@ module Woods
       # Each source's candidates are ranked independently, then RRF
       # merges ranks into a single score.
       #
+      # Raw RRF values live on a tiny scale (~1/(k+1) ≈ 0.016 per source),
+      # while every other ranking signal is 0.0–1.0 — feeding them straight
+      # into the weighted sum as +semantic+ made the 0.40 weight contribute
+      # at most ~0.02 and let recency/importance spreads dominate any fused
+      # query. Min-max normalizing across the merged list restores the
+      # 0.0–1.0 scale (see {#normalize_scores}); the single-source path is
+      # untouched because provider similarity scores already arrive 0–1.
+      #
       # @param candidates [Array<Candidate>]
-      # @return [Array<Candidate>] Merged candidates with RRF scores
+      # @return [Array<Candidate>] Merged candidates with normalized RRF scores
       def apply_rrf(candidates)
-        rrf_scores, metadata_map = compute_rrf_scores(candidates)
-        rebuild_rrf_candidates(candidates, rrf_scores, metadata_map)
+        rrf_scores, metadata_map, matched_fields_map = compute_rrf_scores(candidates)
+        rebuild_rrf_candidates(candidates, normalize_scores(rrf_scores), metadata_map, matched_fields_map)
       end
 
       # Compute RRF scores across all sources.
       #
-      # @return [Array(Hash, Hash)] [rrf_scores, metadata_map]
+      # Keyed on the chunk-stripped BASE identifier ({#base_identifier}),
+      # not the raw candidate identifier: a chunked unit's vector hit
+      # arrives as `Identifier#chunk_N` while its keyword hit arrives as
+      # the bare `Identifier` — keying on the raw id treated those as two
+      # unrelated identifiers and they never fused, so chunked corpora
+      # (rails_source-heavy hosts) got zero RRF benefit from hybrid search.
+      #
+      # Also accumulates per-identifier metadata (first non-empty wins) and
+      # matched fields (union across sources) so the merged candidate keeps
+      # the keyword signal — dropping +matched_fields+ here would kill
+      # {#keyword_score} on exactly the multi-source (hybrid) path where
+      # keyword evidence matters most.
+      #
+      # @return [Array(Hash, Hash, Hash)] [rrf_scores, metadata_map, matched_fields_map]
       def compute_rrf_scores(candidates)
         rrf_scores = Hash.new(0.0)
         metadata_map = {}
+        matched_fields_map = {}
 
         candidates.group_by(&:source).each_value do |source_candidates|
           ranked = source_candidates.sort_by { |c| -c.score }
           ranked.each_with_index do |candidate, idx|
+            base_id = base_identifier(candidate.identifier)
             # RRF is 1-based (Cormack et al., 2009): top-ranked doc uses rank 1, not 0.
-            rrf_scores[candidate.identifier] += 1.0 / (RRF_K + idx + 1)
-            metadata_map[candidate.identifier] ||= candidate.metadata
+            rrf_scores[base_id] += 1.0 / (RRF_K + idx + 1)
+            # `||=` alone is wrong here: a graph-expansion candidate's `{}`
+            # is truthy, so it would permanently shadow a real metadata hash
+            # arriving from a later-processed source. Only lock in a value
+            # once it's actually non-empty.
+            existing = metadata_map[base_id]
+            metadata_map[base_id] = candidate.metadata if existing.nil? || existing.empty?
+            merge_matched_fields(matched_fields_map, base_id, candidate)
           end
         end
 
-        [rrf_scores, metadata_map]
+        [rrf_scores, metadata_map, matched_fields_map]
+      end
+
+      # Union a candidate's matched fields into the per-identifier map.
+      #
+      # @param matched_fields_map [Hash{String => Array<String>}] Accumulator (mutated)
+      # @param identifier [String] Chunk-stripped base identifier to accumulate under
+      # @param candidate [Candidate]
+      # @return [void]
+      def merge_matched_fields(matched_fields_map, identifier, candidate)
+        fields = candidate.respond_to?(:matched_fields) ? candidate.matched_fields : nil
+        return unless fields
+
+        matched_fields_map[identifier] = (matched_fields_map[identifier] || []) | fields
       end
 
-      # Rebuild candidates with merged RRF scores.
+      # Min-max normalize a score map to the 0.0–1.0 range.
+      #
+      # The degenerate all-equal case (including a single candidate) maps
+      # everything to 1.0 — an identifier every source agreed on is a full-
+      # strength semantic match, not a zero.
+      #
+      # @param scores [Hash{String => Float}]
+      # @return [Hash{String => Float}]
+      def normalize_scores(scores)
+        min, max = scores.values.minmax
+        spread = max - min
+        return scores.transform_values { 1.0 } if spread.zero?
+
+        scores.transform_values { |score| (score - min) / spread }
+      end
+
+      # Rebuild candidates with merged (normalized) RRF scores.
+      #
+      # +identifier+ in +rrf_scores+ is already the chunk-stripped base
+      # identifier (see {#compute_rrf_scores}) — the merged Candidate is
+      # built under that base id, which is also what {#score_candidates}
+      # looks metadata up by, so a fused chunk/base pair scores and
+      # displays as the single unit it represents.
       #
       # @return [Array<Candidate>]
-      def rebuild_rrf_candidates(candidates, rrf_scores, metadata_map)
-        # Plain-Ruby `index_by` substitute — the ActiveSupport version
-        # isn't loaded when the gem runs outside a Rails boot. Preserve
-        # last-wins semantics to match ActiveSupport's `Enumerable#index_by`
-        # so the merged candidate's `source` continues to reflect the
-        # final source a given identifier appeared in (relevant when
-        # observability/debug tools read `.source` on an RRF result).
-        original_by_id = {}
-        candidates.each { |c| original_by_id[c.identifier] = c }
+      def rebuild_rrf_candidates(candidates, rrf_scores, metadata_map, matched_fields_map)
+        original_by_id = pick_merged_source(candidates)
         rrf_scores.sort_by { |_id, score| -score }.map do |identifier, score|
           original = original_by_id[identifier]
           build_candidate(
             identifier: identifier,
             score: score,
             source: original&.source || :rrf,
-            metadata: metadata_map[identifier]
+            metadata: metadata_map[identifier],
+            matched_fields: matched_fields_map[identifier]
           )
         end
       end
 
+      # Pick the representative candidate (for +source+ attribution) per
+      # base identifier across every source that hit it.
+      #
+      # +:graph_expansion+ never wins this pick over a vector/keyword/direct
+      # duplicate: a unit strongly hit by vector search AND reached via
+      # graph expansion is a real primary result, not incidental context,
+      # and {ContextAssembler#add_result_sections} routes solely on
+      # +candidate.source+ to decide primary vs. supporting placement.
+      # Plain last-wins (the prior behavior, matching ActiveSupport's
+      # +Enumerable#index_by+) let whichever duplicate the source-grouping
+      # loop visited last silently demote it. Within a precedence tier,
+      # last-wins is preserved.
+      #
+      # @param candidates [Array<Candidate>]
+      # @return [Hash{String => Candidate}]
+      def pick_merged_source(candidates)
+        original_by_id = {}
+        candidates.each do |c|
+          id = base_identifier(c.identifier)
+          existing = original_by_id[id]
+          next if existing && existing.source != :graph_expansion && c.source == :graph_expansion
+
+          original_by_id[id] = c
+        end
+        original_by_id
+      end
+
       # Score each candidate across all signals.
       #
       # @param candidates [Array<Candidate>]
       # @param classification [QueryClassifier::Classification]
       # @return [Array<Hash>]
       def score_candidates(candidates, classification)
-        # Batch-fetch all metadata in one query instead of per-candidate lookups
-        unit_map = @metadata_store.find_batch(candidates.map(&:identifier))
+        # Look up by *base* identifier. A chunked unit's vector ids carry a
+        # `#chunk_N` suffix, but the metadata store and the PageRank map are
+        # both keyed on the unit — so passing the raw identifier missed every
+        # chunk, and recency, importance, type_match and diversity all fell
+        # back to their neutral values precisely on chunked corpora
+        # (rails_source-heavy hosts), leaving semantic+keyword to decide alone.
+        # The bootstrapper and the assembler already strip; the ranker was the
+        # one consumer that did neither.
+        base_ids = candidates.to_h { |candidate| [candidate.identifier, base_identifier(candidate.identifier)] }
+        unit_map = @metadata_store.find_batch(base_ids.values.uniq)
 
         candidates.map do |candidate|
-          unit = unit_map[candidate.identifier]
+          base_id = base_ids[candidate.identifier]
+          unit = unit_map[base_id]
 
           {
             candidate: candidate,
@@ -144,7 +294,7 @@ module Woods
               semantic: candidate.score.to_f,
               keyword: keyword_score(candidate),
               recency: recency_score(unit),
-              importance: importance_score(unit, candidate.identifier),
+              importance: importance_score(unit, base_id),
               type_match: type_match_score(unit, classification),
               diversity: 1.0 # Adjusted after initial sort
             }
@@ -193,6 +343,14 @@ module Woods
         end
       end
 
+      # The unit identifier behind a possibly chunk-suffixed candidate id.
+      #
+      # @param identifier [String]
+      # @return [String]
+      def base_identifier(identifier)
+        identifier.to_s.sub(CHUNK_SUFFIX_PATTERN, '')
+      end
+
       # Importance score based on PageRank / structural importance.
       #
       # Prefers live PageRank from the graph store (rank-percentile 0.0–1.0) when
@@ -204,7 +362,7 @@ module Woods
       # @param identifier [String] Candidate identifier (matched against PageRank keys)
       # @return [Float] 0.0 to 1.0
       def importance_score(unit, identifier)
-        pagerank = pagerank_importance_map[identifier]
+        pagerank = pagerank_importance_map[(unit && (unit['identifier'] || unit[:identifier])) || identifier]
         return pagerank if pagerank
 
         return 0.5 unless unit
@@ -231,6 +389,12 @@ module Woods
 
       # Compute rank-percentile scores from the graph store's PageRank hash.
       #
+      # `respond_to?` alone is the wrong guard: {Storage::GraphStore::Interface}
+      # *defines* +#pagerank+ as a +NotImplementedError+ stub, so an adapter
+      # that merely includes the interface without overriding it still
+      # answers +respond_to?+ with +true+ (B-108) — and +NotImplementedError+
+      # is a +ScriptError+, which the rescue below does not catch on its own.
+      #
       # @return [Hash{String => Float}] Empty hash when no graph store or no scores.
       def compute_pagerank_importance_map
         return {} unless @graph_store.respond_to?(:pagerank)
@@ -243,7 +407,7 @@ module Woods
         ranked.each_with_index.to_h do |(identifier, _score), rank|
           [identifier, 1.0 - (rank / total)]
         end
-      rescue StandardError
+      rescue StandardError, NotImplementedError
         {}
       end
 
@@ -351,12 +515,13 @@ module Woods
       # Build a Candidate struct compatible with SearchExecutor::Candidate.
       #
       # @return [Candidate-like Struct]
-      def build_candidate(identifier:, score:, source:, metadata:)
+      def build_candidate(identifier:, score:, source:, metadata:, matched_fields: nil)
         SearchExecutor::Candidate.new(
           identifier: identifier,
           score: score,
           source: source,
-          metadata: metadata
+          metadata: metadata,
+          matched_fields: matched_fields
         )
       end
     end