woods 1.6.1 → 2.0.0.beta2

data/lib/woods/retrieval/search_executor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../storage_identity'
+
 module Woods
   module Retrieval
     # SearchExecutor maps a query classification to a retrieval strategy and
@@ -26,7 +28,12 @@ module Woods
     #
     class SearchExecutor
       # A single search candidate with provenance tracking.
-      Candidate = Struct.new(:identifier, :score, :source, :metadata, keyword_init: true)
+      #
+      # +matched_fields+ is populated only on the keyword path — the list of
+      # metadata field names whose values matched a query keyword. It feeds
+      # {Ranker#keyword_score} (0.25 per field, capped at 1.0). Defaults to
+      # nil for every other source (vector/graph/direct), which scores 0.0.
+      Candidate = Struct.new(:identifier, :score, :source, :metadata, :matched_fields, keyword_init: true)
 
       # The result of a search execution.
       ExecutionResult = Struct.new(:candidates, :strategy, :query, keyword_init: true)
@@ -60,7 +67,8 @@ module Woods
       #   strategies push this down into the vector store's metadata
       #   filter — used by {Retriever#retrieve} to rank-within-type when
       #   the unfiltered global top-K had no candidate of the requested type.
-      #   Overrides the classifier-derived +target_type+ in filter construction.
+      #   This explicit caller filter is the ONLY source of vector type
+      #   filters — the classifier's +target_type+ never becomes one (#184).
       # @param strategy [Symbol, nil] Override the classifier-selected strategy.
       #   {Retriever#within_type_fallback} passes +:vector+ here because the
       #   vector path is the only one that honors +type_filter+; if the
@@ -79,7 +87,7 @@ module Woods
         )
 
         ExecutionResult.new(
-          candidates: candidates.first(limit),
+          candidates: bounded_candidates(expand_graph_candidates(candidates), limit, strategy),
           strategy: strategy,
           query: query
         )
@@ -87,6 +95,23 @@ module Woods
 
       private
 
+      def expand_graph_candidates(candidates)
+        candidates.flat_map do |candidate|
+          next candidate unless %i[graph graph_expansion].include?(candidate.source)
+          next candidate if @metadata_store.find(candidate.identifier)
+
+          matches = @metadata_store.search(candidate.identifier).select do |record|
+            record['identifier'] == candidate.identifier
+          end
+          next candidate if matches.empty?
+
+          matches.map do |record|
+            Candidate.new(identifier: record['id'], score: candidate.score,
+                          source: candidate.source, metadata: record)
+          end
+        end
+      end
+
       # Select the best retrieval strategy for a classification.
       #
       # @param classification [QueryClassifier::Classification]
@@ -126,7 +151,7 @@ module Woods
       def run_strategy(strategy, query:, classification:, limit:, type_filter: nil)
         case strategy
         when :vector
-          execute_vector(query, classification: classification, limit: limit, type_filter: type_filter)
+          execute_vector(query, limit: limit, type_filter: type_filter)
         when :keyword
           execute_keyword(classification: classification, limit: limit)
         when :graph
@@ -140,10 +165,13 @@ module Woods
 
       # Vector strategy: embed the query and search by similarity.
       #
+      # Takes no classification — the vector path is filtered only by the
+      # caller's explicit +type_filter+; see {#build_vector_filters} (#184).
+      #
       # @return [Array<Candidate>]
-      def execute_vector(query, classification:, limit:, type_filter: nil)
+      def execute_vector(query, limit:, type_filter: nil)
         query_vector = @embedding_provider.embed(query)
-        filters = build_vector_filters(classification, type_filter: type_filter)
+        filters = build_vector_filters(type_filter)
 
         results = @vector_store.search(query_vector, limit: limit, filters: filters)
         results.map do |r|
@@ -153,8 +181,8 @@ module Woods
 
       # Keyword strategy: search metadata store by extracted keywords.
       #
-      # Searches each keyword individually and merges results, keeping the
-      # best score per identifier.
+      # Searches each keyword individually and merges results by identifier,
+      # scoring each merged hit by match quality (see {#keyword_match_score}).
       #
       # @return [Array<Candidate>]
       def execute_keyword(classification:, limit:)
@@ -165,33 +193,126 @@ module Woods
         rank_keyword_results(all_results, limit)
       end
 
-      # Search each keyword individually and merge, keeping best score per ID.
+      # Search each keyword individually and merge by identifier, unioning
+      # +matched_fields+ across keywords, then score each merged hit once
+      # from its complete matched-field set (see {#keyword_match_score}).
       #
       # @param keywords [Array<String>]
-      # @return [Hash<String, Hash>] id => { score:, metadata: }
+      # @return [Hash<String, Hash>] id => { score:, metadata:, matched_fields: }
       def merge_keyword_results(keywords)
         results_by_id = {}
         keywords.each do |keyword|
           results = @metadata_store.search(keyword)
-          results.each_with_index do |r, index|
-            id = r['id']
-            score = 1.0 - (index.to_f / [results.size, 10].max)
-            results_by_id[id] = { score: score, metadata: r } if !results_by_id[id] || score > results_by_id[id][:score]
+          results.each do |r|
+            merge_keyword_hit(results_by_id, r['id'], r, matched_fields_for(r, keyword))
           end
         end
-        results_by_id
+        score_keyword_hits(results_by_id)
+      end
+
+      # Fold a single keyword hit into the merged results: matched fields
+      # union across keywords, metadata kept from the first hit seen.
+      #
+      # @param results_by_id [Hash<String, Hash>] Accumulator (mutated)
+      # @param id [String] Candidate identifier
+      # @param record [Hash] The metadata record returned by the store
+      # @param matched [Array<String>] Fields that matched this keyword
+      # @return [void]
+      def merge_keyword_hit(results_by_id, id, record, matched)
+        entry = results_by_id[id]
+        if entry.nil?
+          results_by_id[id] = { metadata: record, matched_fields: matched }
+        else
+          entry[:matched_fields] |= matched
+        end
+      end
+
+      # Score each merged hit by match QUALITY (its final matched-field
+      # count) once every keyword has been folded in, rather than by the
+      # metadata store's result-row position.
+      #
+      # The store's +#search+ has no ORDER BY (SQLite runs a LIKE scan;
+      # InMemory iterates a Hash), so row order is arbitrary — scoring by
+      # index made that arbitrary order the dominant ranking signal once
+      # fed into the Ranker at semantic weight 0.40, ahead of the keyword
+      # evidence it was meant to represent.
+      #
+      # @param results_by_id [Hash<String, Hash>] id => { metadata:, matched_fields: }
+      # @return [Hash<String, Hash>] Same shape, each entry gaining +score+
+      def score_keyword_hits(results_by_id)
+        results_by_id.transform_values do |entry|
+          entry.merge(score: keyword_match_score(entry[:matched_fields]))
+        end
+      end
+
+      # Weight per matched field — mirrors {Ranker#keyword_score}'s own
+      # 0.25-per-field scale so the two stages agree on what "many fields
+      # matched" means. A hit with zero matched fields (shouldn't happen —
+      # the store only returned the record because something matched) still
+      # scores a positive floor rather than 0, keeping every keyword score
+      # in the documented (0.0, 1.0] range.
+      #
+      # @param matched_fields [Array<String>]
+      # @return [Float] Always in (0.0, 1.0]
+      def keyword_match_score(matched_fields)
+        return KEYWORD_FIELD_SCORE_WEIGHT if matched_fields.empty?
+
+        [matched_fields.size * KEYWORD_FIELD_SCORE_WEIGHT, 1.0].min
+      end
+
+      # @see #keyword_match_score
+      KEYWORD_FIELD_SCORE_WEIGHT = 0.25
+      private_constant :KEYWORD_FIELD_SCORE_WEIGHT
+
+      # Metadata record fields never counted as keyword matches: +id+
+      # duplicates +identifier+ (the store injects it), and +updated_at+
+      # is bookkeeping.
+      KEYWORD_MATCH_SKIPPED_FIELDS = %w[id updated_at].freeze
+      private_constant :KEYWORD_MATCH_SKIPPED_FIELDS
+      # Approximate which fields of a metadata record matched a keyword.
+      #
+      # The metadata store's +search+ probes fields but does not report
+      # which ones hit (SQLite runs a LIKE over the JSON blob; InMemory
+      # scans serialized records) — so per-candidate matched fields are not
+      # recoverable from the store without changing its interface. Instead,
+      # the executor re-checks the returned record: every top-level field
+      # whose stringified value contains the keyword (case-insensitively,
+      # matching SQLite's ASCII LIKE semantics) counts as matched. This is
+      # the input to {Ranker#keyword_score}'s 0.25-per-field table.
+      #
+      # @param record [Hash] Metadata record from the store (string keys)
+      # @param keyword [String] The keyword that produced this record
+      # @return [Array<String>] Names of fields whose values matched
+      def matched_fields_for(record, keyword)
+        return [] unless record.is_a?(Hash)
+
+        needle = keyword.to_s.downcase
+        return [] if needle.empty?
+
+        record.filter_map do |field, value|
+          next if KEYWORD_MATCH_SKIPPED_FIELDS.include?(field.to_s)
+
+          field.to_s if value.to_s.downcase.include?(needle)
+        end
       end
 
       # Rank merged keyword results into Candidate objects.
       #
+      # Ties (equal matched-field counts) break on identifier, ascending —
+      # deterministic output instead of depending on the store's arbitrary
+      # row order for candidates {#keyword_match_score} can't distinguish.
+      #
       # @param results [Hash<String, Hash>]
       # @param limit [Integer]
       # @return [Array<Candidate>]
       def rank_keyword_results(results, limit)
         scored = results.map do |id, data|
-          Candidate.new(identifier: id, score: data[:score], source: :keyword, metadata: data[:metadata])
+          matched = data[:matched_fields]
+          Candidate.new(identifier: id, score: data[:score], source: :keyword,
+                        metadata: data[:metadata],
+                        matched_fields: matched && !matched.empty? ? matched : nil)
         end
-        scored.sort_by { |c| -c.score }.first(limit)
+        scored.sort_by { |c| [-c.score, c.identifier.to_s] }.first(limit)
       end
 
       # Graph strategy: find related units via dependency traversal.
@@ -206,13 +327,13 @@ module Woods
 
         seeds.each do |seed_id|
           # Forward dependencies
-          deps = @graph_store.dependencies_of(seed_id)
+          deps = @graph_store.dependencies_of(StorageIdentity.identifier(seed_id))
           deps.each do |dep|
             candidates << Candidate.new(identifier: dep, score: 0.8, source: :graph, metadata: {})
           end
 
           # Reverse dependencies (dependents)
-          dependents = @graph_store.dependents_of(seed_id)
+          dependents = @graph_store.dependents_of(StorageIdentity.identifier(seed_id))
           dependents.each do |dep|
             candidates << Candidate.new(identifier: dep, score: 0.7, source: :graph, metadata: {})
           end
@@ -224,19 +345,39 @@ module Woods
         deduplicate(candidates).first(limit)
       end
 
+      # How far past +limit+ to fetch from each source before handing the
+      # merged list to the Ranker's RRF fusion (#apply_rrf). A candidate
+      # found by two sources near either source's own cutoff needs BOTH
+      # hits to survive into the merged list for RRF to see them — fetching
+      # only +limit+ per source (the old behavior) starved that overlap.
+      # Bounded (not unbounded) so a large index doesn't turn one hybrid
+      # query into three full-index scans.
+      HYBRID_PER_SOURCE_LIMIT_MULTIPLIER = 3
+      private_constant :HYBRID_PER_SOURCE_LIMIT_MULTIPLIER
+
       # Hybrid strategy: combine vector, keyword, and graph expansion.
       #
+      # Deliberately does NOT deduplicate across sources — a candidate
+      # found by both vector and keyword search must reach the Ranker as
+      # two separate entries (one per source) so RRF (Ranker#apply_rrf)
+      # can accumulate their per-source rank-based scores. Collapsing to
+      # one entry per identifier here, before ranking, discarded that
+      # cross-source consensus entirely: "hybrid" degraded into plain
+      # concatenation. Deduplication now happens exactly once, inside
+      # RRF's merge, which is where cross-source agreement is computed.
+      #
       # @return [Array<Candidate>]
       def execute_hybrid(query, classification:, limit:, type_filter: nil)
+        per_source_limit = limit * HYBRID_PER_SOURCE_LIMIT_MULTIPLIER
+
         # Gather from all three sources
-        vector_candidates = execute_vector(query, classification: classification, limit: limit,
-                                                  type_filter: type_filter)
-        keyword_candidates = execute_keyword(classification: classification, limit: limit)
+        vector_candidates = execute_vector(query, limit: per_source_limit, type_filter: type_filter)
+        keyword_candidates = execute_keyword(classification: classification, limit: per_source_limit)
 
         # Graph expansion on top vector results
         graph_candidates = []
         vector_candidates.first(3).each do |candidate|
-          deps = @graph_store.dependencies_of(candidate.identifier)
+          deps = @graph_store.dependencies_of(StorageIdentity.identifier(candidate.identifier.sub(/#chunk_\d+\z/, '')))
           deps.each do |dep|
             graph_candidates << Candidate.new(
               identifier: dep, score: 0.5, source: :graph_expansion, metadata: {}
@@ -244,8 +385,10 @@ module Woods
           end
         end
 
-        all = vector_candidates + keyword_candidates + graph_candidates
-        deduplicate(all).first(limit)
+        # Sorted (not deduplicated) so the caller's later `.first(limit)`
+        # keeps the strongest candidates across all sources instead of
+        # exhausting the budget on whichever source is concatenated first.
+        (vector_candidates + keyword_candidates + graph_candidates).sort_by { |c| -c.score }
       end
 
       # Direct strategy: look up specific identifiers from keywords.
@@ -285,24 +428,25 @@ module Woods
         candidates
       end
 
-      # Build metadata filters for vector search based on classification
-      # and an optional explicit type filter from the caller.
+      # Build metadata filters for vector search from the caller's
+      # explicit type filter.
       #
-      # The caller's explicit +type_filter+ overrides classifier-derived
-      # +target_type+ when both are present — the caller opted into a
-      # specific set of types and that intent beats a heuristic.
+      # The classifier-derived +target_type+ is deliberately NOT pushed
+      # down here (#184). It is a heuristic, and hard-filtering on it had
+      # no fallback — a mainline query like "How do we get the current
+      # user?" classified target :route and the vector search excluded
+      # everything but route units. The heuristic still reaches the
+      # {Ranker} as the soft +type_match+ signal, which boosts matching
+      # types without excluding anything. A caller-supplied +type_filter+
+      # keeps hard-filter semantics: the caller opted into specific types,
+      # and {Retriever#retrieve} owns the within-type fallback for that path.
       #
-      # @param classification [QueryClassifier::Classification]
       # @param type_filter [Array<String>, nil]
       # @return [Hash]
-      def build_vector_filters(classification, type_filter: nil)
-        filters = {}
-        if type_filter && !type_filter.empty?
-          filters[:type] = type_filter.map(&:to_s)
-        elsif classification.target_type
-          filters[:type] = classification.target_type.to_s
-        end
-        filters
+      def build_vector_filters(type_filter)
+        return {} if type_filter.nil? || type_filter.empty?
+
+        { type: type_filter.map(&:to_s) }
       end
 
       # Find seed identifiers from classification keywords via metadata search.
@@ -340,6 +484,29 @@ module Woods
         end
         best.values.sort_by { |c| -c.score }
       end
+
+      # Chunk-stripped unit identity, matching Ranker#base_identifier —
+      # hybrid truncation must treat `User#chunk_0` and `User` as one unit.
+      CHUNK_SUFFIX_PATTERN = /#chunk_\d+\z/
+      private_constant :CHUNK_SUFFIX_PATTERN
+
+      # Bound strategy output to +limit+ results. For hybrid, +limit+ counts
+      # UNIQUE units and cross-source duplicates of an already-kept unit ride
+      # along free: the Ranker's RRF consensus needs every source's entry for
+      # a unit, and a raw-score cut dropped the weaker source's duplicate
+      # exactly when the two sources' score scales disagreed.
+      def bounded_candidates(candidates, limit, strategy)
+        return candidates.first(limit) unless strategy == :hybrid
+
+        kept = {}
+        candidates.select do |candidate|
+          base = candidate.identifier.sub(CHUNK_SUFFIX_PATTERN, '')
+          next true if kept.key?(base)
+
+          kept[base] = true if kept.size < limit
+          kept.key?(base)
+        end
+      end
     end
   end
 end