woods 2.0.0.beta2 → 2.0.0.beta4

data/lib/woods/retrieval/lexical_index.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'query_classifier'
+require_relative 'search_executor'
+
+module Woods
+  module Retrieval
+    # Immutable field-aware lexical snapshot. Only published unit values enter
+    # the vocabulary; bookkeeping and arbitrary JSON keys cannot become hits.
+    class LexicalIndex
+      FIELD_WEIGHTS = { 'identifier' => 4.0, 'file_path' => 2.0, 'source_code' => 1.0, 'runtime' => 2.0 }.freeze
+      RUNTIME_FIELDS = %w[callbacks associations validations scopes concerns included_modules
+                          methods instance_methods class_methods actions routes columns table_name
+                          description purpose dependencies].freeze
+      DEFAULT_LIMIT = 20
+      K1 = 1.2
+      B = 0.75
+      Document = Struct.new(:key, :unit, :fields, keyword_init: true)
+      private_constant :Document
+
+      def initialize(metadata_store:)
+        @documents = metadata_store.all_identifiers.sort.map do |key|
+          unit = metadata_store.find(key)
+          raise ArgumentError, "missing unit metadata for #{key.inspect}" unless unit.is_a?(Hash)
+
+          unit = JSON.parse(JSON.generate(unit))
+          fields = field_values(unit).transform_values { |value| tokenize(value).tally.freeze }.freeze
+          Document.new(key: key.freeze, unit: deep_freeze(unit), fields: fields).freeze
+        end.compact.freeze
+        @averages = FIELD_WEIGHTS.to_h do |field, _|
+          lengths = @documents.map { |doc| doc.fields.fetch(field).values.sum }
+          [field, lengths.empty? ? 1.0 : [lengths.sum.fdiv(lengths.size), 1.0].max]
+        end.freeze
+        @frequencies = Hash.new(0)
+        @documents.each { |doc| doc.fields.values.flat_map(&:keys).uniq.each { |term| @frequencies[term] += 1 } }
+        @frequencies.freeze
+      end
+
+      def execute(query:, limit: DEFAULT_LIMIT, type_filter: nil, exclude_types: nil)
+        terms = tokenize(query).uniq
+        candidates = @documents.filter_map do |doc|
+          next unless eligible?(doc.unit, type_filter, exclude_types)
+
+          score, fields = score_document(doc, terms)
+          exact = doc.unit['identifier'].to_s.casecmp?(query.strip)
+          score += 1.0 if exact
+          fields << 'identifier:exact' if exact
+          next unless score.positive?
+
+          SearchExecutor::Candidate.new(identifier: doc.key, score: score, source: :lexical,
+                                        metadata: doc.unit, matched_fields: fields.sort)
+        end
+        candidates.sort_by! do |candidate|
+          [candidate.matched_fields.include?('identifier:exact') ? 0 : 1, -candidate.score, candidate.identifier]
+        end
+        SearchExecutor::ExecutionResult.new(candidates: candidates.first(limit), strategy: :lexical, query: query)
+      end
+
+      private
+
+      def tokenize(value)
+        value.to_s.gsub(/(\p{Ll}|\d)(\p{Lu})/u, '\1 \2')
+             .gsub(/(\p{Lu})(\p{Lu}\p{Ll})/u, '\1 \2').downcase
+             .scan(/[\p{L}\p{N}]+/u).reject { |term| QueryClassifier::STOP_WORDS.include?(term) }
+      end
+
+      def field_values(unit)
+        metadata = unit['metadata'].is_a?(Hash) ? unit['metadata'] : {}
+        runtime = RUNTIME_FIELDS.filter_map { |key| metadata[key] || unit[key] }
+        { 'identifier' => unit['identifier'], 'file_path' => unit['file_path'],
+          'source_code' => unit['source_code'], 'runtime' => values_text(runtime) }
+      end
+
+      def values_text(value)
+        case value
+        when Hash then value.values.map { |child| values_text(child) }.join(' ')
+        when Array then value.map { |child| values_text(child) }.join(' ')
+        when String, Symbol, Numeric then value.to_s
+        else ''
+        end
+      end
+
+      def eligible?(unit, allowed, excluded)
+        return allowed.map(&:to_s).include?(unit['type']) if allowed && !allowed.empty?
+
+        !Array(excluded).map(&:to_s).include?(unit['type'])
+      end
+
+      def score_document(doc, terms)
+        fields = []
+        score = FIELD_WEIGHTS.sum do |field, weight|
+          counts = doc.fields.fetch(field)
+          normalization = K1 * (1 - B + (B * counts.values.sum / @averages.fetch(field)))
+          terms.sum do |term|
+            frequency = counts.fetch(term, 0)
+            next 0.0 if frequency.zero?
+
+            fields << "#{field}:#{term}"
+            document_frequency = @frequencies.fetch(term)
+            idf = Math.log(1 + ((@documents.size - document_frequency + 0.5) / (document_frequency + 0.5)))
+            weight * idf * frequency * (K1 + 1) / (frequency + normalization)
+          end
+        end
+        [score, fields]
+      end
+
+      def deep_freeze(value)
+        case value
+        when Hash then value.each do |key, child|
+          key.freeze
+          deep_freeze(child)
+        end
+        when Array then value.each { |child| deep_freeze(child) }
+        end
+        value.freeze
+      end
+    end
+  end
+end

data/lib/woods/retrieval/ranker.rb

@@ -378,7 +378,9 @@ module Woods
 
       # Lazily-computed rank-percentile map derived from the graph store's PageRank.
       #
-      # Top-ranked identifier gets 1.0, bottom-ranked gets 1/n. Identifiers absent
+      # Top-ranked identifier gets 1.0, bottom-ranked gets 1/n. Equal PageRank scores
+      # are ordered lexically by identifier so their ordinal percentiles do not
+      # depend on graph insertion order or Ruby's unstable sort. Identifiers absent
       # from PageRank (new units, ephemeral candidates) return nil and fall back
       # to the bucketed importance signal.
       #
@@ -402,7 +404,7 @@ module Woods
         scores = @graph_store.pagerank
         return {} if scores.nil? || scores.empty?
 
-        ranked = scores.sort_by { |_id, score| -score }
+        ranked = scores.sort_by { |identifier, score| [-score, identifier] }
         total = ranked.size.to_f
         ranked.each_with_index.to_h do |(identifier, _score), rank|
           [identifier, 1.0 - (rank / total)]

data/lib/woods/retrieval/scope.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'set'
+require_relative '../storage_identity'
+require_relative '../storage/metadata_store'
+
+module Woods
+  module Retrieval
+    # Resolves explicit scope from published metadata, never host filesystem
+    # paths. Ownership is exact; path prefixes match complete directory segments.
+    # Every eligible unit is selected before any search strategy applies a limit.
+    class Scope
+      class InvalidScopeError < ArgumentError; end
+
+      attr_reader :packages, :source_paths, :keys, :metadata_store
+
+      def self.requested?(packages: nil, source_paths: nil)
+        [packages, source_paths].any? { |list| !list.nil? && list != [] }
+      end
+
+      def initialize(metadata_store:, packages: nil, source_paths: nil, types: nil, exclude_types: nil)
+        @packages = normalize_list(packages, 'packages').freeze
+        @source_paths = normalize_list(source_paths, 'source_paths').map do |path|
+          normalize_path(path)
+        end.uniq.sort.freeze
+        records = metadata_store.all_identifiers.sort.to_h do |key|
+          record = metadata_store.find(key)
+          raise InvalidScopeError, "missing metadata for scoped unit #{key.inspect}" unless record.is_a?(Hash)
+
+          [key, JSON.parse(JSON.generate(record))]
+        end
+        validate_packages!(records.values)
+        @metadata_store = Storage::MetadataStore::InMemory.new
+        records.each do |key, record|
+          next unless eligible?(record, types, exclude_types)
+
+          @metadata_store.store(key, record)
+        end
+        @keys = @metadata_store.all_identifiers.sort.freeze
+        @key_set = @keys.to_set.freeze
+      end
+
+      def include?(key)
+        @key_set.include?(key.to_s.sub(/#chunk_\d+\z/, ''))
+      end
+
+      def summary
+        { packages: packages, source_paths: source_paths, eligible_units: keys.size }
+      end
+
+      private
+
+      def normalize_list(list, name)
+        return [] if list.nil?
+        unless list.is_a?(Array) && list.all? { |value| value.is_a?(String) && !value.empty? && !value.include?("\0") }
+          raise InvalidScopeError, "#{name} must be an array of nonempty strings"
+        end
+
+        list.uniq.sort.map { |value| value.dup.freeze }
+      end
+
+      def normalize_path(path)
+        if path.start_with?('/') || path.match?(/\A[A-Za-z]:/) || path.include?('\\')
+          raise InvalidScopeError, "source path must be application-relative: #{path.inspect}"
+        end
+
+        segments = []
+        path.split('/').each do |part|
+          next if part.empty? || part == '.'
+
+          if part == '..'
+            raise InvalidScopeError, "source path escapes application root: #{path.inspect}" if segments.empty?
+
+            segments.pop
+          else
+            segments << part
+          end
+        end
+        (segments.empty? ? '.' : segments.join('/')).freeze
+      end
+
+      def validate_packages!(records)
+        names = records.filter_map { |unit| unit['identifier'] if unit['type'] == 'package' }
+        names.concat(records.filter_map { |unit| unit.dig('metadata', 'package') })
+        unknown = packages - names
+        raise InvalidScopeError, "unknown package scope: #{unknown.join(', ')}" unless unknown.empty?
+      end
+
+      def eligible?(unit, types, excluded)
+        return false unless packages.empty? || packages.include?(unit.dig('metadata', 'package'))
+        return false unless source_paths.empty? || path_match?(unit['file_path'])
+        return Array(types).map(&:to_s).include?(unit['type']) if types && !types.empty?
+
+        !Array(excluded).map(&:to_s).include?(unit['type'])
+      end
+
+      def path_match?(path)
+        return false unless path.is_a?(String) && !path.empty? && !path.include?("\0")
+
+        normalized = normalize_path(path)
+        source_paths.any? { |prefix| prefix == '.' || normalized == prefix || normalized.start_with?("#{prefix}/") }
+      rescue InvalidScopeError
+        false
+      end
+    end
+  end
+end

data/lib/woods/retrieval/scoped_graph_store.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Expansion remains inside caller eligibility. Graphs retain their existing
+    # bare-identifier ambiguity: the executor resolves eligible typed variants.
+    class ScopedGraphStore
+      def initialize(store:, scope:)
+        @store = store
+        @identifiers = scope.keys.to_set { |key| StorageIdentity.identifier(key) }
+      end
+
+      def dependencies_of(identifier)
+        return [] unless @store && @identifiers.include?(identifier)
+
+        @store.dependencies_of(identifier).select { |target| @identifiers.include?(target) }
+      end
+
+      def dependents_of(identifier)
+        return [] unless @store && @identifiers.include?(identifier)
+
+        @store.dependents_of(identifier).select { |target| @identifiers.include?(target) }
+      end
+
+      def pagerank
+        return {} unless @store
+
+        @store.pagerank.slice(*@identifiers)
+      end
+    end
+  end
+end

data/lib/woods/retrieval/scoped_vector_store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Complete, bounded native searches across eligible raw vector IDs. Raw IDs
+    # preserve typed/chunk identity without requiring a new vector payload or an
+    # embedding migration. One outer executor embeds the query once.
+    class ScopedVectorStore
+      BATCH_SIZE = 100
+
+      def initialize(store:, scope:)
+        @store = store
+        @scope = scope
+      end
+
+      def search(query_vector, limit: 10, filters: {})
+        unless @store.respond_to?(:supports_id_filter?) && @store.supports_id_filter?
+          raise Scope::InvalidScopeError, 'vector adapter does not support complete ID-scoped search'
+        end
+
+        ids = @store.each_id.select { |id| eligible_id?(id, filters) }.uniq.sort
+        # Type eligibility comes from authoritative unit metadata, including for
+        # legacy vectors without a type payload. Other payload filters stay native.
+        filters = filters.reject { |key, _| key.to_s == 'type' }
+        best = []
+        ids.each_slice(BATCH_SIZE) do |batch|
+          # Fetch every bounded batch member before our raw-ID tie-break. A
+          # backend may order equal scores differently (or use SQL collation).
+          results = @store.search(query_vector, limit: batch.size, filters: filters, ids: batch)
+          results.each do |result|
+            next if batch.include?(result.id)
+
+            raise Scope::InvalidScopeError, 'vector adapter returned an ID outside the requested scope'
+          end
+          best = (best + results).sort_by { |result| [-result.score, result.id] }.first(limit)
+        end
+        best
+      rescue NotImplementedError
+        raise Scope::InvalidScopeError, 'vector adapter cannot enumerate IDs for complete scoped search'
+      end
+
+      private
+
+      def eligible_id?(id, filters)
+        return false unless @scope.include?(id)
+
+        types = filters[:type] || filters['type']
+        return true unless types
+
+        record = @scope.metadata_store.find(id.sub(/#chunk_\d+\z/, ''))
+        Array(types).map(&:to_s).include?(record['type'])
+      end
+    end
+  end
+end

data/lib/woods/retrieval/search_executor.rb

@@ -86,8 +86,11 @@ module Woods
           type_filter: type_filter
         )
 
+        candidates = expand_graph_candidates(candidates)
+        candidates = order_hybrid_candidates(candidates) if strategy == :hybrid
+
         ExecutionResult.new(
-          candidates: bounded_candidates(expand_graph_candidates(candidates), limit, strategy),
+          candidates: bounded_candidates(candidates, limit, strategy),
           strategy: strategy,
           query: query
         )
@@ -155,7 +158,7 @@ module Woods
         when :keyword
           execute_keyword(classification: classification, limit: limit)
         when :graph
-          execute_graph(classification: classification, limit: limit)
+          execute_graph(query: query, limit: limit)
         when :hybrid
           execute_hybrid(query, classification: classification, limit: limit, type_filter: type_filter)
         when :direct
@@ -318,11 +321,11 @@ module Woods
       # Graph strategy: find related units via dependency traversal.
       #
       # @return [Array<Candidate>]
-      def execute_graph(classification:, limit:)
+      def execute_graph(query:, limit:)
         candidates = []
 
-        # First, use keywords to find seed identifiers in the metadata store
-        seeds = find_seed_identifiers(classification)
+        # Resolve original query subjects to canonical metadata identifiers
+        seeds = find_seed_identifiers(query)
         return [] if seeds.empty?
 
         seeds.each do |seed_id|
@@ -376,7 +379,7 @@ module Woods
 
         # Graph expansion on top vector results
         graph_candidates = []
-        vector_candidates.first(3).each do |candidate|
+        order_hybrid_candidates(vector_candidates).first(3).each do |candidate|
           deps = @graph_store.dependencies_of(StorageIdentity.identifier(candidate.identifier.sub(/#chunk_\d+\z/, '')))
           deps.each do |dep|
             graph_candidates << Candidate.new(
@@ -385,10 +388,16 @@ module Woods
           end
         end
 
-        # Sorted (not deduplicated) so the caller's later `.first(limit)`
-        # keeps the strongest candidates across all sources instead of
-        # exhausting the budget on whichever source is concatenated first.
-        (vector_candidates + keyword_candidates + graph_candidates).sort_by { |c| -c.score }
+        # Keep every source occurrence for RRF. Ordering happens after typed
+        # graph identities expand, before the caller truncates unique units.
+        vector_candidates + keyword_candidates + graph_candidates
+      end
+
+      # Equal scores must not let insertion order or Ruby's sort implementation
+      # choose graph seeds, retained units, or a source's RRF rank. Identity is
+      # the tie-break; source makes cross-source occurrences deterministic too.
+      def order_hybrid_candidates(candidates)
+        candidates.sort_by { |candidate| [-candidate.score, candidate.identifier.to_s, candidate.source.to_s] }
       end
 
       # Direct strategy: look up specific identifiers from keywords.
@@ -449,27 +458,77 @@ module Woods
         { type: type_filter.map(&:to_s) }
       end
 
-      # Find seed identifiers from classification keywords via metadata search.
-      #
-      # @param classification [QueryClassifier::Classification]
-      # @return [Array<String>]
-      def find_seed_identifiers(classification)
-        seeds = []
-
-        # Try direct lookups for capitalized keywords (likely class names)
-        classification.keywords.each do |keyword|
-          capitalized = keyword.split('_').map(&:capitalize).join
-          result = @metadata_store.find(capitalized)
-          seeds << capitalized if result
+      # Keep graph subjects separate from intent words and preserve namespace
+      # separators/case that the general-purpose classifier keywords discard.
+      GRAPH_INTENT_WORDS = %w[trace follow track call calls called depends used find locate search look].freeze
+      private_constant :GRAPH_INTENT_WORDS
+
+      def find_seed_identifiers(query)
+        records = graph_query_records(query)
+        exact = records.flat_map do |term, matches|
+          graph_identifier_matches(matches, term)
+        end
+        return exact.map { |record| record['id'] }.uniq.sort unless exact.empty?
+
+        # A qualified miss must not seed another namespace through source-text
+        # mentions. Unqualified prose can still fall back to subject metadata.
+        records.reject { |term, _| term.include?('::') }.values.flatten
+               .map { |record| record['id'] }.uniq.sort.first(3)
+      end
+
+      # Sentence-capitalized instructions are still prose. Keep an instruction
+      # token only when it is also an explicitly spelled, existing identifier.
+      def graph_query_records(query)
+        query.scan(/\w+(?:::\w+)*/).uniq.filter_map do |term|
+          instruction = QueryClassifier::STOP_WORDS.include?(term.downcase) ||
+                        GRAPH_INTENT_WORDS.include?(term.downcase)
+          next if instruction && !term.match?(/[A-Z]/)
+
+          records = graph_subject_records(term)
+          records = records.select { |record| graph_record_identifier(record) == term } if instruction
+          [term, records]
+        end.to_h
+      end
+
+      # Prefer the spelling supplied by the caller before inferring snake_case
+      # variants. Metadata search recovers canonical and typed storage identities.
+      def graph_subject_records(term)
+        direct = @metadata_store.find(term)
+        return [direct.merge('id' => term)] if direct
+
+        records = @metadata_store.search(term)
+        exact = records.select { |record| graph_record_identifier(record) == term }
+        return exact unless exact.empty?
+
+        camelized = term.split('::').map { |part| part.split('_').map(&:capitalize).join }.join('::')
+        return records if camelized == term
+
+        inferred = @metadata_store.find(camelized)
+        return [inferred.merge('id' => camelized)] if inferred
+
+        (records + @metadata_store.search(camelized)).uniq { |record| record['id'] }
+      end
+
+      # Exact qualified identity wins over a bounded, deterministic set of
+      # unqualified namespace suffix matches; source-text mentions are not roots.
+      def graph_identifier_matches(records, term)
+        literal = records.select { |record| graph_record_identifier(record) == term }
+        return literal unless literal.empty?
+
+        exact = records.select do |record|
+          graph_record_identifier(record).delete('_').casecmp?(term.delete('_'))
         end
+        return exact unless exact.empty?
+        return [] if term.include?('::')
 
-        # Fall back to search if no direct hits
-        if seeds.empty? && classification.keywords.any?
-          results = @metadata_store.search(classification.keywords.join(' '))
-          seeds = results.first(3).map { |r| r['id'] }
+        suffix_matches = records.select do |record|
+          graph_record_identifier(record).split('::').last.delete('_').casecmp?(term.delete('_'))
         end
+        suffix_matches.sort_by { |record| record['id'] }.first(3)
+      end
 
-        seeds
+      def graph_record_identifier(record)
+        record['identifier'] || StorageIdentity.identifier(record['id'])
       end
 
       # Deduplicate candidates, keeping the highest-scored entry per identifier.