woods 2.0.0.beta2 → 2.0.0.beta3

data/lib/woods/retrieval/scope.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'set'
+require_relative '../storage_identity'
+require_relative '../storage/metadata_store'
+
+module Woods
+  module Retrieval
+    # Resolves explicit scope from published metadata, never host filesystem
+    # paths. Ownership is exact; path prefixes match complete directory segments.
+    # Every eligible unit is selected before any search strategy applies a limit.
+    class Scope
+      class InvalidScopeError < ArgumentError; end
+
+      attr_reader :packages, :source_paths, :keys, :metadata_store
+
+      def self.requested?(packages: nil, source_paths: nil)
+        [packages, source_paths].any? { |list| !list.nil? && list != [] }
+      end
+
+      def initialize(metadata_store:, packages: nil, source_paths: nil, types: nil, exclude_types: nil)
+        @packages = normalize_list(packages, 'packages').freeze
+        @source_paths = normalize_list(source_paths, 'source_paths').map do |path|
+          normalize_path(path)
+        end.uniq.sort.freeze
+        records = metadata_store.all_identifiers.sort.to_h do |key|
+          record = metadata_store.find(key)
+          raise InvalidScopeError, "missing metadata for scoped unit #{key.inspect}" unless record.is_a?(Hash)
+
+          [key, JSON.parse(JSON.generate(record))]
+        end
+        validate_packages!(records.values)
+        @metadata_store = Storage::MetadataStore::InMemory.new
+        records.each do |key, record|
+          next unless eligible?(record, types, exclude_types)
+
+          @metadata_store.store(key, record)
+        end
+        @keys = @metadata_store.all_identifiers.sort.freeze
+        @key_set = @keys.to_set.freeze
+      end
+
+      def include?(key)
+        @key_set.include?(key.to_s.sub(/#chunk_\d+\z/, ''))
+      end
+
+      def summary
+        { packages: packages, source_paths: source_paths, eligible_units: keys.size }
+      end
+
+      private
+
+      def normalize_list(list, name)
+        return [] if list.nil?
+        unless list.is_a?(Array) && list.all? { |value| value.is_a?(String) && !value.empty? && !value.include?("\0") }
+          raise InvalidScopeError, "#{name} must be an array of nonempty strings"
+        end
+
+        list.uniq.sort.map { |value| value.dup.freeze }
+      end
+
+      def normalize_path(path)
+        if path.start_with?('/') || path.match?(/\A[A-Za-z]:/) || path.include?('\\')
+          raise InvalidScopeError, "source path must be application-relative: #{path.inspect}"
+        end
+
+        segments = []
+        path.split('/').each do |part|
+          next if part.empty? || part == '.'
+
+          if part == '..'
+            raise InvalidScopeError, "source path escapes application root: #{path.inspect}" if segments.empty?
+
+            segments.pop
+          else
+            segments << part
+          end
+        end
+        (segments.empty? ? '.' : segments.join('/')).freeze
+      end
+
+      def validate_packages!(records)
+        names = records.filter_map { |unit| unit['identifier'] if unit['type'] == 'package' }
+        names.concat(records.filter_map { |unit| unit.dig('metadata', 'package') })
+        unknown = packages - names
+        raise InvalidScopeError, "unknown package scope: #{unknown.join(', ')}" unless unknown.empty?
+      end
+
+      def eligible?(unit, types, excluded)
+        return false unless packages.empty? || packages.include?(unit.dig('metadata', 'package'))
+        return false unless source_paths.empty? || path_match?(unit['file_path'])
+        return Array(types).map(&:to_s).include?(unit['type']) if types && !types.empty?
+
+        !Array(excluded).map(&:to_s).include?(unit['type'])
+      end
+
+      def path_match?(path)
+        return false unless path.is_a?(String) && !path.empty? && !path.include?("\0")
+
+        normalized = normalize_path(path)
+        source_paths.any? { |prefix| prefix == '.' || normalized == prefix || normalized.start_with?("#{prefix}/") }
+      rescue InvalidScopeError
+        false
+      end
+    end
+  end
+end

data/lib/woods/retrieval/scoped_graph_store.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Expansion remains inside caller eligibility. Graphs retain their existing
+    # bare-identifier ambiguity: the executor resolves eligible typed variants.
+    class ScopedGraphStore
+      def initialize(store:, scope:)
+        @store = store
+        @identifiers = scope.keys.to_set { |key| StorageIdentity.identifier(key) }
+      end
+
+      def dependencies_of(identifier)
+        return [] unless @store && @identifiers.include?(identifier)
+
+        @store.dependencies_of(identifier).select { |target| @identifiers.include?(target) }
+      end
+
+      def dependents_of(identifier)
+        return [] unless @store && @identifiers.include?(identifier)
+
+        @store.dependents_of(identifier).select { |target| @identifiers.include?(target) }
+      end
+
+      def pagerank
+        return {} unless @store
+
+        @store.pagerank.slice(*@identifiers)
+      end
+    end
+  end
+end

data/lib/woods/retrieval/scoped_vector_store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Complete, bounded native searches across eligible raw vector IDs. Raw IDs
+    # preserve typed/chunk identity without requiring a new vector payload or an
+    # embedding migration. One outer executor embeds the query once.
+    class ScopedVectorStore
+      BATCH_SIZE = 100
+
+      def initialize(store:, scope:)
+        @store = store
+        @scope = scope
+      end
+
+      def search(query_vector, limit: 10, filters: {})
+        unless @store.respond_to?(:supports_id_filter?) && @store.supports_id_filter?
+          raise Scope::InvalidScopeError, 'vector adapter does not support complete ID-scoped search'
+        end
+
+        ids = @store.each_id.select { |id| eligible_id?(id, filters) }.uniq.sort
+        # Type eligibility comes from authoritative unit metadata, including for
+        # legacy vectors without a type payload. Other payload filters stay native.
+        filters = filters.reject { |key, _| key.to_s == 'type' }
+        best = []
+        ids.each_slice(BATCH_SIZE) do |batch|
+          # Fetch every bounded batch member before our raw-ID tie-break. A
+          # backend may order equal scores differently (or use SQL collation).
+          results = @store.search(query_vector, limit: batch.size, filters: filters, ids: batch)
+          results.each do |result|
+            next if batch.include?(result.id)
+
+            raise Scope::InvalidScopeError, 'vector adapter returned an ID outside the requested scope'
+          end
+          best = (best + results).sort_by { |result| [-result.score, result.id] }.first(limit)
+        end
+        best
+      rescue NotImplementedError
+        raise Scope::InvalidScopeError, 'vector adapter cannot enumerate IDs for complete scoped search'
+      end
+
+      private
+
+      def eligible_id?(id, filters)
+        return false unless @scope.include?(id)
+
+        types = filters[:type] || filters['type']
+        return true unless types
+
+        record = @scope.metadata_store.find(id.sub(/#chunk_\d+\z/, ''))
+        Array(types).map(&:to_s).include?(record['type'])
+      end
+    end
+  end
+end

data/lib/woods/retrieval/search_executor.rb

@@ -86,8 +86,11 @@ module Woods
           type_filter: type_filter
         )
 
+        candidates = expand_graph_candidates(candidates)
+        candidates = order_hybrid_candidates(candidates) if strategy == :hybrid
+
         ExecutionResult.new(
-          candidates: bounded_candidates(expand_graph_candidates(candidates), limit, strategy),
+          candidates: bounded_candidates(candidates, limit, strategy),
           strategy: strategy,
           query: query
         )
@@ -155,7 +158,7 @@ module Woods
         when :keyword
           execute_keyword(classification: classification, limit: limit)
         when :graph
-          execute_graph(classification: classification, limit: limit)
+          execute_graph(query: query, limit: limit)
         when :hybrid
           execute_hybrid(query, classification: classification, limit: limit, type_filter: type_filter)
         when :direct
@@ -318,11 +321,11 @@ module Woods
       # Graph strategy: find related units via dependency traversal.
       #
       # @return [Array<Candidate>]
-      def execute_graph(classification:, limit:)
+      def execute_graph(query:, limit:)
         candidates = []
 
-        # First, use keywords to find seed identifiers in the metadata store
-        seeds = find_seed_identifiers(classification)
+        # Resolve original query subjects to canonical metadata identifiers
+        seeds = find_seed_identifiers(query)
         return [] if seeds.empty?
 
         seeds.each do |seed_id|
@@ -376,7 +379,7 @@ module Woods
 
         # Graph expansion on top vector results
         graph_candidates = []
-        vector_candidates.first(3).each do |candidate|
+        order_hybrid_candidates(vector_candidates).first(3).each do |candidate|
           deps = @graph_store.dependencies_of(StorageIdentity.identifier(candidate.identifier.sub(/#chunk_\d+\z/, '')))
           deps.each do |dep|
             graph_candidates << Candidate.new(
@@ -385,10 +388,16 @@ module Woods
           end
         end
 
-        # Sorted (not deduplicated) so the caller's later `.first(limit)`
-        # keeps the strongest candidates across all sources instead of
-        # exhausting the budget on whichever source is concatenated first.
-        (vector_candidates + keyword_candidates + graph_candidates).sort_by { |c| -c.score }
+        # Keep every source occurrence for RRF. Ordering happens after typed
+        # graph identities expand, before the caller truncates unique units.
+        vector_candidates + keyword_candidates + graph_candidates
+      end
+
+      # Equal scores must not let insertion order or Ruby's sort implementation
+      # choose graph seeds, retained units, or a source's RRF rank. Identity is
+      # the tie-break; source makes cross-source occurrences deterministic too.
+      def order_hybrid_candidates(candidates)
+        candidates.sort_by { |candidate| [-candidate.score, candidate.identifier.to_s, candidate.source.to_s] }
       end
 
       # Direct strategy: look up specific identifiers from keywords.
@@ -449,27 +458,77 @@ module Woods
         { type: type_filter.map(&:to_s) }
       end
 
-      # Find seed identifiers from classification keywords via metadata search.
-      #
-      # @param classification [QueryClassifier::Classification]
-      # @return [Array<String>]
-      def find_seed_identifiers(classification)
-        seeds = []
-
-        # Try direct lookups for capitalized keywords (likely class names)
-        classification.keywords.each do |keyword|
-          capitalized = keyword.split('_').map(&:capitalize).join
-          result = @metadata_store.find(capitalized)
-          seeds << capitalized if result
+      # Keep graph subjects separate from intent words and preserve namespace
+      # separators/case that the general-purpose classifier keywords discard.
+      GRAPH_INTENT_WORDS = %w[trace follow track call calls called depends used find locate search look].freeze
+      private_constant :GRAPH_INTENT_WORDS
+
+      def find_seed_identifiers(query)
+        records = graph_query_records(query)
+        exact = records.flat_map do |term, matches|
+          graph_identifier_matches(matches, term)
+        end
+        return exact.map { |record| record['id'] }.uniq.sort unless exact.empty?
+
+        # A qualified miss must not seed another namespace through source-text
+        # mentions. Unqualified prose can still fall back to subject metadata.
+        records.reject { |term, _| term.include?('::') }.values.flatten
+               .map { |record| record['id'] }.uniq.sort.first(3)
+      end
+
+      # Sentence-capitalized instructions are still prose. Keep an instruction
+      # token only when it is also an explicitly spelled, existing identifier.
+      def graph_query_records(query)
+        query.scan(/\w+(?:::\w+)*/).uniq.filter_map do |term|
+          instruction = QueryClassifier::STOP_WORDS.include?(term.downcase) ||
+                        GRAPH_INTENT_WORDS.include?(term.downcase)
+          next if instruction && !term.match?(/[A-Z]/)
+
+          records = graph_subject_records(term)
+          records = records.select { |record| graph_record_identifier(record) == term } if instruction
+          [term, records]
+        end.to_h
+      end
+
+      # Prefer the spelling supplied by the caller before inferring snake_case
+      # variants. Metadata search recovers canonical and typed storage identities.
+      def graph_subject_records(term)
+        direct = @metadata_store.find(term)
+        return [direct.merge('id' => term)] if direct
+
+        records = @metadata_store.search(term)
+        exact = records.select { |record| graph_record_identifier(record) == term }
+        return exact unless exact.empty?
+
+        camelized = term.split('::').map { |part| part.split('_').map(&:capitalize).join }.join('::')
+        return records if camelized == term
+
+        inferred = @metadata_store.find(camelized)
+        return [inferred.merge('id' => camelized)] if inferred
+
+        (records + @metadata_store.search(camelized)).uniq { |record| record['id'] }
+      end
+
+      # Exact qualified identity wins over a bounded, deterministic set of
+      # unqualified namespace suffix matches; source-text mentions are not roots.
+      def graph_identifier_matches(records, term)
+        literal = records.select { |record| graph_record_identifier(record) == term }
+        return literal unless literal.empty?
+
+        exact = records.select do |record|
+          graph_record_identifier(record).delete('_').casecmp?(term.delete('_'))
         end
+        return exact unless exact.empty?
+        return [] if term.include?('::')
 
-        # Fall back to search if no direct hits
-        if seeds.empty? && classification.keywords.any?
-          results = @metadata_store.search(classification.keywords.join(' '))
-          seeds = results.first(3).map { |r| r['id'] }
+        suffix_matches = records.select do |record|
+          graph_record_identifier(record).split('::').last.delete('_').casecmp?(term.delete('_'))
         end
+        suffix_matches.sort_by { |record| record['id'] }.first(3)
+      end
 
-        seeds
+      def graph_record_identifier(record)
+        record['identifier'] || StorageIdentity.identifier(record['id'])
       end
 
       # Deduplicate candidates, keeping the highest-scored entry per identifier.

data/lib/woods/retrieval/source_evidence.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+require_relative '../ast/parser'
+require_relative 'lexical_index'
+
+module Woods
+  module Retrieval
+    # Selects complete spans from published unit bytes, never from host files.
+    # Published coordinates deliberately do not claim physical source locations:
+    # extractors may synthesize headers or inline commented concern bodies.
+    class SourceEvidence
+      MODES = %w[full compact outline].freeze
+      Result = Struct.new(:text, :provenance, keyword_init: true)
+      Span = Struct.new(:kind, :name, :lexical_owner, :receiver, :start_byte, :end_byte, :start_line, :end_line,
+                        keyword_init: true)
+      private_constant :Span
+
+      def self.validate_mode!(mode)
+        return mode if MODES.include?(mode)
+
+        raise ArgumentError, 'evidence must be full, compact, or outline'
+      end
+
+      def initialize(unit:, query: nil, generation: nil, parser: Ast::Parser.new)
+        @unit = unit
+        @source = field(:source_code).to_s
+        @query = terms(query)
+        @generation = generation
+        @parser = parser
+      end
+
+      # The caller's counter includes the same tokenizer/estimate used by its
+      # response budget. No selected definition or metadata value is cut in half.
+      def render(mode:, budget:, counter:)
+        self.class.validate_mode!(mode)
+        raise ArgumentError, 'budget must be a positive Integer' unless budget.is_a?(Integer) && budget.positive?
+        raise ArgumentError, 'compact evidence requires compact or outline mode' if mode == 'full'
+
+        spans = source_spans
+        selected = []
+        metadata = []
+        ordered = spans.sort_by { |span| [-relevance(span), span.start_byte] }
+        relevant, other = ordered.partition { |span| @query.empty? || relevance(span).positive? }
+        relevant.each do |span|
+          trial = selected + [span]
+          selected = trial if counter.call(format_evidence(mode, trial, metadata, spans.size)) <= budget
+        end
+        runtime_records.each do |record|
+          trial = metadata + [record]
+          metadata = trial if counter.call(format_evidence(mode, selected, trial, spans.size)) <= budget
+        end
+        other.each do |span|
+          trial = selected + [span]
+          selected = trial if counter.call(format_evidence(mode, trial, metadata, spans.size)) <= budget
+        end
+        text = format_evidence(mode, selected, metadata, spans.size)
+        return Result.new(text: '', provenance: provenance(mode, [], [], spans)) if counter.call(text) > budget
+
+        Result.new(text: text, provenance: provenance(mode, selected, metadata, spans))
+      end
+
+      private
+
+      def field(key)
+        @unit[key] || @unit[key.to_s]
+      end
+
+      def source_spans
+        return [] if @source.empty?
+
+        spans = []
+        walk(@parser.parse(@source), nil, spans)
+        return whole_source_span('whole_source_fallback') unless spans.all? { |span| complete_definition?(span) }
+
+        spans.concat(concern_spans)
+        return spans unless spans.empty?
+
+        whole_source_span('published_source')
+      rescue Woods::ExtractionError
+        # Non-Ruby or legacy synthesized source remains available as one whole
+        # published span. An unavailable boundary is never guessed with regex.
+        whole_source_span('unparsed_source')
+      end
+
+      def whole_source_span(kind)
+        [Span.new(kind: kind, name: field(:identifier), lexical_owner: nil,
+                  start_byte: 0, end_byte: @source.bytesize, start_line: 1, end_line: @source.lines.size)]
+      end
+
+      # A heredoc body may live outside a def node's syntactic range (notably
+      # endless defs and same-line `def ...; <<~DOC; end`). Refuse a broken
+      # isolated definition and fall back to the complete published source.
+      def complete_definition?(span)
+        @parser.valid_fragment?(span_text(span))
+      end
+
+      def walk(node, owner, spans, singleton_scope: false)
+        return unless node.is_a?(Ast::Node)
+
+        if %i[class module].include?(node.type)
+          singleton_scope = false
+          name = node.method_name.to_s
+          owner = name.start_with?('::') ? name.delete_prefix('::') : [owner, name].compact.join('::')
+        elsif %i[def defs].include?(node.type)
+          if node.start_byte && node.end_byte
+            kind = if singleton_scope
+                     'singleton_body_method'
+                   else
+                     (node.type == :def ? 'instance_method' : 'singleton_method')
+                   end
+            spans << Span.new(kind: kind,
+                              name: node.method_name, lexical_owner: owner, receiver: node.receiver,
+                              start_byte: node.start_byte,
+                              end_byte: node.end_byte, start_line: node.line, end_line: node.end_line)
+          end
+          return # Nested definitions/blocks are already covered by the complete outer method.
+        end
+        node.children&.each do |child|
+          walk(child, owner, spans, singleton_scope: singleton_scope || node.type == :sclass)
+        end
+      end
+
+      # These are Woods' published display blocks, not physical concern source.
+      # Require matching published metadata and delimiters; preserve every byte,
+      # including comment prefixes, instead of reconstructing executable Ruby.
+      def concern_spans
+        metadata = field(:metadata) || {}
+        Array(metadata['inlined_concerns'] || metadata[:inlined_concerns]).filter_map do |name|
+          escaped = Regexp.escape(name.to_s)
+          pattern = /^# │ Included from: #{escaped}[^\S\r\n]*│\r?\n.*?^# ─+ End #{escaped} ─+\r?$/m
+          match = pattern.match(@source)
+          next unless match
+
+          start_byte = @source[0...match.begin(0)].bytesize
+          end_byte = @source[0...match.end(0)].bytesize
+          Span.new(kind: 'inlined_concern_display', name: name, lexical_owner: name,
+                   start_byte: start_byte, end_byte: end_byte,
+                   start_line: @source.byteslice(0, start_byte).count("\n") + 1,
+                   end_line: @source.byteslice(0, end_byte).count("\n") + 1)
+        end
+      end
+
+      def terms(value)
+        value.to_s.gsub(/(\p{Ll}|\d)(\p{Lu})/u, '\1 \2').downcase.scan(/[\p{L}\p{N}]+/u)
+             .reject { |term| QueryClassifier::STOP_WORDS.include?(term) }.uniq
+      end
+
+      def relevance(span)
+        ((terms(span.name) & @query).size * 4) + (terms(span_text(span)) & @query).size
+      end
+
+      def span_text(span)
+        @source.byteslice(span.start_byte...span.end_byte)
+      end
+
+      def runtime_records
+        metadata = field(:metadata)
+        return [] unless metadata.is_a?(Hash)
+
+        LexicalIndex::RUNTIME_FIELDS.filter_map do |key|
+          value = metadata[key] || metadata[key.to_sym]
+          next if value.nil?
+
+          record = [key, value]
+          record if @query.empty? || (terms(JSON.generate(record)) & @query).any?
+        end
+      end
+
+      def format_evidence(mode, selected, metadata, total)
+        parts = ["Evidence: #{mode}; published-unit coordinates (physical location unavailable).",
+                 "Unit: #{field(:type)}:#{field(:identifier)}; path: #{field(:file_path)}",
+                 "Source SHA256: #{Digest::SHA256.hexdigest(@source)}",
+                 "Generation: #{@generation || 'unavailable (not recorded by this metadata store)'}"]
+        selected.each do |span|
+          label = "#{span.kind} #{span.lexical_owner} #{span.name}; published lines #{span.start_line}-#{span.end_line}"
+          parts << (mode == 'outline' ? label : "#{label}\n```ruby\n#{span_text(span)}\n```")
+        end
+        parts << "Published runtime metadata:\n#{JSON.generate(metadata.to_h)}" unless metadata.empty?
+        parts << "Omitted: #{total - selected.size} source spans; remaining source/metadata not reproduced. " \
+                 'Use lookup with evidence: full for the complete published unit.'
+        parts.join("\n\n")
+      end
+
+      def provenance(mode, selected, metadata, all)
+        { mode: mode, owner: { identifier: field(:identifier), type: field(:type) },
+          coordinate_system: 'published_unit', source_sha256: Digest::SHA256.hexdigest(@source),
+          source_path: field(:file_path), physical_location: nil,
+          physical_location_reason: 'published unit may contain synthesized or inlined source',
+          generation: @generation, generation_status: @generation ? 'recorded' : 'unavailable',
+          spans: selected.map { |span| span.to_h.merge(sha256: Digest::SHA256.hexdigest(span_text(span))) },
+          omitted_spans: all.size - selected.size, runtime_fields: metadata.map(&:first),
+          source_complete: false, full_evidence: { tool: 'lookup', evidence: 'full',
+                                                   identifier: field(:identifier), type: field(:type),
+                                                   source_sha256: Digest::SHA256.hexdigest(@source) } }
+      end
+    end
+  end
+end