codebase_index 0.1.0

data/lib/codebase_index/retrieval/search_executor.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Retrieval
+    # SearchExecutor maps a query classification to a retrieval strategy and
+    # executes it against the configured stores.
+    #
+    # Strategies:
+    # - :vector     — semantic similarity search (understand, implement, debug)
+    # - :keyword    — exact identifier/text matching (locate, reference)
+    # - :graph      — dependency traversal (trace)
+    # - :hybrid     — vector + keyword + graph expansion (exploratory/comprehensive)
+    # - :direct     — direct metadata lookup (pinpoint + locate/reference)
+    #
+    # @example
+    #   executor = SearchExecutor.new(
+    #     vector_store: vector_store,
+    #     metadata_store: metadata_store,
+    #     graph_store: graph_store,
+    #     embedding_provider: embedding_provider
+    #   )
+    #   classification = QueryClassifier.new.classify("How does User model work?")
+    #   result = executor.execute(query: "How does User model work?", classification: classification)
+    #   result.candidates # => [Candidate, ...]
+    #   result.strategy   # => :hybrid
+    #
+    class SearchExecutor
+      # A single search candidate with provenance tracking.
+      Candidate = Struct.new(:identifier, :score, :source, :metadata, keyword_init: true)
+
+      # The result of a search execution.
+      ExecutionResult = Struct.new(:candidates, :strategy, :query, keyword_init: true)
+
+      # Strategy mapping from (intent, scope) → strategy.
+      #
+      # Pinpoint scope always uses :direct for locate/reference.
+      # Comprehensive/exploratory scopes use :hybrid.
+      # Framework intent always uses :keyword against framework sources.
+      STRATEGY_MAP = {
+        # [intent, scope] => strategy
+        # Pinpoint
+        %i[locate pinpoint] => :direct,
+        %i[reference pinpoint] => :direct,
+
+        # Trace always uses graph
+        %i[trace pinpoint] => :graph,
+        %i[trace focused] => :graph,
+        %i[trace exploratory] => :graph,
+        %i[trace comprehensive] => :graph,
+
+        # Framework always keyword
+        %i[framework pinpoint] => :keyword,
+        %i[framework focused] => :keyword,
+        %i[framework exploratory] => :keyword,
+        %i[framework comprehensive] => :keyword
+      }.freeze
+
+      # @param vector_store [Storage::VectorStore::Interface] Vector store adapter
+      # @param metadata_store [Storage::MetadataStore::Interface] Metadata store adapter
+      # @param graph_store [Storage::GraphStore::Interface] Graph store adapter
+      # @param embedding_provider [Embedding::Provider::Interface] Embedding provider
+      def initialize(vector_store:, metadata_store:, graph_store:, embedding_provider:)
+        @vector_store = vector_store
+        @metadata_store = metadata_store
+        @graph_store = graph_store
+        @embedding_provider = embedding_provider
+      end
+
+      # Execute a search based on query classification.
+      #
+      # @param query [String] The original query text
+      # @param classification [QueryClassifier::Classification] Classified query
+      # @param limit [Integer] Maximum candidates to return
+      # @return [ExecutionResult] Candidates with strategy metadata
+      def execute(query:, classification:, limit: 20)
+        strategy = select_strategy(classification)
+        candidates = run_strategy(strategy, query: query, classification: classification, limit: limit)
+
+        ExecutionResult.new(
+          candidates: candidates.first(limit),
+          strategy: strategy,
+          query: query
+        )
+      end
+
+      private
+
+      # Select the best retrieval strategy for a classification.
+      #
+      # @param classification [QueryClassifier::Classification]
+      # @return [Symbol] One of :vector, :keyword, :graph, :hybrid, :direct
+      def select_strategy(classification)
+        intent = classification.intent
+        scope = classification.scope
+
+        # Check explicit mapping first
+        mapped = STRATEGY_MAP[[intent, scope]]
+        return mapped if mapped
+
+        # Comprehensive and exploratory scopes default to hybrid
+        return :hybrid if %i[comprehensive exploratory].include?(scope)
+
+        # Scope-based defaults for remaining intents
+        case intent
+        when :locate, :reference
+          :keyword
+        else
+          :vector
+        end
+      end
+
+      # Execute the selected strategy.
+      #
+      # @param strategy [Symbol] Strategy to execute
+      # @param query [String] Original query text
+      # @param classification [QueryClassifier::Classification]
+      # @param limit [Integer] Max results
+      # @return [Array<Candidate>]
+      def run_strategy(strategy, query:, classification:, limit:)
+        case strategy
+        when :vector
+          execute_vector(query, classification: classification, limit: limit)
+        when :keyword
+          execute_keyword(classification: classification, limit: limit)
+        when :graph
+          execute_graph(classification: classification, limit: limit)
+        when :hybrid
+          execute_hybrid(query, classification: classification, limit: limit)
+        when :direct
+          execute_direct(classification: classification, limit: limit)
+        end
+      end
+
+      # Vector strategy: embed the query and search by similarity.
+      #
+      # @return [Array<Candidate>]
+      def execute_vector(query, classification:, limit:)
+        query_vector = @embedding_provider.embed(query)
+        filters = build_vector_filters(classification)
+
+        results = @vector_store.search(query_vector, limit: limit, filters: filters)
+        results.map do |r|
+          Candidate.new(identifier: r.id, score: r.score, source: :vector, metadata: r.metadata)
+        end
+      end
+
+      # Keyword strategy: search metadata store by extracted keywords.
+      #
+      # Searches each keyword individually and merges results, keeping the
+      # best score per identifier.
+      #
+      # @return [Array<Candidate>]
+      def execute_keyword(classification:, limit:)
+        keywords = classification.keywords
+        return [] if keywords.empty?
+
+        all_results = merge_keyword_results(keywords)
+        rank_keyword_results(all_results, limit)
+      end
+
+      # Search each keyword individually and merge, keeping best score per ID.
+      #
+      # @param keywords [Array<String>]
+      # @return [Hash<String, Hash>] id => { score:, metadata: }
+      def merge_keyword_results(keywords)
+        results_by_id = {}
+        keywords.each do |keyword|
+          results = @metadata_store.search(keyword)
+          results.each_with_index do |r, index|
+            id = r['id']
+            score = 1.0 - (index.to_f / [results.size, 10].max)
+            results_by_id[id] = { score: score, metadata: r } if !results_by_id[id] || score > results_by_id[id][:score]
+          end
+        end
+        results_by_id
+      end
+
+      # Rank merged keyword results into Candidate objects.
+      #
+      # @param results [Hash<String, Hash>]
+      # @param limit [Integer]
+      # @return [Array<Candidate>]
+      def rank_keyword_results(results, limit)
+        scored = results.map do |id, data|
+          Candidate.new(identifier: id, score: data[:score], source: :keyword, metadata: data[:metadata])
+        end
+        scored.sort_by { |c| -c.score }.first(limit)
+      end
+
+      # Graph strategy: find related units via dependency traversal.
+      #
+      # @return [Array<Candidate>]
+      def execute_graph(classification:, limit:)
+        candidates = []
+
+        # First, use keywords to find seed identifiers in the metadata store
+        seeds = find_seed_identifiers(classification)
+        return [] if seeds.empty?
+
+        seeds.each do |seed_id|
+          # Forward dependencies
+          deps = @graph_store.dependencies_of(seed_id)
+          deps.each do |dep|
+            candidates << Candidate.new(identifier: dep, score: 0.8, source: :graph, metadata: {})
+          end
+
+          # Reverse dependencies (dependents)
+          dependents = @graph_store.dependents_of(seed_id)
+          dependents.each do |dep|
+            candidates << Candidate.new(identifier: dep, score: 0.7, source: :graph, metadata: {})
+          end
+
+          # The seed itself
+          candidates << Candidate.new(identifier: seed_id, score: 1.0, source: :graph, metadata: {})
+        end
+
+        deduplicate(candidates).first(limit)
+      end
+
+      # Hybrid strategy: combine vector, keyword, and graph expansion.
+      #
+      # @return [Array<Candidate>]
+      def execute_hybrid(query, classification:, limit:)
+        # Gather from all three sources
+        vector_candidates = execute_vector(query, classification: classification, limit: limit)
+        keyword_candidates = execute_keyword(classification: classification, limit: limit)
+
+        # Graph expansion on top vector results
+        graph_candidates = []
+        vector_candidates.first(3).each do |candidate|
+          deps = @graph_store.dependencies_of(candidate.identifier)
+          deps.each do |dep|
+            graph_candidates << Candidate.new(
+              identifier: dep, score: 0.5, source: :graph_expansion, metadata: {}
+            )
+          end
+        end
+
+        all = vector_candidates + keyword_candidates + graph_candidates
+        deduplicate(all).first(limit)
+      end
+
+      # Direct strategy: look up specific identifiers from keywords.
+      #
+      # Tries each keyword as-is and capitalized (e.g. "user" → "User")
+      # since keywords are lowercased but identifiers are typically PascalCase.
+      #
+      # @return [Array<Candidate>]
+      def execute_direct(classification:, limit:)
+        keywords = classification.keywords
+        return [] if keywords.empty?
+
+        candidates = lookup_keyword_variants(keywords)
+
+        # Fall back to keyword search if direct lookups miss
+        return execute_keyword(classification: classification, limit: limit) if candidates.empty?
+
+        candidates.first(limit)
+      end
+
+      # Try each keyword as-is and in capitalized forms against the metadata store.
+      #
+      # @param keywords [Array<String>]
+      # @return [Array<Candidate>]
+      def lookup_keyword_variants(keywords)
+        candidates = []
+        keywords.each do |keyword|
+          variants = [keyword, keyword.capitalize, keyword.split('_').map(&:capitalize).join].uniq
+          variants.each do |variant|
+            result = @metadata_store.find(variant)
+            next unless result
+
+            candidates << Candidate.new(identifier: variant, score: 1.0, source: :direct, metadata: result)
+            break
+          end
+        end
+        candidates
+      end
+
+      # Build metadata filters for vector search based on classification.
+      #
+      # @param classification [QueryClassifier::Classification]
+      # @return [Hash]
+      def build_vector_filters(classification)
+        filters = {}
+        filters[:type] = classification.target_type.to_s if classification.target_type
+        filters
+      end
+
+      # Find seed identifiers from classification keywords via metadata search.
+      #
+      # @param classification [QueryClassifier::Classification]
+      # @return [Array<String>]
+      def find_seed_identifiers(classification)
+        seeds = []
+
+        # Try direct lookups for capitalized keywords (likely class names)
+        classification.keywords.each do |keyword|
+          capitalized = keyword.split('_').map(&:capitalize).join
+          result = @metadata_store.find(capitalized)
+          seeds << capitalized if result
+        end
+
+        # Fall back to search if no direct hits
+        if seeds.empty? && classification.keywords.any?
+          results = @metadata_store.search(classification.keywords.join(' '))
+          seeds = results.first(3).map { |r| r['id'] }
+        end
+
+        seeds
+      end
+
+      # Deduplicate candidates, keeping the highest-scored entry per identifier.
+      #
+      # @param candidates [Array<Candidate>]
+      # @return [Array<Candidate>]
+      def deduplicate(candidates)
+        best = {}
+        candidates.each do |c|
+          existing = best[c.identifier]
+          best[c.identifier] = c if existing.nil? || c.score > existing.score
+        end
+        best.values.sort_by { |c| -c.score }
+      end
+    end
+  end
+end

data/lib/codebase_index/retriever.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative 'retrieval/query_classifier'
+require_relative 'retrieval/search_executor'
+require_relative 'retrieval/ranker'
+require_relative 'retrieval/context_assembler'
+
+module CodebaseIndex
+  # Retriever orchestrates the full retrieval pipeline: classify, execute,
+  # rank, and assemble context from a natural language query.
+  #
+  # Coordinates four internal components:
+  # - {Retrieval::QueryClassifier} — determines intent, scope, target type
+  # - {Retrieval::SearchExecutor} — maps classification to search strategy
+  # - {Retrieval::Ranker} — re-ranks candidates with weighted signals
+  # - {Retrieval::ContextAssembler} — builds token-budgeted context string
+  #
+  # Optionally builds a structural context overview (codebase unit counts
+  # by type) that is prepended to the assembled context.
+  #
+  # @example
+  #   retriever = CodebaseIndex::Retriever.new(
+  #     vector_store: vector_store,
+  #     metadata_store: metadata_store,
+  #     graph_store: graph_store,
+  #     embedding_provider: embedding_provider
+  #   )
+  #   result = retriever.retrieve("How does the User model work?")
+  #   result.context        # => "Codebase: 42 units (10 models, ...)\n\n---\n\n## User (model)..."
+  #   result.strategy       # => :vector
+  #   result.tokens_used    # => 4200
+  #
+  class Retriever
+    # Diagnostic trace for retrieval quality analysis.
+    RetrievalTrace = Struct.new(:classification, :strategy, :candidate_count,
+                                :ranked_count, :tokens_used, :elapsed_ms,
+                                keyword_init: true)
+
+    # The result of a retrieval operation.
+    RetrievalResult = Struct.new(:context, :sources, :classification, :strategy, :tokens_used, :budget, :trace,
+                                 keyword_init: true)
+
+    # Unit types queried for the structural context overview.
+    STRUCTURAL_TYPES = %w[model controller service job mailer component graphql].freeze
+
+    # @param vector_store [Storage::VectorStore::Interface] Vector store adapter
+    # @param metadata_store [Storage::MetadataStore::Interface] Metadata store adapter
+    # @param graph_store [Storage::GraphStore::Interface] Graph store adapter
+    # @param embedding_provider [Embedding::Provider::Interface] Embedding provider
+    # @param formatter [#call, nil] Optional callable to post-process the context string
+    def initialize(vector_store:, metadata_store:, graph_store:, embedding_provider:, formatter: nil)
+      @metadata_store = metadata_store
+      @formatter = formatter
+
+      @classifier = Retrieval::QueryClassifier.new
+      @executor = Retrieval::SearchExecutor.new(
+        vector_store: vector_store,
+        metadata_store: metadata_store,
+        graph_store: graph_store,
+        embedding_provider: embedding_provider
+      )
+      @ranker = Retrieval::Ranker.new(metadata_store: metadata_store)
+      @assembler = Retrieval::ContextAssembler.new(metadata_store: metadata_store)
+    end
+
+    # Execute the full retrieval pipeline for a natural language query.
+    #
+    # Pipeline: classify -> execute -> rank -> assemble -> format
+    #
+    # @param query [String] Natural language query
+    # @param budget [Integer] Token budget for context assembly
+    # @return [RetrievalResult] Complete retrieval result
+    def retrieve(query, budget: 8000)
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      classification = @classifier.classify(query)
+      execution_result = @executor.execute(query: query, classification: classification)
+      ranked = @ranker.rank(execution_result.candidates, classification: classification)
+      assembled = assemble_context(ranked, classification, budget)
+
+      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(1)
+
+      trace = RetrievalTrace.new(
+        classification: classification,
+        strategy: execution_result.strategy,
+        candidate_count: execution_result.candidates.size,
+        ranked_count: ranked.size,
+        tokens_used: assembled.tokens_used,
+        elapsed_ms: elapsed_ms
+      )
+
+      build_result(assembled, classification, execution_result.strategy, budget, trace)
+    end
+
+    private
+
+    # Assemble token-budgeted context from ranked candidates.
+    #
+    # @param ranked [Array<Candidate>] Ranked search candidates
+    # @param classification [QueryClassifier::Classification] Query classification
+    # @return [AssembledContext]
+    def assemble_context(ranked, classification, budget)
+      @assembler.assemble(
+        candidates: ranked,
+        classification: classification,
+        structural_context: build_structural_context,
+        budget: budget
+      )
+    end
+
+    # Build a RetrievalResult from assembled context and pipeline metadata.
+    #
+    # @param assembled [AssembledContext] Assembled context
+    # @param classification [QueryClassifier::Classification] Query classification
+    # @param strategy [Symbol] Search strategy used
+    # @param budget [Integer] Token budget
+    # @return [RetrievalResult]
+    def build_result(assembled, classification, strategy, budget, trace = nil)
+      context = @formatter ? @formatter.call(assembled.context) : assembled.context
+
+      RetrievalResult.new(
+        context: context,
+        sources: assembled.sources,
+        classification: classification,
+        strategy: strategy,
+        tokens_used: assembled.tokens_used,
+        budget: budget,
+        trace: trace
+      )
+    end
+
+    # Build a structural context overview from the metadata store.
+    #
+    # Queries the metadata store for total unit count and counts per type,
+    # producing a summary like "Codebase: 42 units (10 models, 5 controllers, ...)".
+    #
+    # @return [String, nil] Overview string, or nil if the store is empty or on error
+    def build_structural_context
+      total = @metadata_store.count
+      return nil if total.zero?
+
+      type_counts = STRUCTURAL_TYPES.filter_map do |type|
+        count = count_by_type(type)
+        "#{count} #{type}s" if count.positive?
+      end
+
+      "Codebase: #{total} units (#{type_counts.join(', ')})"
+    rescue StandardError
+      nil
+    end
+
+    # Count units of a given type in the metadata store.
+    #
+    # @param type [String] The unit type to count
+    # @return [Integer] Number of units of this type
+    def count_by_type(type)
+      @metadata_store.find_by_type(type).size
+    end
+  end
+end

data/lib/codebase_index/ruby_analyzer/class_analyzer.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require_relative '../ast/parser'
+require_relative '../extracted_unit'
+require_relative 'fqn_builder'
+
+module CodebaseIndex
+  module RubyAnalyzer
+    # Extracts class and module definitions from Ruby source code using the AST layer.
+    #
+    # Produces ExtractedUnit objects with type :ruby_class or :ruby_module, including
+    # metadata about superclass, includes, extends, constants, and method count.
+    #
+    # @example
+    #   analyzer = RubyAnalyzer::ClassAnalyzer.new
+    #   units = analyzer.analyze(source: File.read(path), file_path: path)
+    #   units.first.type #=> :ruby_class
+    #
+    class ClassAnalyzer
+      include FqnBuilder
+
+      # @param parser [Ast::Parser, nil] Parser instance (creates default if nil)
+      def initialize(parser: nil)
+        @parser = parser || Ast::Parser.new
+      end
+
+      # Analyze source code and extract class/module units.
+      #
+      # @param source [String] Ruby source code
+      # @param file_path [String] Absolute path to the source file
+      # @return [Array<ExtractedUnit>] Extracted class and module units
+      def analyze(source:, file_path:)
+        root = @parser.parse(source)
+        units = []
+        extract_definitions(root, source, file_path, [], units)
+        units
+      end
+
+      private
+
+      def extract_definitions(node, source, file_path, namespace_stack, units)
+        return unless node.is_a?(Ast::Node)
+
+        case node.type
+        when :class
+          process_class(node, source, file_path, namespace_stack, units)
+        when :module
+          process_module(node, source, file_path, namespace_stack, units)
+        else
+          (node.children || []).each do |child|
+            extract_definitions(child, source, file_path, namespace_stack, units)
+          end
+        end
+      end
+
+      def process_class(node, source, file_path, namespace_stack, units)
+        process_definition(node, :ruby_class, source, file_path, namespace_stack, units)
+      end
+
+      def process_module(node, source, file_path, namespace_stack, units)
+        process_definition(node, :ruby_module, source, file_path, namespace_stack, units)
+      end
+
+      def process_definition(node, type, source, file_path, namespace_stack, units)
+        name = node.method_name
+        fqn = build_fqn(name, namespace_stack)
+        namespace = build_namespace(name, namespace_stack)
+
+        superclass = type == :ruby_class ? extract_superclass(node) : nil
+        children = body_children(node, type)
+        includes = extract_mixins(children, 'include')
+        extends = extract_mixins(children, 'extend')
+        constants = extract_constants(children)
+        method_count = count_methods(children)
+
+        unit = ExtractedUnit.new(type: type, identifier: fqn, file_path: file_path)
+        unit.namespace = namespace
+        unit.source_code = extract_source(node, source)
+        unit.metadata = {
+          superclass: superclass,
+          includes: includes,
+          extends: extends,
+          constants: constants,
+          method_count: method_count
+        }
+        unit.dependencies = build_dependencies(superclass, includes, extends)
+        units << unit
+
+        # Recurse into body for nested definitions
+        inner_ns = namespace_stack + fqn_parts(name)
+        children.each do |child|
+          extract_definitions(child, source, file_path, inner_ns, units)
+        end
+      end
+
+      # Build namespace string (everything except the leaf name).
+      def build_namespace(name, namespace_stack)
+        parts = namespace_stack + fqn_parts(name)
+        parts.pop # Remove leaf
+        parts.empty? ? nil : parts.join('::')
+      end
+
+      # Split a name that may contain :: into parts.
+      def fqn_parts(name)
+        name.to_s.split('::')
+      end
+
+      # Extract superclass name from a class node.
+      # Children[0] is name, children[1] is superclass (or nil).
+      def extract_superclass(class_node)
+        superclass_node = class_node.children[1]
+        return nil unless superclass_node.is_a?(Ast::Node) && superclass_node.type == :const
+
+        build_const_name(superclass_node)
+      end
+
+      # Get body children of a class or module node.
+      # Class: children[0] = name, children[1] = superclass, rest = body
+      # Module: children[0] = name, rest = body
+      def body_children(node, type)
+        offset = type == :ruby_class ? 2 : 1
+        (node.children || [])[offset..] || []
+      end
+
+      # Extract include/extend module names from body send nodes.
+      def extract_mixins(body_children, method_name)
+        body_children.filter_map do |child|
+          next unless child.is_a?(Ast::Node) && child.type == :send
+          next unless child.method_name == method_name
+          next if child.arguments.nil? || child.arguments.empty?
+
+          child.arguments.first
+        end
+      end
+
+      # Extract constant assignment names from body.
+      def extract_constants(body_children)
+        body_children.filter_map do |child|
+          next unless child.is_a?(Ast::Node) && child.type == :casgn
+
+          child.method_name
+        end
+      end
+
+      # Count def and defs nodes in body children (non-recursive — only direct methods).
+      def count_methods(body_children)
+        count = 0
+        body_children.each do |child|
+          next unless child.is_a?(Ast::Node)
+
+          count += 1 if %i[def defs].include?(child.type)
+        end
+        count
+      end
+
+      # Build the constant name from a :const node (may have receiver for namespaced).
+      def build_const_name(const_node)
+        parts = []
+        parts << const_node.receiver if const_node.receiver
+        parts << const_node.method_name if const_node.method_name
+        parts.join('::')
+      end
+
+      # Extract source text for a node using line range.
+      def extract_source(node, source)
+        return nil unless node.line && node.end_line
+
+        lines = source.lines
+        start_idx = node.line - 1
+        end_idx = node.end_line - 1
+        return nil if start_idx.negative? || end_idx >= lines.length
+
+        lines[start_idx..end_idx].join
+      end
+
+      # Build dependency list from superclass, includes, and extends.
+      def build_dependencies(superclass, includes, extends)
+        deps = []
+        deps << { type: :ruby_class, target: superclass, via: :inheritance } if superclass
+        includes.each do |mod|
+          deps << { type: :ruby_class, target: mod, via: :include }
+        end
+        extends.each do |mod|
+          deps << { type: :ruby_class, target: mod, via: :extend }
+        end
+        deps
+      end
+    end
+  end
+end