codebase_index 0.1.0

data/lib/codebase_index/embedding/provider.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+module CodebaseIndex
+  module Embedding
+    # Interface and adapters for embedding providers.
+    #
+    # All embedding providers implement the {Interface} module, which defines
+    # the contract for generating vector embeddings from text.
+    module Provider
+      # Interface that all embedding providers must implement.
+      #
+      # Defines the contract for embedding text into vector representations.
+      # Implementations must provide single-text embedding, batch embedding,
+      # dimension reporting, and model identification.
+      module Interface
+        # Embed a single text string into a vector.
+        #
+        # @param text [String] the text to embed
+        # @return [Array<Float>] the embedding vector
+        # @raise [NotImplementedError] if not implemented by the provider
+        def embed(text)
+          raise NotImplementedError
+        end
+
+        # Embed multiple texts into vectors in a single request.
+        #
+        # @param texts [Array<String>] the texts to embed
+        # @return [Array<Array<Float>>] array of embedding vectors
+        # @raise [NotImplementedError] if not implemented by the provider
+        def embed_batch(texts)
+          raise NotImplementedError
+        end
+
+        # Return the dimensionality of the embedding vectors.
+        #
+        # @return [Integer] number of dimensions
+        # @raise [NotImplementedError] if not implemented by the provider
+        def dimensions
+          raise NotImplementedError
+        end
+
+        # Return the name of the embedding model.
+        #
+        # @return [String] model name
+        # @raise [NotImplementedError] if not implemented by the provider
+        def model_name
+          raise NotImplementedError
+        end
+      end
+
+      # Ollama adapter for local embeddings via the Ollama HTTP API.
+      #
+      # Uses the `/api/embed` endpoint to generate embeddings. Requires a running
+      # Ollama instance (default: localhost:11434) with the specified model pulled.
+      #
+      # @example
+      #   provider = CodebaseIndex::Embedding::Provider::Ollama.new
+      #   vector = provider.embed("class User < ApplicationRecord; end")
+      #   vectors = provider.embed_batch(["text1", "text2"])
+      class Ollama
+        include Interface
+
+        DEFAULT_MODEL = 'nomic-embed-text'
+        DEFAULT_HOST = 'http://localhost:11434'
+
+        # @param model [String] Ollama model name (default: nomic-embed-text)
+        # @param host [String] Ollama server URL (default: http://localhost:11434)
+        def initialize(model: DEFAULT_MODEL, host: DEFAULT_HOST)
+          @model = model
+          @host = host
+          @uri = URI("#{host}/api/embed")
+        end
+
+        # Embed a single text string.
+        #
+        # @param text [String] the text to embed
+        # @return [Array<Float>] the embedding vector
+        # @raise [CodebaseIndex::Error] if the API returns an error
+        def embed(text)
+          response = post_request({ model: @model, input: text })
+          response['embeddings'].first
+        end
+
+        # Embed multiple texts in a single request.
+        #
+        # @param texts [Array<String>] the texts to embed
+        # @return [Array<Array<Float>>] array of embedding vectors
+        # @raise [CodebaseIndex::Error] if the API returns an error
+        def embed_batch(texts)
+          response = post_request({ model: @model, input: texts })
+          response['embeddings']
+        end
+
+        # Return the dimensionality of vectors produced by this model.
+        #
+        # Determined dynamically by embedding a test string on first call.
+        #
+        # @return [Integer] number of dimensions
+        def dimensions
+          @dimensions ||= embed('test').length
+        end
+
+        # Return the model name.
+        #
+        # @return [String] the Ollama model name
+        def model_name
+          @model
+        end
+
+        private
+
+        # Send a POST request to the Ollama API.
+        #
+        # @param body [Hash] request body
+        # @return [Hash] parsed JSON response
+        # @raise [CodebaseIndex::Error] if the API returns a non-success status
+        def post_request(body)
+          http = Net::HTTP.new(@uri.host, @uri.port)
+          request = Net::HTTP::Post.new(@uri.path, 'Content-Type' => 'application/json')
+          request.body = body.to_json
+          response = http.request(request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise CodebaseIndex::Error, "Ollama API error: #{response.code} #{response.body}"
+          end
+
+          JSON.parse(response.body)
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/embedding/text_preparer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Embedding
+    # Prepares ExtractedUnit data for embedding by building context-prefixed text.
+    #
+    # Follows the context prefix format from docs/CONTEXT_AND_CHUNKING.md:
+    #   [type] identifier
+    #   namespace: ...
+    #   file: ...
+    #   dependencies: dep1, dep2, ...
+    #
+    # Handles token limit enforcement by truncating text that exceeds the
+    # embedding model's context window.
+    #
+    # @example
+    #   preparer = CodebaseIndex::Embedding::TextPreparer.new(max_tokens: 8192)
+    #   text = preparer.prepare(unit)
+    #   chunks = preparer.prepare_chunks(unit)
+    class TextPreparer
+      DEFAULT_MAX_TOKENS = 8192
+
+      # @param max_tokens [Integer] maximum token budget for prepared text
+      def initialize(max_tokens: DEFAULT_MAX_TOKENS)
+        @max_tokens = max_tokens
+      end
+
+      # Prepare text for embedding from an ExtractedUnit.
+      #
+      # Builds a context prefix and appends the unit's source code (or first
+      # chunk content for chunked units). Enforces token limits via truncation.
+      #
+      # @param unit [CodebaseIndex::ExtractedUnit] the unit to prepare
+      # @return [String] context-prefixed text ready for embedding
+      def prepare(unit)
+        prefix = build_prefix(unit)
+        content = select_content(unit)
+        text = "#{prefix}\n#{content}"
+        enforce_token_limit(text)
+      end
+
+      # Prepare text for each chunk of an ExtractedUnit.
+      #
+      # If the unit has no chunks, returns a single-element array with the
+      # full prepared text. For chunked units, each chunk gets the same
+      # context prefix prepended.
+      #
+      # @param unit [CodebaseIndex::ExtractedUnit] the unit to prepare
+      # @return [Array<String>] array of context-prefixed texts
+      def prepare_chunks(unit)
+        return [prepare(unit)] unless unit.chunks&.any?
+
+        prefix = build_prefix(unit)
+        unit.chunks.map do |chunk|
+          text = "#{prefix}\n#{chunk[:content]}"
+          enforce_token_limit(text)
+        end
+      end
+
+      private
+
+      # Build the context prefix for a unit.
+      #
+      # @param unit [CodebaseIndex::ExtractedUnit] the unit
+      # @return [String] formatted prefix lines
+      def build_prefix(unit)
+        lines = []
+        lines << "[#{unit.type}] #{unit.identifier}"
+        lines << "namespace: #{unit.namespace}" if unit.namespace
+        lines << "file: #{unit.file_path}" if unit.file_path
+        append_dependency_line(lines, unit.dependencies)
+        lines.join("\n")
+      end
+
+      # Append a formatted dependency line if dependencies exist.
+      #
+      # @param lines [Array<String>] lines to append to
+      # @param dependencies [Array<Hash>, nil] dependency list
+      # @return [void]
+      def append_dependency_line(lines, dependencies)
+        return unless dependencies&.any?
+
+        dep_names = dependencies.map { |d| d[:target] }.compact.first(10)
+        lines << "dependencies: #{dep_names.join(', ')}" if dep_names.any?
+      end
+
+      # Select the content to embed for a unit.
+      #
+      # @param unit [CodebaseIndex::ExtractedUnit] the unit
+      # @return [String] source code or first chunk content
+      def select_content(unit)
+        if unit.chunks&.any?
+          unit.chunks.first[:content]
+        else
+          unit.source_code || ''
+        end
+      end
+
+      # Truncate text to fit within the token budget.
+      #
+      # @param text [String] the text to truncate
+      # @return [String] text within token limits
+      def enforce_token_limit(text)
+        estimated = (text.length / 4.0).ceil
+        return text if estimated <= @max_tokens
+
+        max_chars = (@max_tokens * 4.0).floor
+        text[0...max_chars]
+      end
+    end
+  end
+end

data/lib/codebase_index/evaluation/baseline_runner.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Evaluation
+    # Runs simple baseline strategies for comparison against the full
+    # retrieval pipeline.
+    #
+    # Provides three baseline strategies:
+    # - `:grep` — substring match on unit identifiers
+    # - `:random` — random selection from available units
+    # - `:file_level` — returns identifiers matching file paths
+    #
+    # @example
+    #   runner = BaselineRunner.new(metadata_store: store)
+    #   results = runner.run("User model", strategy: :grep, limit: 10)
+    #   results  # => ["User", "UserProfile", "UserSerializer"]
+    #
+    class BaselineRunner
+      VALID_STRATEGIES = %i[grep random file_level].freeze
+
+      # @param metadata_store [Object] Store that responds to #all_identifiers and #find_by_type
+      def initialize(metadata_store:)
+        @metadata_store = metadata_store
+      end
+
+      # Run a baseline strategy for a query.
+      #
+      # @param query [String] Natural language query
+      # @param strategy [Symbol] Baseline strategy (:grep, :random, :file_level)
+      # @param limit [Integer] Maximum number of results
+      # @return [Array<String>] Unit identifiers
+      # @raise [ArgumentError] if the strategy is invalid
+      def run(query, strategy:, limit: 10)
+        unless VALID_STRATEGIES.include?(strategy)
+          raise ArgumentError, "Invalid strategy: #{strategy}. Must be one of #{VALID_STRATEGIES.join(', ')}"
+        end
+
+        send(:"run_#{strategy}", query, limit)
+      end
+
+      private
+
+      # Grep strategy: substring match on unit identifiers.
+      #
+      # Extracts words from the query and matches identifiers that contain
+      # any query word (case-insensitive).
+      #
+      # @param query [String] Query string
+      # @param limit [Integer] Max results
+      # @return [Array<String>]
+      def run_grep(query, limit)
+        all_ids = @metadata_store.all_identifiers
+        keywords = extract_keywords(query)
+
+        return all_ids.first(limit) if keywords.empty?
+
+        matches = all_ids.select do |id|
+          id_lower = id.downcase
+          keywords.any? { |kw| id_lower.include?(kw) }
+        end
+
+        matches.first(limit)
+      end
+
+      # Random strategy: random selection from all available units.
+      #
+      # @param _query [String] Query string (unused)
+      # @param limit [Integer] Max results
+      # @return [Array<String>]
+      def run_random(_query, limit)
+        @metadata_store.all_identifiers.sample(limit)
+      end
+
+      # File-level strategy: matches identifiers that look like file paths
+      # or class names extracted from the query.
+      #
+      # @param query [String] Query string
+      # @param limit [Integer] Max results
+      # @return [Array<String>]
+      def run_file_level(query, limit)
+        all_ids = @metadata_store.all_identifiers
+        keywords = extract_keywords(query)
+
+        return all_ids.first(limit) if keywords.empty?
+
+        # Score each identifier by how many keywords it matches
+        scored = all_ids.map do |id|
+          id_lower = id.downcase
+          score = keywords.count { |kw| id_lower.include?(kw) }
+          [id, score]
+        end
+
+        scored.select { |_, score| score.positive? }
+              .sort_by { |_, score| -score }
+              .first(limit)
+              .map(&:first)
+      end
+
+      # Extract lowercase keywords from a query string.
+      #
+      # Filters out common stop words and short words.
+      #
+      # @param query [String] Query text
+      # @return [Array<String>] Keywords
+      def extract_keywords(query)
+        stop_words = %w[the a an is are was were how does do what which where when why
+                        this that these those in on at to for of and or but with from by]
+
+        query.downcase
+             .scan(/[a-z0-9_]+/)
+             .reject { |w| stop_words.include?(w) || w.length < 2 }
+      end
+    end
+  end
+end

data/lib/codebase_index/evaluation/evaluator.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require_relative 'metrics'
+
+module CodebaseIndex
+  module Evaluation
+    # Runs evaluation queries through a Retriever and scores results
+    # against ground truth annotations.
+    #
+    # Takes a configured retriever and a query set, runs each query,
+    # and produces per-query and aggregate metrics.
+    #
+    # @example
+    #   evaluator = Evaluator.new(retriever: retriever, query_set: query_set)
+    #   report = evaluator.evaluate
+    #   report.aggregates[:mean_mrr]  # => 0.75
+    #
+    class Evaluator
+      # Result for a single evaluation query.
+      QueryResult = Struct.new(:query, :expected_units, :retrieved_units, :scores, :tokens_used,
+                               keyword_init: true)
+
+      # Aggregate report across all queries.
+      EvaluationReport = Struct.new(:results, :aggregates, keyword_init: true)
+
+      # @param retriever [CodebaseIndex::Retriever] Configured retriever instance
+      # @param query_set [QuerySet] Set of evaluation queries with ground truth
+      # @param budget [Integer] Token budget per query
+      def initialize(retriever:, query_set:, budget: 8000)
+        @retriever = retriever
+        @query_set = query_set
+        @budget = budget
+      end
+
+      # Run all queries and produce an evaluation report.
+      #
+      # @return [EvaluationReport] Per-query results and aggregate metrics
+      def evaluate
+        results = @query_set.queries.map { |q| evaluate_query(q) }
+        aggregates = compute_aggregates(results)
+        EvaluationReport.new(results: results, aggregates: aggregates)
+      end
+
+      private
+
+      # Evaluate a single query against the retriever.
+      #
+      # @param query [QuerySet::Query] Evaluation query
+      # @return [QueryResult]
+      def evaluate_query(query)
+        retrieval_result = @retriever.retrieve(query.query, budget: @budget)
+        retrieved_ids = extract_identifiers(retrieval_result)
+
+        scores = compute_scores(retrieved_ids, query.expected_units, retrieval_result)
+
+        QueryResult.new(
+          query: query.query,
+          expected_units: query.expected_units,
+          retrieved_units: retrieved_ids,
+          scores: scores,
+          tokens_used: retrieval_result.tokens_used
+        )
+      end
+
+      # Extract unit identifiers from retrieval result sources.
+      #
+      # @param result [Retriever::RetrievalResult] Retrieval result
+      # @return [Array<String>] Ordered list of unit identifiers
+      def extract_identifiers(result)
+        return [] unless result.sources
+
+        result.sources.map { |s| s.is_a?(Hash) ? s[:identifier] || s['identifier'] : s.to_s }
+      end
+
+      # Compute all metrics for a query result.
+      #
+      # @param retrieved [Array<String>] Retrieved identifiers
+      # @param expected [Array<String>] Expected identifiers
+      # @param result [Retriever::RetrievalResult] Retrieval result
+      # @return [Hash] Metric scores
+      def compute_scores(retrieved, expected, result)
+        {
+          precision_at5: Metrics.precision_at_k(retrieved, expected, cutoff: 5),
+          precision_at10: Metrics.precision_at_k(retrieved, expected, cutoff: 10),
+          recall: Metrics.recall(retrieved, expected),
+          mrr: Metrics.mrr(retrieved, expected),
+          context_completeness: Metrics.context_completeness(retrieved, expected),
+          token_efficiency: compute_token_efficiency(retrieved, expected, result)
+        }
+      end
+
+      # Compute token efficiency from the retrieval result.
+      #
+      # @param retrieved [Array<String>] Retrieved identifiers
+      # @param expected [Array<String>] Expected identifiers
+      # @param result [Retriever::RetrievalResult] Retrieval result
+      # @return [Float]
+      def compute_token_efficiency(retrieved, expected, result)
+        return 0.0 if result.tokens_used.nil? || result.tokens_used.zero?
+
+        expected_set = expected.to_set
+        relevant_count = retrieved.count { |id| expected_set.include?(id) }
+        total_count = [retrieved.size, 1].max
+        relevant_ratio = relevant_count.to_f / total_count
+
+        Metrics.token_efficiency((result.tokens_used * relevant_ratio).ceil, result.tokens_used)
+      end
+
+      # Compute aggregate metrics across all query results.
+      #
+      # @param results [Array<QueryResult>] Individual query results
+      # @return [Hash] Aggregate metrics
+      def compute_aggregates(results)
+        return empty_aggregates if results.empty?
+
+        metric_keys = %i[precision_at5 precision_at10 recall mrr context_completeness token_efficiency]
+        aggregates = {}
+
+        metric_keys.each do |key|
+          values = results.map { |r| r.scores[key] }
+          aggregates[:"mean_#{key}"] = values.sum / values.size.to_f
+        end
+
+        aggregates[:total_queries] = results.size
+        aggregates[:mean_tokens_used] = results.sum(&:tokens_used) / results.size.to_f
+        aggregates
+      end
+
+      # Return zero-valued aggregates for empty result sets.
+      #
+      # @return [Hash]
+      def empty_aggregates
+        {
+          mean_precision_at5: 0.0,
+          mean_precision_at10: 0.0,
+          mean_recall: 0.0,
+          mean_mrr: 0.0,
+          mean_context_completeness: 0.0,
+          mean_token_efficiency: 0.0,
+          total_queries: 0,
+          mean_tokens_used: 0.0
+        }
+      end
+    end
+  end
+end

data/lib/codebase_index/evaluation/metrics.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Evaluation
+    # Retrieval quality metrics.
+    #
+    # All methods are stateless pure functions that take arrays of identifiers
+    # and return numeric scores.
+    #
+    module Metrics
+      module_function
+
+      # Fraction of top-k results that are relevant.
+      #
+      # @param retrieved [Array<String>] Retrieved unit identifiers (ordered)
+      # @param relevant [Array<String>] Ground-truth relevant identifiers
+      # @param cutoff [Integer] Number of top results to consider
+      # @return [Float] 0.0 to 1.0
+      def precision_at_k(retrieved, relevant, cutoff: 5)
+        return 0.0 if retrieved.empty? || relevant.empty?
+
+        top_k = retrieved.first(cutoff)
+        relevant_set = relevant.to_set
+        hits = top_k.count { |id| relevant_set.include?(id) }
+        hits.to_f / cutoff
+      end
+
+      # Fraction of relevant items that were retrieved.
+      #
+      # @param retrieved [Array<String>] Retrieved identifiers
+      # @param relevant [Array<String>] Ground-truth relevant identifiers
+      # @return [Float] 0.0 to 1.0
+      def recall(retrieved, relevant)
+        return 0.0 if relevant.empty?
+
+        retrieved_set = retrieved.to_set
+        found = relevant.count { |id| retrieved_set.include?(id) }
+        found.to_f / relevant.size
+      end
+
+      # Mean Reciprocal Rank — inverse of the rank of the first relevant result.
+      #
+      # @param retrieved [Array<String>] Retrieved identifiers (ordered)
+      # @param relevant [Array<String>] Ground-truth relevant identifiers
+      # @return [Float] 0.0 to 1.0
+      def mrr(retrieved, relevant)
+        relevant_set = relevant.to_set
+        retrieved.each_with_index do |id, idx|
+          return 1.0 / (idx + 1) if relevant_set.include?(id)
+        end
+        0.0
+      end
+
+      # Fraction of required units present in retrieved results.
+      #
+      # @param retrieved [Array<String>] Retrieved identifiers
+      # @param required [Array<String>] Required identifiers (subset of relevant)
+      # @return [Float] 0.0 to 1.0
+      def context_completeness(retrieved, required)
+        return 1.0 if required.empty?
+
+        retrieved_set = retrieved.to_set
+        found = required.count { |id| retrieved_set.include?(id) }
+        found.to_f / required.size
+      end
+
+      # Ratio of relevant tokens to total tokens in context.
+      #
+      # @param relevant_tokens [Integer] Tokens from relevant units
+      # @param total_tokens [Integer] Total tokens in assembled context
+      # @return [Float] 0.0 to 1.0
+      def token_efficiency(relevant_tokens, total_tokens)
+        return 0.0 if total_tokens.zero?
+
+        [relevant_tokens.to_f / total_tokens, 1.0].min
+      end
+    end
+  end
+end