codebase_index 0.3.2 → 0.4.0

data/lib/codebase_index/db/migrator.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'schema_version'
-require_relative 'migrations/001_create_units'
-require_relative 'migrations/002_create_edges'
-require_relative 'migrations/003_create_embeddings'
-require_relative 'migrations/004_create_snapshots'
-require_relative 'migrations/005_create_snapshot_units'
-
-module CodebaseIndex
-  module Db
-    # Runs schema migrations against a database connection.
-    #
-    # Tracks applied migrations via {SchemaVersion} and only runs pending ones.
-    # Migrations are defined as modules in `db/migrations/` with a VERSION
-    # constant and a `.up(connection)` class method.
-    #
-    # @example
-    #   db = SQLite3::Database.new('codebase_index.db')
-    #   migrator = Migrator.new(connection: db)
-    #   migrator.migrate!  # => [1, 2, 3]
-    #
-    class Migrator
-      MIGRATIONS = [
-        Migrations::CreateUnits,
-        Migrations::CreateEdges,
-        Migrations::CreateEmbeddings,
-        Migrations::CreateSnapshots,
-        Migrations::CreateSnapshotUnits
-      ].freeze
-
-      attr_reader :schema_version
-
-      # @param connection [Object] Database connection supporting #execute
-      def initialize(connection:)
-        @connection = connection
-        @schema_version = SchemaVersion.new(connection: connection)
-        @schema_version.ensure_table!
-      end
-
-      # Run all pending migrations.
-      #
-      # @return [Array<Integer>] Version numbers of newly applied migrations
-      def migrate!
-        applied = []
-        pending_migrations.each do |migration|
-          migration.up(@connection)
-          @schema_version.record_version(migration::VERSION)
-          applied << migration::VERSION
-        end
-        applied
-      end
-
-      # List version numbers of pending (unapplied) migrations.
-      #
-      # @return [Array<Integer>]
-      def pending_versions
-        applied = @schema_version.applied_versions
-        MIGRATIONS.map { |m| m::VERSION }.reject { |v| applied.include?(v) }
-      end
-
-      private
-
-      # @return [Array<Module>] Pending migration modules
-      def pending_migrations
-        applied = @schema_version.applied_versions
-        MIGRATIONS.reject { |m| applied.include?(m::VERSION) }
-      end
-    end
-  end
-end

data/lib/codebase_index/db/schema_version.rb

@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module CodebaseIndex
-  module Db
-    # Tracks which schema migrations have been applied.
-    #
-    # Uses a simple `codebase_index_schema_migrations` table with a single
-    # `version` column. Works with any database connection that supports
-    # `execute` and returns arrays (SQLite3, pg, mysql2).
-    #
-    # @example
-    #   db = SQLite3::Database.new('codebase_index.db')
-    #   sv = SchemaVersion.new(connection: db)
-    #   sv.ensure_table!
-    #   sv.current_version  # => 0
-    #   sv.record_version(1)
-    #   sv.current_version  # => 1
-    #
-    class SchemaVersion
-      TABLE_NAME = 'codebase_index_schema_migrations'
-
-      # @param connection [Object] Database connection supporting #execute
-      def initialize(connection:)
-        @connection = connection
-      end
-
-      # Create the schema migrations table if it does not exist.
-      #
-      # @return [void]
-      def ensure_table!
-        @connection.execute(<<~SQL)
-          CREATE TABLE IF NOT EXISTS #{TABLE_NAME} (
-            version INTEGER PRIMARY KEY NOT NULL,
-            applied_at TEXT NOT NULL DEFAULT (datetime('now'))
-          )
-        SQL
-      end
-
-      # List all applied migration version numbers, sorted ascending.
-      #
-      # @return [Array<Integer>]
-      def applied_versions
-        rows = @connection.execute("SELECT version FROM #{TABLE_NAME} ORDER BY version ASC")
-        rows.map { |row| row.is_a?(Array) ? row[0] : row['version'] }
-      end
-
-      # Record a migration version as applied.
-      #
-      # @param version [Integer] The migration version number
-      # @return [void]
-      def record_version(version)
-        @connection.execute(
-          "INSERT OR IGNORE INTO #{TABLE_NAME} (version) VALUES (?)", [version]
-        )
-      end
-
-      # Check whether a version has been applied.
-      #
-      # @param version [Integer]
-      # @return [Boolean]
-      def applied?(version)
-        applied_versions.include?(version)
-      end
-
-      # The highest applied version, or 0 if none.
-      #
-      # @return [Integer]
-      def current_version
-        applied_versions.last || 0
-      end
-    end
-  end
-end

data/lib/codebase_index/dependency_graph.rb

@@ -1,236 +0,0 @@
-# frozen_string_literal: true
-
-require 'set'
-require 'json'
-
-module CodebaseIndex
-  # DependencyGraph tracks relationships between code units for:
-  # 1. Understanding what depends on what
-  # 2. Computing "blast radius" for incremental re-indexing
-  # 3. Enabling graph-based retrieval queries
-  #
-  # The graph is bidirectional - we track both what a unit depends on
-  # and what depends on that unit (reverse edges).
-  #
-  # @example Building and querying the graph
-  #   graph = DependencyGraph.new
-  #   graph.register(user_model_unit)
-  #   graph.register(user_service_unit)
-  #
-  #   # Find everything affected by a change to user.rb
-  #   affected = graph.affected_by(["app/models/user.rb"])
-  #
-  class DependencyGraph
-    def initialize
-      @nodes = {}      # identifier => { type:, file_path: }
-      @edges = {}      # identifier => [dependency identifiers]
-      @reverse = {}    # identifier => Set of dependent identifiers
-      @file_map = {}   # file_path => identifier
-      @type_index = {} # type => Set of identifiers
-      @to_h = nil
-    end
-
-    # Register a unit in the graph
-    #
-    # @param unit [ExtractedUnit] The unit to register
-    def register(unit)
-      @to_h = nil
-
-      @nodes[unit.identifier] = {
-        type: unit.type,
-        file_path: unit.file_path,
-        namespace: unit.namespace
-      }
-
-      @edges[unit.identifier] = unit.dependencies.map { |d| d[:target] }
-      @file_map[unit.file_path] = unit.identifier if unit.file_path
-
-      # Type index for filtering (Set-based for O(1) insert)
-      (@type_index[unit.type] ||= Set.new).add(unit.identifier)
-
-      # Build reverse edges (Set-based for O(1) insert)
-      unit.dependencies.each do |dep|
-        (@reverse[dep[:target]] ||= Set.new).add(unit.identifier)
-      end
-    end
-
-    # Find all units affected by changes to given files
-    # Uses BFS to find transitive dependents
-    #
-    # @param changed_files [Array<String>] List of changed file paths
-    # @param max_depth [Integer] Maximum traversal depth (nil for unlimited)
-    # @return [Array<String>] List of affected unit identifiers
-    def affected_by(changed_files, max_depth: nil)
-      directly_changed = changed_files.filter_map { |f| @file_map[f] }
-
-      affected = Set.new(directly_changed)
-      queue = directly_changed.map { |id| [id, 0] } # [identifier, depth]
-
-      while queue.any?
-        current, depth = queue.shift
-        next if max_depth && depth >= max_depth
-
-        dependents = @reverse[current] || []
-
-        dependents.each do |dep|
-          unless affected.include?(dep)
-            affected.add(dep)
-            queue.push([dep, depth + 1])
-          end
-        end
-      end
-
-      affected.to_a
-    end
-
-    # Check if a node exists in the graph by exact identifier.
-    #
-    # @param identifier [String] Unit identifier to check
-    # @return [Boolean] true if the node exists
-    def node_exists?(identifier)
-      @nodes.key?(identifier)
-    end
-
-    # Find a node by suffix matching (e.g., "Update" matches "Order::Update").
-    #
-    # When multiple nodes share the same suffix, the first match wins.
-    # Suffix matching requires a "::" separator — bare identifiers (no namespace)
-    # are not matched by this method; use {#node_exists?} for exact lookups.
-    #
-    # @param suffix [String] The suffix to match against
-    # @return [String, nil] The first matching identifier, or nil
-    def find_node_by_suffix(suffix)
-      target_suffix = "::#{suffix}"
-      @nodes.keys.find { |id| id.end_with?(target_suffix) }
-    end
-
-    # Get direct dependencies of a unit
-    #
-    # @param identifier [String] Unit identifier
-    # @return [Array<String>] List of dependency identifiers
-    def dependencies_of(identifier)
-      @edges[identifier] || []
-    end
-
-    # Get direct dependents of a unit (what depends on it)
-    #
-    # @param identifier [String] Unit identifier
-    # @return [Array<String>] List of dependent identifiers
-    def dependents_of(identifier)
-      @reverse.fetch(identifier, Set.new).to_a
-    end
-
-    # Get all units of a specific type
-    #
-    # @param type [Symbol] Unit type (:model, :controller, etc.)
-    # @return [Array<String>] List of unit identifiers
-    def units_of_type(type)
-      @type_index.fetch(type, Set.new).to_a
-    end
-
-    # Compute PageRank scores for all nodes
-    #
-    # Uses the reverse edges (dependents) as the link structure: a node
-    # with many dependents gets a higher score. This matches Aider's insight
-    # that structural importance correlates with retrieval relevance.
-    #
-    # @param damping [Float] Damping factor (default: 0.85)
-    # @param iterations [Integer] Number of iterations (default: 20)
-    # @return [Hash<String, Float>] Identifier => PageRank score
-    def pagerank(damping: 0.85, iterations: 20)
-      n = @nodes.size
-      return {} if n.zero?
-
-      node_ids = @nodes.keys
-      base_score = 1.0 / n
-      scores = node_ids.to_h { |id| [id, base_score] }
-
-      iterations.times do
-        # Collect rank from dangling nodes (no outgoing edges) and redistribute
-        dangling_sum = node_ids.sum do |id|
-          @edges[id].nil? || @edges[id].empty? ? scores[id] : 0.0
-        end
-
-        new_scores = {}
-
-        node_ids.each do |id|
-          # Sum contributions from nodes that depend on this one
-          incoming = @reverse[id] || []
-          rank_sum = incoming.sum do |src|
-            out_degree = (@edges[src] || []).size
-            out_degree.positive? ? scores[src] / out_degree : 0.0
-          end
-
-          new_scores[id] = ((1.0 - damping) / n) + (damping * (rank_sum + (dangling_sum / n)))
-        end
-
-        scores = new_scores
-      end
-
-      scores
-    end
-
-    # Serialize graph for persistence. Memoized — cache is invalidated on register.
-    # Returns a dup so callers can't pollute the cached hash.
-    #
-    # @return [Hash] Complete graph data
-    def to_h
-      @to_h ||= {
-        nodes: @nodes,
-        edges: @edges,
-        reverse: @reverse.transform_values(&:to_a),
-        file_map: @file_map,
-        type_index: @type_index.transform_values(&:to_a),
-        stats: {
-          node_count: @nodes.size,
-          edge_count: @edges.values.sum(&:size),
-          types: @type_index.transform_values(&:size)
-        }
-      }
-      @to_h.dup
-    end
-
-    # Load graph from persisted data
-    #
-    # After JSON round-trip all keys become strings. This method normalizes
-    # them back to the expected types: node values use symbol keys (:type,
-    # :file_path, :namespace), and type_index uses symbol keys for types.
-    #
-    # @param data [Hash] Previously serialized graph data
-    # @return [DependencyGraph] Restored graph
-    def self.from_h(data)
-      graph = new
-
-      raw_nodes = data[:nodes] || data['nodes'] || {}
-      graph.instance_variable_set(:@nodes, raw_nodes.transform_values { |v| symbolize_node(v) })
-
-      graph.instance_variable_set(:@edges, data[:edges] || data['edges'] || {})
-
-      raw_reverse = data[:reverse] || data['reverse'] || {}
-      graph.instance_variable_set(:@reverse, raw_reverse.transform_values { |v| v.is_a?(Set) ? v : Set.new(v) })
-
-      graph.instance_variable_set(:@file_map, data[:file_map] || data['file_map'] || {})
-
-      raw_type_index = data[:type_index] || data['type_index'] || {}
-      graph.instance_variable_set(:@type_index, raw_type_index.transform_keys(&:to_sym).transform_values do |v|
-        v.is_a?(Set) ? v : Set.new(v)
-      end)
-
-      graph
-    end
-
-    # Normalize a node hash to use symbol keys
-    #
-    # @param node [Hash] Node data with string or symbol keys
-    # @return [Hash] Node data with symbol keys
-    def self.symbolize_node(node)
-      return node unless node.is_a?(Hash)
-
-      {
-        type: (node[:type] || node['type'])&.to_sym,
-        file_path: node[:file_path] || node['file_path'],
-        namespace: node[:namespace] || node['namespace']
-      }
-    end
-  end
-end

data/lib/codebase_index/embedding/indexer.rb

@@ -1,140 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require 'digest'
-
-module CodebaseIndex
-  module Embedding
-    # Orchestrates the indexing pipeline: reads extracted units, prepares text,
-    # generates embeddings, and stores vectors. Supports full and incremental
-    # modes with checkpoint-based resumability.
-    class Indexer
-      # @param checkpoint_interval [Integer] Save checkpoint every N batches (default: 10)
-      def initialize(provider:, text_preparer:, vector_store:, output_dir:, batch_size: 32, checkpoint_interval: 10) # rubocop:disable Metrics/ParameterLists
-        @provider = provider
-        @text_preparer = text_preparer
-        @vector_store = vector_store
-        @output_dir = output_dir
-        @batch_size = batch_size
-        @checkpoint_interval = checkpoint_interval
-      end
-
-      # Index all extracted units (full mode). Returns stats hash.
-      # @return [Hash] Stats with :processed, :skipped, :errors counts
-      def index_all
-        process_units(load_units, incremental: false)
-      end
-
-      # Index only changed units (incremental mode). Returns stats hash.
-      # @return [Hash] Stats with :processed, :skipped, :errors counts
-      def index_incremental
-        process_units(load_units, incremental: true)
-      end
-
-      private
-
-      def load_units
-        Dir.glob(File.join(@output_dir, '**', '*.json')).filter_map do |path|
-          next if File.basename(path) == 'checkpoint.json'
-
-          JSON.parse(File.read(path))
-        rescue JSON::ParserError
-          nil
-        end
-      end
-
-      def process_units(units, incremental:)
-        checkpoint = incremental ? load_checkpoint : {}
-        stats = { processed: 0, skipped: 0, errors: 0 }
-        batch_count = 0
-
-        units.each_slice(@batch_size) do |batch|
-          process_batch(batch, checkpoint, stats, incremental: incremental)
-          batch_count += 1
-          save_checkpoint(checkpoint) if (batch_count % @checkpoint_interval).zero?
-        end
-
-        # Always save final checkpoint
-        save_checkpoint(checkpoint)
-
-        stats
-      end
-
-      def process_batch(batch, checkpoint, stats, incremental:)
-        to_embed = batch.each_with_object([]) do |unit_data, items|
-          if incremental && checkpoint[unit_data['identifier']] == unit_data['source_hash']
-            stats[:skipped] += 1
-            next
-          end
-          collect_embed_items(unit_data, items)
-        end
-
-        embed_and_store(to_embed, checkpoint, stats)
-      end
-
-      def collect_embed_items(unit_data, items)
-        texts = prepare_texts(unit_data)
-        identifier = unit_data['identifier']
-
-        texts.each_with_index do |text, idx|
-          embed_id = texts.length > 1 ? "#{identifier}#chunk_#{idx}" : identifier
-          items << { id: embed_id, text: text, unit_data: unit_data,
-                     source_hash: unit_data['source_hash'], identifier: identifier }
-        end
-      end
-
-      def prepare_texts(unit_data)
-        unit = build_unit(unit_data)
-        unit.chunks&.any? ? @text_preparer.prepare_chunks(unit) : [@text_preparer.prepare(unit)]
-      end
-
-      def build_unit(data)
-        unit = ExtractedUnit.new(type: data['type']&.to_sym, identifier: data['identifier'],
-                                 file_path: data['file_path'])
-        unit.namespace = data['namespace']
-        unit.source_code = data['source_code']
-        unit.dependencies = data['dependencies'] || []
-        unit.chunks = (data['chunks'] || []).map { |c| c.transform_keys(&:to_sym) }
-        unit
-      end
-
-      def embed_and_store(items, checkpoint, stats)
-        return if items.empty?
-
-        vectors = @provider.embed_batch(items.map { |i| i[:text] })
-        store_vectors(items, vectors, checkpoint, stats)
-      rescue StandardError => e
-        stats[:errors] += items.size
-        raise CodebaseIndex::Error, "Embedding failed: #{e.message}"
-      end
-
-      def store_vectors(items, vectors, checkpoint, stats)
-        entries = items.each_with_index.map do |item, idx|
-          { id: item[:id], vector: vectors[idx],
-            metadata: { type: item[:unit_data]['type'], identifier: item[:identifier],
-                        file_path: item[:unit_data]['file_path'] } }
-        end
-
-        @vector_store.store_batch(entries)
-
-        items.each do |item|
-          checkpoint[item[:identifier]] = item[:source_hash]
-          stats[:processed] += 1
-        end
-      end
-
-      def load_checkpoint
-        path = File.join(@output_dir, 'checkpoint.json')
-        return {} unless File.exist?(path)
-
-        JSON.parse(File.read(path))
-      rescue JSON::ParserError
-        {}
-      end
-
-      def save_checkpoint(checkpoint)
-        File.write(File.join(@output_dir, 'checkpoint.json'), JSON.generate(checkpoint))
-      end
-    end
-  end
-end

data/lib/codebase_index/embedding/openai.rb

@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/http'
-require 'json'
-
-module CodebaseIndex
-  module Embedding
-    module Provider
-      # OpenAI adapter for cloud embeddings via the OpenAI HTTP API.
-      #
-      # Uses the `/v1/embeddings` endpoint to generate embeddings. Requires a valid
-      # OpenAI API key.
-      #
-      # @example
-      #   provider = CodebaseIndex::Embedding::Provider::OpenAI.new(api_key: ENV['OPENAI_API_KEY'])
-      #   vector = provider.embed("class User < ApplicationRecord; end")
-      #   vectors = provider.embed_batch(["text1", "text2"])
-      class OpenAI
-        include Interface
-
-        ENDPOINT = URI('https://api.openai.com/v1/embeddings')
-        DEFAULT_MODEL = 'text-embedding-3-small'
-        DIMENSIONS = {
-          'text-embedding-3-small' => 1536,
-          'text-embedding-3-large' => 3072
-        }.freeze
-
-        # @param api_key [String] OpenAI API key
-        # @param model [String] OpenAI embedding model name (default: text-embedding-3-small)
-        def initialize(api_key:, model: DEFAULT_MODEL)
-          @api_key = api_key
-          @model = model
-        end
-
-        # Embed a single text string.
-        #
-        # @param text [String] the text to embed
-        # @return [Array<Float>] the embedding vector
-        # @raise [CodebaseIndex::Error] if the API returns an error
-        def embed(text)
-          response = post_request({ model: @model, input: text })
-          response['data'].first['embedding']
-        end
-
-        # Embed multiple texts in a single request.
-        #
-        # Sorts results by the index field to guarantee ordering matches input.
-        #
-        # @param texts [Array<String>] the texts to embed
-        # @return [Array<Array<Float>>] array of embedding vectors
-        # @raise [CodebaseIndex::Error] if the API returns an error
-        def embed_batch(texts)
-          response = post_request({ model: @model, input: texts })
-          response['data']
-            .sort_by { |item| item['index'] }
-            .map { |item| item['embedding'] }
-        end
-
-        # Return the dimensionality of vectors produced by this model.
-        #
-        # Uses the known dimensions for standard models, falling back to a
-        # test embedding for unknown models.
-        #
-        # @return [Integer] number of dimensions
-        def dimensions
-          DIMENSIONS[@model] || embed('test').length
-        end
-
-        # Return the model name.
-        #
-        # @return [String] the OpenAI model name
-        def model_name
-          @model
-        end
-
-        private
-
-        # Send a POST request to the OpenAI embeddings API.
-        #
-        # @param body [Hash] request body
-        # @return [Hash] parsed JSON response
-        # @raise [CodebaseIndex::Error] if the API returns a non-success status
-        def post_request(body)
-          request = Net::HTTP::Post.new(ENDPOINT.path)
-          request['Content-Type'] = 'application/json'
-          request['Authorization'] = "Bearer #{@api_key}"
-          request.body = body.to_json
-
-          response = http_client.request(request)
-
-          unless response.is_a?(Net::HTTPSuccess)
-            raise CodebaseIndex::Error, "OpenAI API error: #{response.code} #{response.body}"
-          end
-
-          JSON.parse(response.body)
-        rescue Errno::ECONNRESET, Net::OpenTimeout, IOError
-          # Connection dropped — reset and retry once
-          @http_client = nil
-          response = http_client.request(request)
-          unless response.is_a?(Net::HTTPSuccess)
-            raise CodebaseIndex::Error, "OpenAI API error: #{response.code} #{response.body}"
-          end
-
-          JSON.parse(response.body)
-        end
-
-        # Return a reusable, started HTTP client for the OpenAI API.
-        # Calling http.start opens a persistent TCP connection so
-        # keep_alive_timeout actually takes effect across requests.
-        #
-        # @return [Net::HTTP]
-        def http_client
-          return @http_client if @http_client&.started?
-
-          http = Net::HTTP.new(ENDPOINT.host, ENDPOINT.port)
-          http.use_ssl = true
-          http.open_timeout = 10
-          http.read_timeout = 30
-          http.keep_alive_timeout = 30
-          http.start
-          @http_client = http
-        end
-      end
-    end
-  end
-end