codebase_index 0.1.0

data/lib/codebase_index.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+# CodebaseIndex - Rails Codebase Indexing and Retrieval
+#
+# A system for extracting, indexing, and retrieving context from Rails codebases
+# to enable AI-assisted development, debugging, and analytics.
+#
+# ## Quick Start
+#
+#   # Extract codebase
+#   CodebaseIndex.extract!
+#
+#   # Or via rake
+#   bundle exec rake codebase_index:extract
+#
+# ## Configuration
+#
+#   CodebaseIndex.configure do |config|
+#     config.output_dir = Rails.root.join("tmp/codebase_index")
+#     config.max_context_tokens = 8000
+#     config.include_framework_sources = true
+#   end
+#
+require_relative 'codebase_index/version'
+
+module CodebaseIndex
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class ExtractionError < Error; end
+  class SessionTracerError < Error; end
+
+  CONFIG_MUTEX = Mutex.new
+
+  # ════════════════════════════════════════════════════════════════════════
+  # Configuration
+  # ════════════════════════════════════════════════════════════════════════
+
+  class Configuration
+    attr_accessor :embedding_model, :include_framework_sources, :gem_configs,
+                  :vector_store, :metadata_store, :graph_store, :embedding_provider, :log_level,
+                  :vector_store_options, :metadata_store_options, :embedding_options,
+                  :concurrent_extraction, :precompute_flows, :enable_snapshots,
+                  :session_tracer_enabled, :session_store, :session_id_proc, :session_exclude_paths,
+                  :notion_api_token, :notion_database_ids
+    attr_reader :max_context_tokens, :similarity_threshold, :extractors, :pretty_json, :context_format
+
+    def initialize
+      @output_dir = nil # Resolved lazily; Rails.root is nil at require time
+      @embedding_model = 'text-embedding-3-small'
+      @max_context_tokens = 8000
+      @similarity_threshold = 0.7
+      @include_framework_sources = true
+      @gem_configs = {}
+      @extractors = %i[models controllers services components view_components jobs mailers graphql serializers
+                       managers policies validators rails_source]
+      @pretty_json = true
+      @concurrent_extraction = false
+      @precompute_flows = false
+      @enable_snapshots = false
+      @context_format = :markdown
+      @session_tracer_enabled = false
+      @session_store = nil
+      @session_id_proc = nil
+      @session_exclude_paths = []
+      @notion_api_token = nil
+      @notion_database_ids = {}
+    end
+
+    # @return [Pathname, String] Output directory, defaulting to Rails.root/tmp/codebase_index
+    def output_dir
+      @output_dir ||= defined?(Rails) && Rails.root ? Rails.root.join('tmp/codebase_index') : 'tmp/codebase_index'
+    end
+
+    # @param value [Object] Must respond to #to_s
+    # @raise [ConfigurationError] if value is nil
+    def output_dir=(value)
+      raise ConfigurationError, 'output_dir cannot be nil' if value.nil?
+
+      @output_dir = value
+    end
+
+    # @param value [Integer] Must be a positive Integer
+    # @raise [ConfigurationError] if value is not a positive Integer
+    def max_context_tokens=(value)
+      unless value.is_a?(Integer) && value.positive?
+        raise ConfigurationError, "max_context_tokens must be a positive Integer, got #{value.inspect}"
+      end
+
+      @max_context_tokens = value
+    end
+
+    # @param value [Numeric] Must be between 0.0 and 1.0 inclusive
+    # @raise [ConfigurationError] if value is out of range or not numeric
+    def similarity_threshold=(value)
+      raise ConfigurationError, "similarity_threshold must be Numeric, got #{value.inspect}" unless value.is_a?(Numeric)
+
+      float_val = value.to_f
+      unless float_val.between?(0.0, 1.0)
+        raise ConfigurationError, "similarity_threshold must be between 0.0 and 1.0, got #{value.inspect}"
+      end
+
+      @similarity_threshold = float_val
+    end
+
+    # @param value [Array<Symbol>] List of extractor names
+    # @raise [ConfigurationError] if value is not an Array of Symbols
+    def extractors=(value)
+      unless value.is_a?(Array) && value.all?(Symbol)
+        raise ConfigurationError, "extractors must be an Array of Symbols, got #{value.inspect}"
+      end
+
+      @extractors = value
+    end
+
+    # @param value [Boolean] Must be true or false
+    # @raise [ConfigurationError] if value is not a boolean
+    def pretty_json=(value)
+      unless value.is_a?(TrueClass) || value.is_a?(FalseClass)
+        raise ConfigurationError, "pretty_json must be true or false, got #{value.inspect}"
+      end
+
+      @pretty_json = value
+    end
+
+    # @param value [Symbol] Must be one of :claude, :markdown, :plain, :json
+    # @raise [ConfigurationError] if value is not a valid format
+    def context_format=(value)
+      valid = %i[claude markdown plain json]
+      unless valid.include?(value)
+        raise ConfigurationError, "context_format must be one of #{valid.inspect}, got #{value.inspect}"
+      end
+
+      @context_format = value
+    end
+
+    # Add a gem to be indexed
+    #
+    # @param gem_name [String] Name of the gem
+    # @param paths [Array<String>] Relative paths within the gem to index
+    # @param priority [Symbol] :high, :medium, or :low
+    def add_gem(gem_name, paths:, priority: :medium)
+      @gem_configs[gem_name] = { paths: paths, priority: priority }
+    end
+  end
+
+  # ════════════════════════════════════════════════════════════════════════
+  # Module Interface
+  # ════════════════════════════════════════════════════════════════════════
+
+  class << self
+    attr_accessor :configuration
+
+    def configure
+      CONFIG_MUTEX.synchronize do
+        self.configuration ||= Configuration.new
+        yield(configuration) if block_given?
+        configuration
+      end
+    end
+
+    # Configure the module using a named preset and optional block customization.
+    #
+    # Valid preset names: :local, :postgresql, :production
+    #
+    # @param name [Symbol] Preset name
+    # @yield [config] Optional block for further customization after preset is applied
+    # @yieldparam config [Configuration] The configuration object
+    # @return [Configuration] The applied configuration
+    def configure_with_preset(name)
+      CONFIG_MUTEX.synchronize do
+        self.configuration = Builder.preset_config(name)
+        yield configuration if block_given?
+        configuration
+      end
+    end
+
+    # Build a Retriever wired with adapters from the current configuration.
+    #
+    # @return [Retriever] A fully wired retriever instance
+    def build_retriever
+      Builder.new(configuration).build_retriever
+    end
+
+    # Retrieve context for a natural language query using the current configuration.
+    #
+    # @param query [String] Natural language query
+    # @param opts [Hash] Options passed through to the retriever (e.g., budget:)
+    # @return [Retriever::RetrievalResult] Retrieval result
+    def retrieve(query, **opts)
+      build_retriever.retrieve(query, **opts)
+    end
+
+    # Perform full extraction
+    #
+    # @param output_dir [String] Override output directory
+    # @return [Hash] Extraction results
+    def extract!(output_dir: nil)
+      require_relative 'codebase_index/extractor'
+
+      dir = output_dir || configuration.output_dir
+      extractor = Extractor.new(output_dir: dir)
+      extractor.extract_all
+    end
+
+    # Perform incremental extraction
+    #
+    # @param changed_files [Array<String>] List of changed files
+    # @return [Array<String>] Re-extracted unit identifiers
+    def extract_changed!(changed_files)
+      require_relative 'codebase_index/extractor'
+
+      extractor = Extractor.new(output_dir: configuration.output_dir)
+      extractor.extract_changed(changed_files)
+    end
+  end
+
+  # Initialize with defaults
+  configure
+end
+
+require_relative 'codebase_index/builder'
+require_relative 'codebase_index/cost_model'
+require_relative 'codebase_index/railtie' if defined?(Rails::Railtie)

data/lib/generators/codebase_index/install_generator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module CodebaseIndex
+  module Generators
+    # Rails generator that creates a migration for CodebaseIndex tables.
+    #
+    # Usage:
+    #   rails generate codebase_index:install
+    #
+    # Creates a migration with codebase_units, codebase_edges, and
+    # codebase_embeddings tables. Works with PostgreSQL, MySQL, and SQLite.
+    #
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Creates a migration for CodebaseIndex tables (units, edges, embeddings)'
+
+      # @return [void]
+      def create_migration_file
+        migration_template(
+          'create_codebase_index_tables.rb.erb',
+          'db/migrate/create_codebase_index_tables.rb'
+        )
+      end
+    end
+  end
+end

data/lib/generators/codebase_index/pgvector_generator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module CodebaseIndex
+  module Generators
+    # Rails generator that adds pgvector support to CodebaseIndex.
+    #
+    # Requires the pgvector PostgreSQL extension. Adds a native vector column
+    # and HNSW index to the codebase_embeddings table.
+    #
+    # Usage:
+    #   rails generate codebase_index:pgvector
+    #   rails generate codebase_index:pgvector --dimensions 3072
+    #
+    class PgvectorGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Adds pgvector native vector column and HNSW index to codebase_embeddings'
+
+      class_option :dimensions, type: :numeric, default: 1536,
+                                desc: 'Vector dimensions (1536 for text-embedding-3-small, 3072 for large)'
+
+      # @return [void]
+      def create_migration_file
+        @dimensions = options[:dimensions]
+        migration_template(
+          'add_pgvector_to_codebase_index.rb.erb',
+          'db/migrate/add_pgvector_to_codebase_index.rb'
+        )
+      end
+    end
+  end
+end

data/lib/generators/codebase_index/templates/add_pgvector_to_codebase_index.rb.erb

@@ -0,0 +1,15 @@
+class AddPgvectorToCodebaseIndex < ActiveRecord::Migration[7.0]
+  def change
+    enable_extension 'vector' unless extension_enabled?('vector')
+
+    add_column :codebase_embeddings, :embedding_vector, :vector,
+               limit: <%= @dimensions || 1536 %>, null: true
+
+    # HNSW index for fast approximate nearest neighbor search
+    # Using cosine distance operator (vector_cosine_ops)
+    add_index :codebase_embeddings, :embedding_vector,
+              using: :hnsw,
+              opclass: :vector_cosine_ops,
+              name: 'idx_codebase_embeddings_vector_hnsw'
+  end
+end

data/lib/generators/codebase_index/templates/create_codebase_index_tables.rb.erb

@@ -0,0 +1,43 @@
+class CreateCodebaseIndexTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :codebase_units do |t|
+      t.string :unit_type, null: false
+      t.string :identifier, null: false
+      t.string :namespace
+      t.string :file_path, null: false
+      t.text :source_code
+      t.string :source_hash
+      t.json :metadata
+
+      t.timestamps
+    end
+
+    add_index :codebase_units, :unit_type
+    add_index :codebase_units, :identifier, unique: true
+    add_index :codebase_units, :file_path
+
+    create_table :codebase_edges do |t|
+      t.references :source, null: false, foreign_key: { to_table: :codebase_units }
+      t.references :target, null: false, foreign_key: { to_table: :codebase_units }
+      t.string :relationship, null: false
+      t.string :via
+
+      t.datetime :created_at, null: false
+    end
+
+    add_index :codebase_edges, [:source_id, :target_id, :relationship], unique: true,
+              name: 'idx_codebase_edges_unique'
+
+    create_table :codebase_embeddings do |t|
+      t.references :unit, null: false, foreign_key: { to_table: :codebase_units }
+      t.string :chunk_type
+      t.text :embedding, null: false
+      t.string :content_hash, null: false
+      t.integer :dimensions, null: false
+
+      t.datetime :created_at, null: false
+    end
+
+    add_index :codebase_embeddings, :content_hash
+  end
+end