codebase_index 0.1.0

data/lib/codebase_index/coordination/pipeline_lock.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module CodebaseIndex
+  module Coordination
+    class LockError < CodebaseIndex::Error; end
+
+    # File-based lock for preventing concurrent pipeline operations.
+    #
+    # Creates a lock file with PID and timestamp. Supports stale lock
+    # detection for crashed processes.
+    #
+    # @example
+    #   lock = PipelineLock.new(lock_dir: '/tmp', name: 'extraction')
+    #   lock.with_lock do
+    #     # extraction runs here
+    #   end
+    #
+    class PipelineLock
+      DEFAULT_STALE_TIMEOUT = 3600 # 1 hour
+
+      # @param lock_dir [String] Directory for lock files
+      # @param name [String] Lock name (used as filename prefix)
+      # @param stale_timeout [Integer] Seconds after which a lock is considered stale
+      def initialize(lock_dir:, name:, stale_timeout: DEFAULT_STALE_TIMEOUT)
+        @lock_dir = lock_dir
+        @name = name
+        @stale_timeout = stale_timeout
+        @lock_path = File.join(lock_dir, "#{name}.lock")
+        @held = false
+      end
+
+      # Attempt to acquire the lock.
+      #
+      # @return [Boolean] true if lock acquired, false if already held
+      def acquire
+        FileUtils.mkdir_p(@lock_dir)
+
+        # Check for stale lock first (separate from atomic creation)
+        if File.exist?(@lock_path)
+          return false unless stale?
+
+          # Remove stale lock
+          FileUtils.rm_f(@lock_path)
+        end
+
+        # Atomic lock creation: File::EXCL ensures this fails if file already exists
+        File.open(@lock_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
+          f.write(lock_content)
+        end
+        @held = true
+        true
+      rescue Errno::EEXIST
+        false
+      end
+
+      # Release the lock.
+      #
+      # @return [void]
+      def release
+        FileUtils.rm_f(@lock_path) if @held
+        @held = false
+      end
+
+      # Execute a block while holding the lock.
+      #
+      # @yield Block to execute
+      # @return [Object] Return value of the block
+      # @raise [LockError] if lock cannot be acquired
+      def with_lock(&block)
+        raise LockError, "Cannot acquire lock '#{@name}' — another process is running" unless acquire
+
+        begin
+          block.call
+        ensure
+          release
+        end
+      end
+
+      # Whether the lock is currently held by this instance.
+      #
+      # @return [Boolean]
+      def locked?
+        @held && File.exist?(@lock_path)
+      end
+
+      private
+
+      # Check if the existing lock file is stale.
+      #
+      # @return [Boolean]
+      def stale?
+        return false unless File.exist?(@lock_path)
+
+        age = Time.now - File.mtime(@lock_path)
+        age > @stale_timeout
+      rescue Errno::ENOENT
+        true
+      end
+
+      # @return [String] Lock file content (JSON with PID and timestamp)
+      def lock_content
+        JSON.generate(pid: Process.pid, locked_at: Time.now.iso8601, name: @name)
+      end
+    end
+  end
+end

data/lib/codebase_index/cost_model/embedding_cost.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module CostModel
+    # Calculates embedding costs for full-index, incremental, and query-time
+    # scenarios using the token-based pricing from {ProviderPricing}.
+    #
+    # The cost model uses a constant of 450 tokens per chunk, derived from the
+    # BACKEND_MATRIX.md tables (e.g. 500 units × 2.5 chunks = 1250 chunks × 450 = 562K tokens).
+    #
+    # @example
+    #   calc = EmbeddingCost.new(provider: :openai_small)
+    #   calc.full_index_cost(units: 500, chunk_multiplier: 2.5) # => 0.01125
+    #
+    class EmbeddingCost
+      # Average tokens per chunk after hierarchical chunking with context prefix.
+      TOKENS_PER_CHUNK = 450
+
+      # Average tokens per retrieval query.
+      TOKENS_PER_QUERY = 100
+
+      # @param provider [Symbol] Embedding provider key from {ProviderPricing}
+      def initialize(provider:)
+        @cost_per_million = ProviderPricing.cost_per_million(provider)
+      end
+
+      # Cost to embed the full codebase index.
+      #
+      # @param units [Integer] Number of extracted units
+      # @param chunk_multiplier [Float] Average chunks per unit (default 2.5)
+      # @return [Float] Cost in USD
+      def full_index_cost(units:, chunk_multiplier: 2.5)
+        tokens = total_tokens(units, chunk_multiplier)
+        token_cost(tokens)
+      end
+
+      # Cost to re-embed changed units from a single merge.
+      #
+      # @param changed_units [Integer] Number of units changed (default 5)
+      # @param chunk_multiplier [Float] Average chunks per unit (default 2.5)
+      # @return [Float] Cost in USD
+      def incremental_cost(changed_units: 5, chunk_multiplier: 2.5)
+        tokens = total_tokens(changed_units, chunk_multiplier)
+        token_cost(tokens)
+      end
+
+      # Monthly cost for query-time embedding.
+      #
+      # @param daily_queries [Integer] Number of queries per day
+      # @return [Float] Cost in USD per month
+      def monthly_query_cost(daily_queries:)
+        monthly_tokens = daily_queries * 30 * TOKENS_PER_QUERY
+        token_cost(monthly_tokens)
+      end
+
+      # Yearly embedding cost from incremental re-indexing.
+      #
+      # @param merges_per_year [Integer] Number of merges per year (default 2400)
+      # @param changed_units_per_merge [Integer] Units changed per merge (default 5)
+      # @param chunk_multiplier [Float] Average chunks per unit (default 2.5)
+      # @return [Float] Cost in USD per year
+      def yearly_incremental_cost(merges_per_year: 2400, changed_units_per_merge: 5, chunk_multiplier: 2.5)
+        tokens_per_merge = total_tokens(changed_units_per_merge, chunk_multiplier)
+        token_cost(tokens_per_merge * merges_per_year)
+      end
+
+      # Total tokens for a given number of units and chunk multiplier.
+      #
+      # @param units [Integer] Number of units
+      # @param chunk_multiplier [Float] Chunks per unit
+      # @return [Integer] Total embedding tokens
+      def total_tokens(units, chunk_multiplier)
+        chunks = (units * chunk_multiplier).ceil
+        chunks * TOKENS_PER_CHUNK
+      end
+
+      private
+
+      # Convert token count to cost in USD.
+      #
+      # @param tokens [Numeric] Number of tokens
+      # @return [Float] Cost in USD
+      def token_cost(tokens)
+        (tokens.to_f / 1_000_000) * @cost_per_million
+      end
+    end
+  end
+end

data/lib/codebase_index/cost_model/estimator.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module CostModel
+    # Unified cost estimator that combines embedding, storage, and query costs
+    # into a single breakdown for a given configuration.
+    #
+    # @example
+    #   estimate = Estimator.new(
+    #     units: 500,
+    #     chunk_multiplier: 2.5,
+    #     embedding_provider: :openai_small,
+    #     dimensions: 1536,
+    #     daily_queries: 100
+    #   )
+    #   estimate.full_index_cost    # => 0.01125
+    #   estimate.monthly_query_cost # => 0.006
+    #   estimate.storage_bytes      # => 9_984_000
+    #   estimate.to_h               # => { full_index_cost: ..., ... }
+    #
+    class Estimator
+      # @return [Integer] Number of extracted units
+      attr_reader :units
+
+      # @return [Float] Average chunks per unit
+      attr_reader :chunk_multiplier
+
+      # @return [Symbol] Embedding provider key
+      attr_reader :embedding_provider
+
+      # @return [Integer] Embedding vector dimensions
+      attr_reader :dimensions
+
+      # @return [Integer] Number of retrieval queries per day
+      attr_reader :daily_queries
+
+      # @param units [Integer] Number of extracted units
+      # @param chunk_multiplier [Float] Average chunks per unit (default 2.5)
+      # @param embedding_provider [Symbol] Provider key from {ProviderPricing}
+      # @param dimensions [Integer, nil] Vector dimensions (defaults to provider default)
+      # @param daily_queries [Integer] Retrieval queries per day (default 100)
+      def initialize(units:, embedding_provider:, chunk_multiplier: 2.5, dimensions: nil, daily_queries: 100)
+        @units = units
+        @chunk_multiplier = chunk_multiplier
+        @embedding_provider = embedding_provider
+        @dimensions = dimensions || ProviderPricing.default_dimensions(embedding_provider)
+        @daily_queries = daily_queries
+
+        @embedding_cost = EmbeddingCost.new(provider: embedding_provider)
+        @storage_cost = StorageCost.new(dimensions: @dimensions)
+      end
+
+      # Cost to embed the full codebase index.
+      #
+      # @return [Float] Cost in USD
+      def full_index_cost
+        @embedding_cost.full_index_cost(units: units, chunk_multiplier: chunk_multiplier)
+      end
+
+      # Cost to re-embed a single merge (default 5 changed units).
+      #
+      # @param changed_units [Integer] Units changed per merge (default 5)
+      # @return [Float] Cost in USD
+      def incremental_per_merge_cost(changed_units: 5)
+        @embedding_cost.incremental_cost(changed_units: changed_units, chunk_multiplier: chunk_multiplier)
+      end
+
+      # Monthly cost for query-time embedding.
+      #
+      # @return [Float] Cost in USD per month
+      def monthly_query_cost
+        @embedding_cost.monthly_query_cost(daily_queries: daily_queries)
+      end
+
+      # Yearly embedding cost from incremental re-indexing.
+      #
+      # @param merges_per_year [Integer] Merges per year (default 2400)
+      # @return [Float] Cost in USD per year
+      def yearly_incremental_cost(merges_per_year: 2400)
+        @embedding_cost.yearly_incremental_cost(
+          merges_per_year: merges_per_year,
+          chunk_multiplier: chunk_multiplier
+        )
+      end
+
+      # Total number of chunks for the codebase.
+      #
+      # @return [Integer]
+      def total_chunks
+        @total_chunks ||= (units * chunk_multiplier).ceil
+      end
+
+      # Total storage in bytes for vector data.
+      #
+      # @return [Integer]
+      def storage_bytes
+        @storage_cost.storage_bytes(chunks: total_chunks)
+      end
+
+      # Total storage in megabytes for vector data.
+      #
+      # @return [Float]
+      def storage_mb
+        @storage_cost.storage_mb(chunks: total_chunks)
+      end
+
+      # Full cost breakdown as a Hash.
+      #
+      # @return [Hash{Symbol => Numeric}]
+      def to_h
+        {
+          full_index_cost: full_index_cost,
+          incremental_per_merge_cost: incremental_per_merge_cost,
+          monthly_query_cost: monthly_query_cost,
+          yearly_incremental_cost: yearly_incremental_cost,
+          storage_bytes: storage_bytes,
+          storage_mb: storage_mb,
+          total_chunks: total_chunks,
+          units: units,
+          chunk_multiplier: chunk_multiplier,
+          embedding_provider: embedding_provider,
+          dimensions: dimensions,
+          daily_queries: daily_queries
+        }
+      end
+    end
+  end
+end

data/lib/codebase_index/cost_model/provider_pricing.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module CostModel
+    # Frozen pricing data for embedding providers.
+    #
+    # Costs are expressed as dollars per 1 million tokens, sourced from
+    # BACKEND_MATRIX.md. Each provider is identified by a Symbol key.
+    #
+    # @example
+    #   ProviderPricing.cost_per_million(:openai_small) # => 0.02
+    #   ProviderPricing.providers                        # => [:openai_small, ...]
+    #
+    module ProviderPricing
+      # Cost per 1 million tokens, in USD.
+      #
+      # @return [Hash{Symbol => Float}]
+      COSTS_PER_MILLION_TOKENS = {
+        openai_small: 0.02,
+        openai_large: 0.13,
+        voyage_code3: 0.06,
+        ollama: 0.00
+      }.freeze
+
+      # Default embedding dimensions per provider.
+      #
+      # @return [Hash{Symbol => Integer}]
+      DEFAULT_DIMENSIONS = {
+        openai_small: 1536,
+        openai_large: 3072,
+        voyage_code3: 1024,
+        ollama: 768
+      }.freeze
+
+      # Look up the cost per 1M tokens for a provider.
+      #
+      # @param provider [Symbol] Provider key (e.g. :openai_small)
+      # @return [Float] Cost in USD per 1M tokens
+      # @raise [ArgumentError] if provider is unknown
+      def self.cost_per_million(provider)
+        COSTS_PER_MILLION_TOKENS.fetch(provider) do
+          raise ArgumentError, "Unknown embedding provider: #{provider.inspect}. " \
+                               "Valid providers: #{providers.join(', ')}"
+        end
+      end
+
+      # Look up the default dimensions for a provider.
+      #
+      # @param provider [Symbol] Provider key
+      # @return [Integer] Default embedding dimensions
+      # @raise [ArgumentError] if provider is unknown
+      def self.default_dimensions(provider)
+        DEFAULT_DIMENSIONS.fetch(provider) do
+          raise ArgumentError, "Unknown embedding provider: #{provider.inspect}. " \
+                               "Valid providers: #{providers.join(', ')}"
+        end
+      end
+
+      # List all known provider keys.
+      #
+      # @return [Array<Symbol>]
+      def self.providers
+        COSTS_PER_MILLION_TOKENS.keys
+      end
+    end
+  end
+end

data/lib/codebase_index/cost_model/storage_cost.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module CostModel
+    # Calculates vector storage requirements based on embedding dimensions
+    # and chunk count.
+    #
+    # Bytes per vector = dimensions × 4 (float32), with a 1.3× metadata
+    # overhead factor applied per BACKEND_MATRIX.md.
+    #
+    # @example
+    #   calc = StorageCost.new(dimensions: 1536)
+    #   calc.storage_bytes(chunks: 1250) # => 9_984_000
+    #   calc.storage_mb(chunks: 1250)    # => 9.52
+    #
+    class StorageCost
+      # Bytes per float32 value.
+      BYTES_PER_FLOAT = 4
+
+      # Metadata overhead multiplier (JSONB payload, indexes, etc.).
+      METADATA_OVERHEAD = 1.3
+
+      # @param dimensions [Integer] Embedding vector dimensions
+      def initialize(dimensions:)
+        @dimensions = dimensions
+      end
+
+      # Bytes per vector including metadata overhead.
+      #
+      # @return [Integer]
+      def bytes_per_vector
+        @bytes_per_vector ||= (@dimensions * BYTES_PER_FLOAT * METADATA_OVERHEAD).ceil
+      end
+
+      # Total storage in bytes for a given number of chunks.
+      #
+      # @param chunks [Integer] Total number of chunks (units × chunk_multiplier)
+      # @return [Integer]
+      def storage_bytes(chunks:)
+        chunks * bytes_per_vector
+      end
+
+      # Total storage in megabytes for a given number of chunks.
+      #
+      # @param chunks [Integer] Total number of chunks
+      # @return [Float] Storage in MB, rounded to 2 decimal places
+      def storage_mb(chunks:)
+        (storage_bytes(chunks: chunks).to_f / (1024 * 1024)).round(2)
+      end
+    end
+  end
+end

data/lib/codebase_index/cost_model.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative 'cost_model/provider_pricing'
+require_relative 'cost_model/embedding_cost'
+require_relative 'cost_model/storage_cost'
+require_relative 'cost_model/estimator'
+
+module CodebaseIndex
+  # Cost modeling for embedding, storage, and query costs across different
+  # backend configurations. Based on the cost analysis in BACKEND_MATRIX.md.
+  #
+  # @example
+  #   estimate = CodebaseIndex::CostModel::Estimator.new(
+  #     units: 500,
+  #     embedding_provider: :openai_small
+  #   )
+  #   estimate.full_index_cost    # => 0.011
+  #   estimate.monthly_query_cost # => 0.006
+  #
+  module CostModel
+  end
+end

data/lib/codebase_index/db/migrations/001_create_units.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Db
+    module Migrations
+      # Creates the codebase_units table for storing extracted unit metadata.
+      module CreateUnits
+        VERSION = 1
+
+        # @param connection [Object] Database connection
+        # @return [void]
+        def self.up(connection) # rubocop:disable Metrics/MethodLength
+          connection.execute(<<~SQL)
+            CREATE TABLE IF NOT EXISTS codebase_units (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              unit_type TEXT NOT NULL,
+              identifier TEXT NOT NULL,
+              namespace TEXT,
+              file_path TEXT NOT NULL,
+              source_code TEXT,
+              source_hash TEXT,
+              metadata TEXT,
+              created_at TEXT NOT NULL DEFAULT (datetime('now')),
+              updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+              UNIQUE(identifier)
+            )
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_units_type ON codebase_units(unit_type)
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_units_file_path ON codebase_units(file_path)
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/db/migrations/002_create_edges.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Db
+    module Migrations
+      # Creates the codebase_edges table for storing unit relationships.
+      module CreateEdges
+        VERSION = 2
+
+        # @param connection [Object] Database connection
+        # @return [void]
+        def self.up(connection)
+          connection.execute(<<~SQL)
+            CREATE TABLE IF NOT EXISTS codebase_edges (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              source_id INTEGER NOT NULL,
+              target_id INTEGER NOT NULL,
+              relationship TEXT NOT NULL,
+              via TEXT,
+              created_at TEXT NOT NULL DEFAULT (datetime('now')),
+              FOREIGN KEY (source_id) REFERENCES codebase_units(id),
+              FOREIGN KEY (target_id) REFERENCES codebase_units(id)
+            )
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_edges_source ON codebase_edges(source_id)
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_edges_target ON codebase_edges(target_id)
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/db/migrations/003_create_embeddings.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Db
+    module Migrations
+      # Creates the codebase_embeddings table for storing vector embeddings.
+      # Uses TEXT for embedding storage (JSON array) for database portability.
+      # Pgvector users should use the pgvector generator for native vector columns.
+      module CreateEmbeddings
+        VERSION = 3
+
+        # @param connection [Object] Database connection
+        # @return [void]
+        def self.up(connection)
+          connection.execute(<<~SQL)
+            CREATE TABLE IF NOT EXISTS codebase_embeddings (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              unit_id INTEGER NOT NULL,
+              chunk_type TEXT,
+              embedding TEXT NOT NULL,
+              content_hash TEXT NOT NULL,
+              dimensions INTEGER NOT NULL,
+              created_at TEXT NOT NULL DEFAULT (datetime('now')),
+              FOREIGN KEY (unit_id) REFERENCES codebase_units(id)
+            )
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_embeddings_unit ON codebase_embeddings(unit_id)
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_codebase_embeddings_hash ON codebase_embeddings(content_hash)
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/db/migrations/004_create_snapshots.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Db
+    module Migrations
+      # Creates the codebase_snapshots table for temporal index tracking.
+      #
+      # Each row represents one extraction run anchored to a git commit SHA.
+      # Stores aggregate stats and diff counts vs. the previous snapshot.
+      module CreateSnapshots
+        VERSION = 4
+
+        # @param connection [Object] Database connection
+        # @return [void]
+        def self.up(connection) # rubocop:disable Metrics/MethodLength
+          connection.execute(<<~SQL)
+            CREATE TABLE IF NOT EXISTS codebase_snapshots (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              git_sha TEXT NOT NULL,
+              git_branch TEXT,
+              extracted_at TEXT NOT NULL,
+              rails_version TEXT,
+              ruby_version TEXT,
+              total_units INTEGER NOT NULL DEFAULT 0,
+              unit_counts TEXT,
+              gemfile_lock_sha TEXT,
+              schema_sha TEXT,
+              units_added INTEGER DEFAULT 0,
+              units_modified INTEGER DEFAULT 0,
+              units_deleted INTEGER DEFAULT 0,
+              created_at TEXT NOT NULL DEFAULT (datetime('now')),
+              UNIQUE(git_sha)
+            )
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_snapshots_extracted_at ON codebase_snapshots(extracted_at)
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_snapshots_branch ON codebase_snapshots(git_branch)
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/db/migrations/005_create_snapshot_units.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Db
+    module Migrations
+      # Creates the codebase_snapshot_units table for per-unit temporal tracking.
+      #
+      # Each row links a unit (by identifier) to a snapshot, storing content hashes
+      # for efficient diff computation without duplicating full source code.
+      module CreateSnapshotUnits
+        VERSION = 5
+
+        # @param connection [Object] Database connection
+        # @return [void]
+        def self.up(connection)
+          connection.execute(<<~SQL)
+            CREATE TABLE IF NOT EXISTS codebase_snapshot_units (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              snapshot_id INTEGER NOT NULL,
+              identifier TEXT NOT NULL,
+              unit_type TEXT NOT NULL,
+              source_hash TEXT,
+              metadata_hash TEXT,
+              dependencies_hash TEXT,
+              created_at TEXT NOT NULL DEFAULT (datetime('now')),
+              FOREIGN KEY (snapshot_id) REFERENCES codebase_snapshots(id),
+              UNIQUE(snapshot_id, identifier)
+            )
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_snapshot_units_identifier ON codebase_snapshot_units(identifier)
+          SQL
+          connection.execute(<<~SQL)
+            CREATE INDEX IF NOT EXISTS idx_snapshot_units_snapshot ON codebase_snapshot_units(snapshot_id)
+          SQL
+        end
+      end
+    end
+  end
+end