codebase_index 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 982e7949df0e0db9249705ab9f009121c3c8156582c63712f0613fccc998337d
-  data.tar.gz: 4fb41c658901cd26606e44164da7059a7d62aa39c795a682020cfbb6252311be
+  metadata.gz: a55ea6a46950c68c6cc1b39f9008bbff466dea9e20d8b646564d4cc489060552
+  data.tar.gz: ca3f117e515fb14f2ec6069c4e6ef541b661e90f0e3d33daa9f9fab43afb746a
 SHA512:
-  metadata.gz: 6b62fe0a1d8b0db683744214461ec5d0029e41cf4538b7313ce2701a3a985bf9b1d06ce955acb225db09e93aaa72bcbc5e3b40cd534f7d682d98d190a670d722
-  data.tar.gz: f0a948295982aa85951fa8cca96cc8c30b317176a53424fe18e50cc1d3e28b17df5fc9dfb6a191e6450a1318b3b0f81a0f2cdf20fe7326c0c4e492a7f8b47f70
+  metadata.gz: e9ff9f423902e1651540b9f933e9a1afab31b3b4ebb0347a368769a83aaf8c1a493e7a33ac68d14a7733753da085dc830e9509911d0bc49205b21e2e07d82de3
+  data.tar.gz: ab8630e12cd123cfeeeee6796d0529cdbaa0f8e6e8c0fb9d7d95dadf753b8a8fbb0d5e2136ff2353cc3aac8b0acb5f525f75f903bc777e86326468c6a759ae8a

data/exe/codebase-index-mcp

@@ -21,8 +21,9 @@ require_relative '../lib/codebase_index/embedding/indexer'
 
 index_dir = CodebaseIndex::MCP::Bootstrapper.resolve_index_dir(ARGV)
 retriever = CodebaseIndex::MCP::Bootstrapper.build_retriever
+snapshot_store = CodebaseIndex::MCP::Bootstrapper.build_snapshot_store(index_dir)
 
-server = CodebaseIndex::MCP::Server.build(index_dir: index_dir, retriever: retriever)
+server = CodebaseIndex::MCP::Server.build(index_dir: index_dir, retriever: retriever, snapshot_store: snapshot_store)
 
 # Pin protocol version for broad client compatibility (Claude Code, Cursor, etc.)
 if ENV['MCP_PROTOCOL_VERSION']

data/exe/codebase-index-mcp-http

@@ -22,11 +22,12 @@ require_relative '../lib/codebase_index/embedding/indexer'
 
 index_dir = CodebaseIndex::MCP::Bootstrapper.resolve_index_dir(ARGV)
 retriever = CodebaseIndex::MCP::Bootstrapper.build_retriever
+snapshot_store = CodebaseIndex::MCP::Bootstrapper.build_snapshot_store(index_dir)
 
 port = (ENV['PORT'] || 9292).to_i
 host = ENV['HOST'] || 'localhost'
 
-server = CodebaseIndex::MCP::Server.build(index_dir: index_dir, retriever: retriever)
+server = CodebaseIndex::MCP::Server.build(index_dir: index_dir, retriever: retriever, snapshot_store: snapshot_store)
 transport = MCP::Server::Transports::StreamableHTTPTransport.new(server)
 server.transport = transport
 

data/lib/codebase_index/console/embedded_executor.rb

@@ -23,13 +23,23 @@ module CodebaseIndex
 
       TIER1_TOOLS = Bridge::TIER1_TOOLS
 
+      # Tools gated behind the read_tools_enabled flag.
+      # sql/query have existing safety gates (SqlValidator, SafeContext rollback)
+      # but require explicit opt-in for embedded mode.
+      EMBEDDED_READ_TOOLS = %w[sql query].freeze
+
+      MAX_SQL_LIMIT = 10_000
+      MAX_QUERY_LIMIT = 10_000
+
       # @param model_validator [ModelValidator] Validates model/column names
       # @param safe_context [SafeContext] Wraps execution in rolled-back transaction
       # @param connection [Object, nil] Database connection for adapter detection
-      def initialize(model_validator:, safe_context:, connection: nil)
+      # @param read_tools_enabled [Boolean] Enable sql/query tools in embedded mode (default: false)
+      def initialize(model_validator:, safe_context:, connection: nil, read_tools_enabled: false)
         @model_validator = model_validator
         @safe_context = safe_context
         @connection = connection
+        @read_tools_enabled = read_tools_enabled
       end
 
       # Execute a tool request and return a response hash.
@@ -46,7 +56,7 @@ module CodebaseIndex
         tool = request['tool']
         params = request['params'] || {}
 
-        unless TIER1_TOOLS.include?(tool)
+        unless TIER1_TOOLS.include?(tool) || (@read_tools_enabled && EMBEDDED_READ_TOOLS.include?(tool))
           return { 'ok' => false,
                    'error' => 'Not yet implemented in embedded mode',
                    'error_type' => 'unsupported' }
@@ -72,8 +82,10 @@ module CodebaseIndex
       # @return [Hash] Tool result
       def dispatch(tool, params)
         case tool
-        when 'status'  then handle_status
-        when 'schema'  then handle_schema(params)
+        when 'status' then handle_status
+        when 'schema' then handle_schema(params)
+        when 'sql'    then handle_sql(params)
+        when 'query'  then handle_query(params)
         else
           validate_model!(params)
           send(:"handle_#{tool}", params)
@@ -211,17 +223,86 @@ module CodebaseIndex
         { 'status' => 'ok', 'models' => @model_validator.model_names, 'adapter' => adapter }
       end
 
+      # ── Read tools (sql/query, gated by read_tools_enabled) ────────────
+
+      # Execute validated read-only SQL via ActiveRecord's select_all.
+      #
+      # @param params [Hash] Must contain 'sql'; optional 'limit'
+      # @return [Hash] Columns and rows
+      def handle_sql(params)
+        sql = params['sql']
+        raise ValidationError, 'Missing required parameter: sql' unless sql
+
+        require_relative 'sql_validator'
+        SqlValidator.new.validate!(sql)
+
+        limit = params['limit'] ? [params['limit'].to_i, MAX_SQL_LIMIT].min : nil
+        query_sql = limit ? "SELECT * FROM (#{sql}) AS _limited LIMIT #{limit}" : sql
+        result = active_connection.select_all(query_sql)
+
+        { 'columns' => result.columns, 'rows' => result.rows, 'count' => result.rows.size }
+      rescue SqlValidationError => e
+        raise ValidationError, e.message
+      end
+
+      # Build and execute a structured ActiveRecord query.
+      #
+      # @param params [Hash] Must contain 'model' and 'select'
+      # @return [Hash] Columns and rows
+      def handle_query(params)
+        validate_model!(params)
+        model = resolve_model(params['model'])
+        relation = build_query_relation(model, params)
+        result = active_connection.select_all(relation.to_sql)
+        { 'columns' => result.columns, 'rows' => result.rows, 'count' => result.rows.size }
+      end
+
+      # Build an ActiveRecord relation from structured query parameters.
+      #
+      # @param model [Class] ActiveRecord model class
+      # @param params [Hash] Query parameters (select, joins, scope, group_by, having, order, limit)
+      # @return [ActiveRecord::Relation]
+      def build_query_relation(model, params)
+        relation = apply_query_clauses(model.all, params)
+        limit = params['limit'] ? [params['limit'].to_i, MAX_QUERY_LIMIT].min : MAX_QUERY_LIMIT
+        relation.limit(limit)
+      end
+
+      # Apply select/joins/scope/group/having/order clauses to a relation.
+      #
+      # @param relation [ActiveRecord::Relation]
+      # @param params [Hash]
+      # @return [ActiveRecord::Relation]
+      def apply_query_clauses(relation, params) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        relation = relation.select(params['select']) if params['select']
+        relation = relation.joins(params['joins'].map(&:to_sym)) if params['joins']&.any?
+        relation = apply_scope(relation, params['scope'])
+        relation = relation.group(params['group_by']) if params['group_by']&.any?
+        relation = relation.having(params['having']) if params['having']
+        relation = relation.order(params['order']) if params['order']
+        relation
+      end
+
       # ── Helpers ──────────────────────────────────────────────────────────
 
       # Apply scope conditions (WHERE clauses) to a relation.
       #
+      # Accepts Hash form for simple equality conditions, or Array form
+      # for parameterized SQL (e.g., JSON column queries like
+      # ["preferences->>'theme' = ?", "dark"]).
+      #
       # @param relation [ActiveRecord::Relation, Class] Model or relation
-      # @param scope [Hash, nil] Filter conditions
+      # @param scope [Hash, Array, nil] Filter conditions
       # @return [ActiveRecord::Relation]
       def apply_scope(relation, scope)
-        return relation unless scope.is_a?(Hash) && scope.any?
-
-        relation.where(scope)
+        case scope
+        when Hash
+          scope.any? ? relation.where(scope) : relation
+        when Array
+          scope.any? ? relation.where(*scope) : relation
+        else
+          relation
+        end
       end
 
       # Apply column selection to a relation.

data/lib/codebase_index/console/rack_middleware.rb

@@ -16,9 +16,11 @@ module CodebaseIndex
     class RackMiddleware
       # @param app [#call] The next Rack app in the middleware stack
       # @param path [String] URL path to mount the MCP endpoint (default: '/mcp/console')
-      def initialize(app, path: '/mcp/console')
+      # @param embedded_read_tools [Boolean] Enable sql/query tools in embedded mode (default: false)
+      def initialize(app, path: '/mcp/console', embedded_read_tools: false)
         @app = app
         @path = path
+        @embedded_read_tools = embedded_read_tools
         @mutex = Mutex.new
         @transport = nil
       end
@@ -71,7 +73,8 @@ module CodebaseIndex
           server = Server.build_embedded(
             model_validator: validator,
             safe_context: safe_context,
-            redacted_columns: redacted
+            redacted_columns: redacted,
+            read_tools_enabled: @embedded_read_tools
           )
 
           @transport = MCP::Server::Transports::StreamableHTTPTransport.new(server)

data/lib/codebase_index/console/server.rb

@@ -56,12 +56,15 @@ module CodebaseIndex
         # @param safe_context [SafeContext] Wraps queries in rolled-back transactions
         # @param redacted_columns [Array<String>] Column names to redact from output
         # @param connection [Object, nil] Database connection for adapter detection
+        # @param read_tools_enabled [Boolean] Enable sql/query tools in embedded mode (default: false)
         # @return [MCP::Server] Configured server ready for transport
-        def build_embedded(model_validator:, safe_context:, redacted_columns: [], connection: nil)
+        def build_embedded(model_validator:, safe_context:, redacted_columns: [], connection: nil,
+                           read_tools_enabled: false)
           require_relative 'embedded_executor'
 
           executor = EmbeddedExecutor.new(
-            model_validator: model_validator, safe_context: safe_context, connection: connection
+            model_validator: model_validator, safe_context: safe_context,
+            connection: connection, read_tools_enabled: read_tools_enabled
           )
           redact_ctx = if redacted_columns.any?
                          SafeContext.new(connection: nil,
@@ -568,15 +571,35 @@ module CodebaseIndex
         def define_console_tool(server, conn_mgr, name, description, properties:, required: nil,
                                 safe_ctx: nil, renderer: nil, &tool_block)
           bridge_method = method(:send_to_bridge)
+          coerce_method = method(:coerce_integer_args!)
+          integer_keys = integer_property_keys(properties)
           schema = { properties: properties }
           schema[:required] = required if required&.any?
           server.define_tool(name: name, description: description, input_schema: schema) do |server_context:, **args|
+            coerce_method.call(args, integer_keys)
             request = tool_block.call(args)
             bridge_method.call(conn_mgr, request.transform_keys(&:to_s), safe_ctx, renderer: renderer)
           end
         end
         # rubocop:enable Metrics/ParameterLists
 
+        # Pre-compute property keys declared as integer in a schema.
+        #
+        # @param properties [Hash] Tool schema properties
+        # @return [Array<Symbol>]
+        def integer_property_keys(properties)
+          properties.select { |_k, v| v[:type] == 'integer' }.keys.map(&:to_sym)
+        end
+
+        # Coerce string values to integers for known integer keys.
+        #
+        # @param args [Hash] Tool arguments (mutated in place)
+        # @param keys [Array<Symbol>] Keys that should be integers
+        # @return [void]
+        def coerce_integer_args!(args, keys)
+          keys.each { |k| args[k] = args[k].to_i if args[k].is_a?(String) }
+        end
+
         # Schema property helpers for concise tool definitions.
         def str_prop(desc)  = { type: 'string', description: desc }
         def int_prop(desc)  = { type: 'integer', description: desc }

data/lib/codebase_index/extractor.rb

@@ -207,6 +207,7 @@ module CodebaseIndex
       @output_dir = Pathname.new(output_dir || Rails.root.join('tmp/codebase_index'))
       @dependency_graph = DependencyGraph.new
       @results = {}
+      @extractors = {}
     end
 
     # ══════════════════════════════════════════════════════════════════════
@@ -268,6 +269,7 @@ module CodebaseIndex
       write_graph_analysis
       write_manifest
       write_structural_summary
+      capture_snapshot
 
       log_summary
 
@@ -317,6 +319,7 @@ module CodebaseIndex
       write_dependency_graph
       write_manifest
       write_structural_summary
+      capture_snapshot
 
       affected_ids
     end
@@ -377,6 +380,7 @@ module CodebaseIndex
         start_time = Time.current
 
         extractor = extractor_class.new
+        @extractors[type] = extractor
         units = extractor.extract_all
 
         @results[type] = units
@@ -410,12 +414,16 @@ module CodebaseIndex
           start_time = Time.current
 
           extractor = extractor_class.new
+          results_mutex.synchronize { @extractors[type] = extractor }
+
           units = extractor.extract_all
 
           elapsed = Time.current - start_time
           Rails.logger.info "[CodebaseIndex] [Thread] Extracted #{units.size} #{type} in #{elapsed.round(2)}s"
 
-          results_mutex.synchronize { @results[type] = units }
+          results_mutex.synchronize do
+            @results[type] = units
+          end
         rescue StandardError => e
           Rails.logger.error "[CodebaseIndex] [Thread] #{type} failed: #{e.message}"
           results_mutex.synchronize { @results[type] = [] }
@@ -743,9 +751,10 @@ module CodebaseIndex
         total_units: @results.values.sum(&:size),
         total_chunks: @results.sum { |_, units| units.sum { |u| u.chunks.size } },
 
-        # Git info
-        git_sha: run_git('rev-parse', 'HEAD').presence,
-        git_branch: run_git('rev-parse', '--abbrev-ref', 'HEAD').presence,
+        # Git info — fall back to env vars for Docker/worktree environments
+        # where the git repo may not be directly accessible
+        git_sha: run_git('rev-parse', 'HEAD').presence || ENV['GIT_SHA'].presence,
+        git_branch: run_git('rev-parse', '--abbrev-ref', 'HEAD').presence || ENV['GIT_BRANCH'].presence,
 
         # For change detection
         gemfile_lock_sha: gemfile_lock_sha,
@@ -758,6 +767,63 @@ module CodebaseIndex
       )
     end
 
+    # Capture a temporal snapshot after extraction completes.
+    #
+    # Reads the manifest and computes per-unit content hashes, then delegates
+    # to the SnapshotStore for storage and diff computation. Requires
+    # enable_snapshots and a valid git_sha in the manifest.
+    #
+    # @return [void]
+    def capture_snapshot
+      return unless CodebaseIndex.configuration.enable_snapshots
+
+      manifest_path = @output_dir.join('manifest.json')
+      return unless manifest_path.exist?
+
+      manifest = JSON.parse(File.read(manifest_path))
+      return unless manifest['git_sha']
+
+      store = build_snapshot_store
+      return unless store
+
+      unit_hashes = @results.flat_map do |type, units|
+        units.map do |unit|
+          {
+            'identifier' => unit.identifier,
+            'type' => type.to_s,
+            'source_hash' => Digest::SHA256.hexdigest(unit.source_code.to_s),
+            'metadata_hash' => Digest::SHA256.hexdigest(unit.metadata.to_json),
+            'dependencies_hash' => Digest::SHA256.hexdigest(unit.dependencies.to_json)
+          }
+        end
+      end
+
+      store.capture(manifest, unit_hashes)
+      Rails.logger.info "[CodebaseIndex] Snapshot captured for #{manifest['git_sha'][0..7]}"
+    rescue StandardError => e
+      Rails.logger.error "[CodebaseIndex] Snapshot capture failed (#{e.class}): #{e.message}"
+    end
+
+    # Build a snapshot store, preferring SQLite with JSON file fallback.
+    #
+    # @return [CodebaseIndex::Temporal::SnapshotStore, CodebaseIndex::Temporal::JsonSnapshotStore, nil]
+    def build_snapshot_store
+      require 'sqlite3'
+      require_relative 'db/migrator'
+      require_relative 'temporal/snapshot_store'
+
+      db_path = @output_dir.join('codebase_index.sqlite3')
+      db = SQLite3::Database.new(db_path.to_s)
+      db.results_as_hash = true
+
+      Db::Migrator.new(connection: db).migrate!
+      Temporal::SnapshotStore.new(connection: db)
+    rescue LoadError
+      Rails.logger.info '[CodebaseIndex] sqlite3 gem not available, using JSON snapshot store'
+      require_relative 'temporal/json_snapshot_store'
+      Temporal::JsonSnapshotStore.new(dir: @output_dir.to_s)
+    end
+
     # Write a compact TOC-style summary of extracted units.
     #
     # Produces a SUMMARY.md under 8K tokens (~24KB) by listing one line per
@@ -886,6 +952,16 @@ module CodebaseIndex
       Rails.logger.info "[CodebaseIndex]   Total: #{total} units, #{chunks} chunks"
       Rails.logger.info "[CodebaseIndex]   Output: #{@output_dir}"
       Rails.logger.info '[CodebaseIndex] ═══════════════════════════════════════════'
+
+      all_warnings = @extractors.flat_map do |_type, ext|
+        ext.respond_to?(:warnings) ? ext.warnings : []
+      end
+
+      return if all_warnings.empty?
+
+      Rails.logger.warn '[CodebaseIndex] ───────────────────────────────────────────'
+      Rails.logger.warn "[CodebaseIndex]   Warnings (#{all_warnings.size}):"
+      all_warnings.each { |w| Rails.logger.warn "[CodebaseIndex]     #{w}" }
     end
 
     # ──────────────────────────────────────────────────────────────────────

data/lib/codebase_index/extractors/concern_extractor.rb

@@ -8,7 +8,8 @@ module CodebaseIndex
     # ConcernExtractor handles ActiveSupport::Concern module extraction.
     #
     # Concerns are mixins that extend model and controller behavior.
-    # They live in `app/models/concerns/` and `app/controllers/concerns/`.
+    # They live in `app/models/concerns/` and `app/controllers/concerns/`,
+    # as well as nested directories like `app/models/gateway/stripe/concerns/`.
     #
     # We extract:
     # - Module name and namespace
@@ -25,13 +26,20 @@ module CodebaseIndex
       include SharedUtilityMethods
       include SharedDependencyScanner
 
-      # Directories to scan for concern modules
+      # Canonical concern directories (used as fallback if glob finds nothing).
       CONCERN_DIRECTORIES = %w[
         app/models/concerns
         app/controllers/concerns
       ].freeze
 
       def initialize
+        # Discover all concerns/ directories under app/, including deeply nested ones
+        # like app/models/gateway/stripe/webhook/concerns/.
+        @directories = Dir[Rails.root.join('app/**/concerns')].map { |d| Pathname.new(d) }
+                                                              .select(&:directory?)
+        # Fall back to canonical directories if glob finds nothing.
+        return unless @directories.empty?
+
         @directories = CONCERN_DIRECTORIES.map { |d| Rails.root.join(d) }
                                           .select(&:directory?)
       end
@@ -91,10 +99,11 @@ module CodebaseIndex
         modules = source.scan(/^\s*module\s+([\w:]+)/).flatten
         return modules.last if modules.any?
 
-        # Infer from file path
+        # Infer from file path — strip everything up to and including the first concerns/ dir.
+        # Handles canonical (app/models/concerns/) and nested (app/models/foo/concerns/) paths.
         relative = file_path.sub("#{Rails.root}/", '')
         relative
-          .sub(%r{^app/(models|controllers)/concerns/}, '')
+          .sub(%r{^app/.*?/concerns/}, '')
           .sub('.rb', '')
           .split('/')
           .map { |segment| segment.split('_').map(&:capitalize).join }
@@ -179,9 +188,9 @@ module CodebaseIndex
       # @param file_path [String] Path to the concern file
       # @return [String] One of "model", "controller", "unknown"
       def detect_concern_scope(file_path)
-        if file_path.include?('app/models/concerns')
+        if file_path.include?('app/models/')
           'model'
-        elsif file_path.include?('app/controllers/concerns')
+        elsif file_path.include?('app/controllers/')
           'controller'
         else
           'unknown'
@@ -196,7 +205,10 @@ module CodebaseIndex
         source.scan(/^\s*def\s+(\w+[?!=]?)/).flatten.reject { |m| m.start_with?('self.') }
       end
 
-      # Detect other modules included by this concern.
+      # Detect other modules included or extended by this concern (for metadata display).
+      #
+      # Returns all module names found in include/extend calls, excluding
+      # ActiveSupport::Concern itself.
       #
       # @param source [String] Ruby source code
       # @return [Array<String>] Module names
@@ -205,6 +217,28 @@ module CodebaseIndex
               .reject { |m| m == 'ActiveSupport::Concern' }
       end
 
+      # Detect modules explicitly included by this concern.
+      #
+      # Scans for bare +include ModuleName+ calls, excluding ActiveSupport::Concern.
+      #
+      # @param source [String] Ruby source code
+      # @return [Array<String>] Included module names
+      def detect_includes(source)
+        source.scan(/\binclude\s+([\w:]+)/).flatten
+              .reject { |m| m == 'ActiveSupport::Concern' }
+      end
+
+      # Detect modules explicitly extended by this concern.
+      #
+      # Scans for bare +extend ModuleName+ calls, excluding ActiveSupport::Concern.
+      #
+      # @param source [String] Ruby source code
+      # @return [Array<String>] Extended module names
+      def detect_extends(source)
+        source.scan(/\bextend\s+([\w:]+)/).flatten
+              .reject { |m| m == 'ActiveSupport::Concern' }
+      end
+
       # Detect callback declarations.
       #
       # @param source [String] Ruby source code
@@ -236,11 +270,16 @@ module CodebaseIndex
       # @param source [String] Ruby source code
       # @return [Array<Hash>] Dependency hashes
       def extract_dependencies(source)
-        # Other concerns included by this concern
-        deps = detect_included_modules(source).map do |mod|
+        # Concerns included by this concern (add instance-level behavior)
+        deps = detect_includes(source).map do |mod|
           { type: :concern, target: mod, via: :include }
         end
 
+        # Concerns extended by this concern (add class-level behavior)
+        detect_extends(source).each do |mod|
+          deps << { type: :concern, target: mod, via: :extend }
+        end
+
         # Standard dependency scanning
         deps.concat(scan_model_dependencies(source))
         deps.concat(scan_service_dependencies(source))

data/lib/codebase_index/extractors/model_extractor.rb

@@ -36,8 +36,12 @@ module CodebaseIndex
         (?:after|before)_(?:add|remove)_for_         # collection callbacks
       )/x
 
+      # Warnings collected during extraction (skipped associations, failed models)
+      attr_reader :warnings
+
       def initialize
         @concern_cache = {}
+        @warnings = []
       end
 
       # Extract all ActiveRecord models in the application
@@ -79,6 +83,7 @@ module CodebaseIndex
 
         unit
       rescue StandardError => e
+        @warnings << "Failed to extract model #{model.name}: #{e.message}"
         Rails.logger.error("Failed to extract model #{model.name}: #{e.message}")
         nil
       end
@@ -232,6 +237,37 @@ module CodebaseIndex
         end
       end
 
+      # Get modules extended specifically in this model (not inherited).
+      #
+      # Extended modules live on the singleton class and add class-level methods.
+      # Ruby builtins (Kernel, PP, etc.) are filtered out by comparing against
+      # Object.singleton_class.included_modules.
+      #
+      # @param model [Class] The ActiveRecord model class
+      # @return [Array<Module>] App-defined modules extended by this model
+      def extract_extended_modules(model)
+        app_root = Rails.root.to_s
+        builtin_modules = Object.singleton_class.included_modules.map(&:name).compact.to_set
+
+        model.singleton_class.included_modules.select do |mod|
+          next false unless mod.name
+          next false if builtin_modules.include?(mod.name)
+
+          # Skip obvious non-app modules (from gems/stdlib)
+          if Object.respond_to?(:const_source_location)
+            loc = Object.const_source_location(mod.name)
+            next false if loc && !app_source?(loc.first, app_root)
+          end
+
+          # Include if it's in app/models/concerns or app/controllers/concerns
+          mod.name.include?('Concerns') ||
+            # Or if it's namespaced under the model's parent
+            mod.name.start_with?("#{model.module_parent}::") ||
+            # Or if it's defined within the application
+            defined_in_app?(mod)
+        end
+      end
+
       # Check if a module is defined within the Rails application
       #
       # @param mod [Module] The module to check
@@ -312,6 +348,13 @@ module CodebaseIndex
           table_exists: model.table_exists?,
           column_count: model.table_exists? ? model.columns.size : 0,
           column_names: model.table_exists? ? model.column_names : [],
+          columns: if model.table_exists?
+                     model.columns.map do |col|
+                       { 'name' => col.name, 'type' => col.sql_type, 'null' => col.null, 'default' => col.default }
+                     end
+                   else
+                     []
+                   end,
 
           # ActiveStorage / ActionText
           active_storage_attachments: extract_active_storage_attachments(source),
@@ -415,9 +458,11 @@ module CodebaseIndex
         result
       end
 
-      # Extract all associations with full details
+      # Extract all associations with full details.
+      # Broken associations (e.g. missing class_name) are skipped with a warning
+      # instead of aborting the entire model extraction.
       def extract_associations(model)
-        model.reflect_on_all_associations.map do |assoc|
+        model.reflect_on_all_associations.filter_map do |assoc|
           {
             name: assoc.name,
             type: assoc.macro, # :belongs_to, :has_many, :has_one, :has_and_belongs_to_many
@@ -428,6 +473,9 @@ module CodebaseIndex
             foreign_key: assoc.foreign_key,
             inverse_of: assoc.inverse_of&.name
           }
+        rescue NameError => e
+          @warnings << "[#{model.name}] Skipping broken association #{assoc.name}: #{e.message}"
+          nil
         end
       end
 
@@ -562,8 +610,21 @@ module CodebaseIndex
       # Extract what this model depends on
       def extract_dependencies(model, source = nil)
         # Associations point to other models
-        deps = model.reflect_on_all_associations.map do |assoc|
+        deps = model.reflect_on_all_associations.filter_map do |assoc|
           { type: :model, target: assoc.class_name, via: :association }
+        rescue NameError => e
+          @warnings << "[#{model.name}] Skipping broken association dep #{assoc.name}: #{e.message}"
+          nil
+        end
+
+        # Included concerns add instance-level behavior
+        extract_included_modules(model).each do |mod|
+          deps << { type: :concern, target: mod.name, via: :include }
+        end
+
+        # Extended modules add class-level behavior (not inlined into source)
+        extract_extended_modules(model).each do |mod|
+          deps << { type: :concern, target: mod.name, via: :extend }
         end
 
         # Parse source for service/mailer/job references

data/lib/codebase_index/mcp/bootstrapper.rb

@@ -30,6 +30,44 @@ module CodebaseIndex
         dir
       end
 
+      # Build a snapshot store for temporal tracking.
+      #
+      # Auto-enables when a SQLite database already exists in the index directory,
+      # or when CODEBASE_INDEX_SNAPSHOTS=true is set. The database is created and
+      # migrated automatically. Falls back to JSON file store when SQLite is
+      # unavailable or encounters errors.
+      #
+      # @param index_dir [String] Path to extraction output directory
+      # @return [CodebaseIndex::Temporal::SnapshotStore, CodebaseIndex::Temporal::JsonSnapshotStore, nil]
+      def self.build_snapshot_store(index_dir)
+        db_path = File.join(index_dir, 'codebase_index.sqlite3')
+        enabled = ENV['CODEBASE_INDEX_SNAPSHOTS'] == 'true' ||
+                  CodebaseIndex.configuration.enable_snapshots ||
+                  File.exist?(db_path)
+
+        return nil unless enabled
+
+        begin
+          require 'sqlite3'
+          require_relative '../db/migrator'
+          require_relative '../temporal/snapshot_store'
+
+          db = SQLite3::Database.new(db_path)
+          db.results_as_hash = true
+
+          CodebaseIndex::Db::Migrator.new(connection: db).migrate!
+          CodebaseIndex::Temporal::SnapshotStore.new(connection: db)
+        rescue LoadError
+          warn 'Note: sqlite3 gem not available, using JSON file-based snapshot store.'
+          require_relative '../temporal/json_snapshot_store'
+          CodebaseIndex::Temporal::JsonSnapshotStore.new(dir: index_dir)
+        rescue StandardError => e
+          warn "Note: SQLite snapshot store failed (#{e.class}: #{e.message}), using JSON fallback."
+          require_relative '../temporal/json_snapshot_store'
+          CodebaseIndex::Temporal::JsonSnapshotStore.new(dir: index_dir)
+        end
+      end
+
       # Attempt to build a retriever for semantic search.
       #
       # Auto-configures from environment variables when no explicit configuration

data/lib/codebase_index/mcp/server.rb

@@ -106,6 +106,16 @@ module CodebaseIndex
           value.is_a?(String) ? [value] : value
         end
 
+        # Coerce a value to an Integer. Converts String representations
+        # to Integer; leaves existing Integers and nil unchanged.
+        # MCP clients may send "2" (string) instead of 2 (integer).
+        #
+        # @param value [String, Integer, nil] The input value
+        # @return [Integer, nil]
+        def coerce_integer(value)
+          value.is_a?(String) ? value.to_i : value
+        end
+
         # Apply offset+limit pagination to a single section key within a container hash.
         # Adds `_total`, `_truncated`, and `_offset` metadata keys when truncating.
         #
@@ -166,6 +176,7 @@ module CodebaseIndex
 
         def define_search_tool(server, reader, respond, renderer)
           coerce = method(:coerce_array)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'search',
             description: 'Search code units by pattern. Matches against identifiers by default; can also search source_code and metadata fields.',
@@ -187,6 +198,7 @@ module CodebaseIndex
           ) do |query:, server_context:, types: nil, fields: nil, limit: nil|
             types = coerce.call(types)
             fields = coerce.call(fields)
+            limit = coerce_int.call(limit)
             results = reader.search(
               query,
               types: types,
@@ -203,6 +215,7 @@ module CodebaseIndex
 
         def define_traversal_tool(server, reader, respond, renderer, name:, description:, reader_method:, render_key:)
           coerce = method(:coerce_array)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: name,
             description: description,
@@ -219,6 +232,7 @@ module CodebaseIndex
             }
           ) do |identifier:, server_context:, depth: nil, types: nil|
             types = coerce.call(types)
+            depth = coerce_int.call(depth)
             result = reader.send(reader_method, identifier, depth: depth || 2, types: types)
             if result[:found] == false
               result[:message] =
@@ -249,6 +263,7 @@ module CodebaseIndex
 
         def define_graph_analysis_tool(server, reader, respond, renderer)
           paginate = method(:paginate_section)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'graph_analysis',
             description: 'Get structural analysis of the dependency graph: orphans, dead ends, hubs, cycles, and bridges.',
@@ -264,6 +279,8 @@ module CodebaseIndex
               }
             }
           ) do |server_context:, analysis: nil, limit: nil, offset: nil|
+            limit = coerce_int.call(limit)
+            offset = coerce_int.call(offset)
             data = reader.graph_analysis
             section = analysis || 'all'
             effective_offset = offset || 0
@@ -290,6 +307,7 @@ module CodebaseIndex
 
         def define_pagerank_tool(server, reader, respond, renderer)
           coerce = method(:coerce_array)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'pagerank',
             description: 'Get PageRank importance scores for code units. Higher scores indicate more structurally important nodes.',
@@ -304,6 +322,7 @@ module CodebaseIndex
             }
           ) do |server_context:, limit: nil, types: nil|
             types = coerce.call(types)
+            limit = coerce_int.call(limit)
             scores = reader.dependency_graph.pagerank
             graph_data = reader.raw_graph_data
             nodes = graph_data['nodes'] || {}
@@ -329,6 +348,7 @@ module CodebaseIndex
         end
 
         def define_framework_tool(server, reader, respond, renderer)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'framework',
             description: 'Search Rails framework source units by concept keyword. Matches against identifier, ' \
@@ -342,6 +362,7 @@ module CodebaseIndex
               required: ['keyword']
             }
           ) do |keyword:, server_context:, limit: nil|
+            limit = coerce_int.call(limit)
             results = reader.framework_sources(keyword, limit: limit || 20)
             respond.call(renderer.render(:framework, {
                                            keyword: keyword,
@@ -353,6 +374,7 @@ module CodebaseIndex
 
         def define_recent_changes_tool(server, reader, respond, renderer)
           coerce = method(:coerce_array)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'recent_changes',
             description: 'List recently modified code units sorted by git last_modified timestamp. ' \
@@ -368,6 +390,7 @@ module CodebaseIndex
             }
           ) do |server_context:, limit: nil, types: nil|
             types = coerce.call(types)
+            limit = coerce_int.call(limit)
             results = reader.recent_changes(limit: limit || 10, types: types)
             respond.call(renderer.render(:recent_changes, {
                                            result_count: results.size,
@@ -395,6 +418,7 @@ module CodebaseIndex
         end
 
         def define_retrieve_tool(server, retriever, respond)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'codebase_retrieve',
             description: 'Retrieve relevant codebase context for a natural language query using semantic search. ' \
@@ -408,6 +432,7 @@ module CodebaseIndex
               required: ['query']
             }
           ) do |query:, server_context:, budget: nil|
+            budget = coerce_int.call(budget)
             if retriever
               result = retriever.retrieve(query, budget: budget || 8000)
               respond.call(result.context)
@@ -423,6 +448,7 @@ module CodebaseIndex
         def define_trace_flow_tool(server, reader, index_dir, respond, renderer)
           require_relative '../flow_assembler'
           require_relative '../dependency_graph'
+          coerce_int = method(:coerce_integer)
 
           server.define_tool(
             name: 'trace_flow',
@@ -441,7 +467,7 @@ module CodebaseIndex
               required: ['entry_point']
             }
           ) do |entry_point:, server_context:, depth: nil|
-            max_depth = depth || 3
+            max_depth = coerce_int.call(depth) || 3
             graph = reader.dependency_graph
 
             assembler = CodebaseIndex::FlowAssembler.new(
@@ -457,6 +483,7 @@ module CodebaseIndex
         end
 
         def define_session_trace_tool(server, reader, respond)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'session_trace',
             description: 'Assemble context from a browser session trace (requires session tracer middleware)',
@@ -469,6 +496,8 @@ module CodebaseIndex
               required: ['session_id']
             }
           ) do |session_id:, server_context:, budget: nil, depth: nil|
+            budget = coerce_int.call(budget)
+            depth = coerce_int.call(depth)
             store = CodebaseIndex.configuration.session_store
             next respond.call(JSON.pretty_generate({ error: 'Session tracer not configured' })) unless store
 
@@ -651,6 +680,7 @@ module CodebaseIndex
         end
 
         def define_retrieval_rate_tool(server, feedback_store, respond)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'retrieval_rate',
             description: 'Record a quality rating for a retrieval result (1-5 scale).',
@@ -665,6 +695,7 @@ module CodebaseIndex
           ) do |query:, score:, server_context:, comment: nil|
             next respond.call('Feedback store is not configured.') unless feedback_store
 
+            score = coerce_int.call(score)
             feedback_store.record_rating(query: query, score: score, comment: comment)
             respond.call(JSON.pretty_generate({ recorded: true, type: 'rating', query: query, score: score }))
           end
@@ -740,6 +771,7 @@ module CodebaseIndex
         end
 
         def define_list_snapshots_tool(server, snapshot_store, respond)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'list_snapshots',
             description: 'List temporal snapshots of past extraction runs, optionally filtered by branch.',
@@ -752,6 +784,7 @@ module CodebaseIndex
           ) do |server_context:, limit: nil, branch: nil|
             next respond.call('Snapshot store is not configured. Set enable_snapshots: true.') unless snapshot_store
 
+            limit = coerce_int.call(limit)
             results = snapshot_store.list(limit: limit || 20, branch: branch)
             respond.call(JSON.pretty_generate({ snapshot_count: results.size, snapshots: results }))
           end
@@ -783,6 +816,7 @@ module CodebaseIndex
         end
 
         def define_unit_history_tool(server, snapshot_store, respond)
+          coerce_int = method(:coerce_integer)
           server.define_tool(
             name: 'unit_history',
             description: 'Show the history of a single unit across extraction snapshots. Tracks when source changed.',
@@ -796,6 +830,7 @@ module CodebaseIndex
           ) do |identifier:, server_context:, limit: nil|
             next respond.call('Snapshot store is not configured. Set enable_snapshots: true.') unless snapshot_store
 
+            limit = coerce_int.call(limit)
             entries = snapshot_store.unit_history(identifier, limit: limit || 20)
             respond.call(JSON.pretty_generate({
                                                 identifier: identifier,

data/lib/codebase_index/temporal/json_snapshot_store.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'digest'
+
+module CodebaseIndex
+  module Temporal
+    # JSON-file-based snapshot store for temporal tracking without SQLite.
+    #
+    # Stores snapshots as individual JSON files in a `snapshots/` subdirectory
+    # of the index output directory. Each file is named by git SHA and contains
+    # manifest metadata plus per-unit content hashes.
+    #
+    # Implements the same public interface as SnapshotStore so the MCP server
+    # tools work identically.
+    #
+    # @example
+    #   store = JsonSnapshotStore.new(dir: '/app/tmp/codebase_index')
+    #   store.capture(manifest, unit_hashes)
+    #   store.list                    # => [{ git_sha: "abc123", ... }]
+    #   store.diff("abc123", "def456") # => { added: [...], modified: [...], deleted: [...] }
+    #
+    class JsonSnapshotStore # rubocop:disable Metrics/ClassLength
+      def initialize(dir:)
+        @dir = File.join(dir, 'snapshots')
+        FileUtils.mkdir_p(@dir)
+      end
+
+      def capture(manifest, unit_hashes)
+        git_sha = mget(manifest, 'git_sha')
+        return nil unless git_sha
+
+        previous = find_latest
+        snapshot = build_snapshot(manifest, git_sha, unit_hashes)
+
+        if previous
+          diff_result = compute_diff(previous[:units], index_units(unit_hashes))
+          snapshot[:units_added] = diff_result[:added].size
+          snapshot[:units_modified] = diff_result[:modified].size
+          snapshot[:units_deleted] = diff_result[:deleted].size
+        end
+
+        write_snapshot(git_sha, snapshot)
+        snapshot.except(:units)
+      end
+
+      def list(limit: 20, branch: nil)
+        snapshots = load_all_summaries
+        snapshots.select! { |s| s[:git_branch] == branch } if branch
+        snapshots.sort_by { |s| s[:extracted_at] || '' }.reverse.first(limit)
+      end
+
+      def find(git_sha)
+        path = snapshot_path(git_sha)
+        return nil unless File.exist?(path)
+
+        data = JSON.parse(File.read(path))
+        symbolize_snapshot(data).except(:units)
+      end
+
+      def diff(sha_a, sha_b)
+        snap_a = load_snapshot_with_units(sha_a)
+        snap_b = load_snapshot_with_units(sha_b)
+
+        return { added: [], modified: [], deleted: [] } unless snap_a && snap_b
+
+        compute_diff(snap_a[:units], snap_b[:units])
+      end
+
+      def unit_history(identifier, limit: 20)
+        snapshots = load_all_with_units
+                    .sort_by { |s| s[:extracted_at] || '' }
+                    .reverse
+                    .first(limit)
+
+        entries = snapshots.filter_map do |snap|
+          unit = snap[:units]&.[](identifier)
+          next unless unit
+
+          {
+            git_sha: snap[:git_sha],
+            extracted_at: snap[:extracted_at],
+            git_branch: snap[:git_branch],
+            unit_type: unit[:unit_type],
+            source_hash: unit[:source_hash],
+            metadata_hash: unit[:metadata_hash],
+            dependencies_hash: unit[:dependencies_hash]
+          }
+        end
+
+        mark_changed_entries(entries)
+      end
+
+      private
+
+      def mget(hash, key)
+        hash[key] || hash[key.to_sym]
+      end
+
+      def build_snapshot(manifest, git_sha, unit_hashes)
+        {
+          git_sha: git_sha,
+          git_branch: mget(manifest, 'git_branch'),
+          extracted_at: mget(manifest, 'extracted_at') || Time.now.iso8601,
+          rails_version: mget(manifest, 'rails_version'),
+          ruby_version: mget(manifest, 'ruby_version'),
+          total_units: mget(manifest, 'total_units') || unit_hashes.size,
+          unit_counts: mget(manifest, 'counts') || {},
+          gemfile_lock_sha: mget(manifest, 'gemfile_lock_sha'),
+          schema_sha: mget(manifest, 'schema_sha'),
+          units_added: 0,
+          units_modified: 0,
+          units_deleted: 0,
+          units: index_units(unit_hashes)
+        }
+      end
+
+      def index_units(unit_hashes)
+        unit_hashes.filter_map do |uh|
+          id = mget(uh, 'identifier')
+          next if id.nil?
+
+          [id, {
+            unit_type: mget(uh, 'type').to_s,
+            source_hash: mget(uh, 'source_hash'),
+            metadata_hash: mget(uh, 'metadata_hash'),
+            dependencies_hash: mget(uh, 'dependencies_hash')
+          }]
+        end.to_h
+      end
+
+      def compute_diff(units_a, units_b) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        added = []
+        modified = []
+        deleted = []
+
+        units_b.each do |identifier, data_b|
+          if units_a.key?(identifier)
+            data_a = units_a[identifier]
+            if data_a[:source_hash] != data_b[:source_hash] ||
+               data_a[:metadata_hash] != data_b[:metadata_hash] ||
+               data_a[:dependencies_hash] != data_b[:dependencies_hash]
+              modified << { identifier: identifier, unit_type: data_b[:unit_type] }
+            end
+          else
+            added << { identifier: identifier, unit_type: data_b[:unit_type] }
+          end
+        end
+
+        units_a.each do |identifier, data_a|
+          deleted << { identifier: identifier, unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
+        end
+
+        { added: added, modified: modified, deleted: deleted }
+      end
+
+      def mark_changed_entries(entries)
+        entries.each_with_index do |entry, i|
+          entry[:changed] = if i == entries.size - 1
+                              true
+                            else
+                              entry[:source_hash] != entries[i + 1][:source_hash]
+                            end
+        end
+        entries
+      end
+
+      def snapshot_path(git_sha)
+        raise ArgumentError, "Invalid git SHA: #{git_sha}" unless git_sha.match?(/\A[0-9a-f]+\z/i)
+
+        File.join(@dir, "#{git_sha}.json")
+      end
+
+      def write_snapshot(git_sha, data)
+        File.write(snapshot_path(git_sha), JSON.pretty_generate(data))
+      end
+
+      def load_snapshot_with_units(git_sha)
+        path = snapshot_path(git_sha)
+        return nil unless File.exist?(path)
+
+        symbolize_snapshot(JSON.parse(File.read(path)))
+      end
+
+      def load_all_summaries
+        Dir.glob(File.join(@dir, '*.json')).filter_map do |path|
+          data = JSON.parse(File.read(path))
+          symbolize_snapshot(data).except(:units)
+        rescue JSON::ParserError => e
+          warn "[CodebaseIndex] Skipping corrupt snapshot #{File.basename(path)}: #{e.message}"
+          nil
+        end
+      end
+
+      def load_all_with_units
+        Dir.glob(File.join(@dir, '*.json')).filter_map do |path|
+          symbolize_snapshot(JSON.parse(File.read(path)))
+        rescue JSON::ParserError => e
+          warn "[CodebaseIndex] Skipping corrupt snapshot #{File.basename(path)}: #{e.message}"
+          nil
+        end
+      end
+
+      def find_latest
+        snapshots = load_all_summaries
+        return nil if snapshots.empty?
+
+        latest = snapshots.max_by { |s| s[:extracted_at] || '' }
+        load_snapshot_with_units(latest[:git_sha])
+      end
+
+      def symbolize_snapshot(data)
+        {
+          git_sha: data['git_sha'],
+          git_branch: data['git_branch'],
+          extracted_at: data['extracted_at'],
+          rails_version: data['rails_version'],
+          ruby_version: data['ruby_version'],
+          total_units: data['total_units'],
+          unit_counts: data['unit_counts'] || {},
+          gemfile_lock_sha: data['gemfile_lock_sha'],
+          schema_sha: data['schema_sha'],
+          units_added: data['units_added'],
+          units_modified: data['units_modified'],
+          units_deleted: data['units_deleted'],
+          units: symbolize_units(data['units'])
+        }
+      end
+
+      def symbolize_units(units)
+        return {} unless units
+
+        units.transform_values do |v|
+          {
+            unit_type: v['unit_type'],
+            source_hash: v['source_hash'],
+            metadata_hash: v['metadata_hash'],
+            dependencies_hash: v['dependencies_hash']
+          }
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CodebaseIndex
-  VERSION = '0.3.1'
+  VERSION = '0.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: codebase_index
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Leah Armstrong
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-03-04 00:00:00.000000000 Z
+date: 2026-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mcp
@@ -227,6 +227,7 @@ files:
 - lib/codebase_index/storage/pgvector.rb
 - lib/codebase_index/storage/qdrant.rb
 - lib/codebase_index/storage/vector_store.rb
+- lib/codebase_index/temporal/json_snapshot_store.rb
 - lib/codebase_index/temporal/snapshot_store.rb
 - lib/codebase_index/token_utils.rb
 - lib/codebase_index/version.rb