codebase_index 0.2.1 → 0.3.1

data/lib/codebase_index/extractors/shared_utility_methods.rb

@@ -20,6 +20,184 @@ module CodebaseIndex
     #   end
     #
     module SharedUtilityMethods
+      # Check whether a path points to application source (under app_root, but
+      # not inside vendor/ or node_modules/ directories).
+      #
+      # In Docker environments where Rails.root is `/app`, a naive
+      # `start_with?(app_root)` also matches vendor bundle paths like
+      # `/app/vendor/bundle/ruby/…`. This helper rejects those.
+      #
+      # @param path [String, nil] Absolute file path
+      # @param app_root [String] Rails.root.to_s
+      # @return [Boolean]
+      def app_source?(path, app_root)
+        return false unless path
+
+        path.start_with?(app_root) && !path.include?('/vendor/') && !path.include?('/node_modules/')
+      end
+
+      # Resolve the source file for a class using reliable introspection,
+      # filtered through {#app_source?} to reject vendor/gem paths.
+      #
+      # Tier order:
+      #   1. +const_source_location+ (returns the class definition site)
+      #   2. Instance method source locations (first match wins)
+      #   3. Class/singleton method source locations (first match wins)
+      #
+      # @param klass [Class, Module] The class to resolve
+      # @param app_root [String] Rails.root.to_s
+      # @param fallback [String] Path to return when resolution fails
+      # @return [String] Resolved source path or fallback
+      def resolve_source_location(klass, app_root:, fallback:)
+        # Tier 1: const_source_location (most reliable — returns class definition site)
+        if Object.respond_to?(:const_source_location) && klass.name
+          loc = Object.const_source_location(klass.name)&.first
+          return loc if app_source?(loc, app_root)
+        end
+
+        # Tier 2: Instance methods defined directly on this class
+        klass.instance_methods(false).each do |method_name|
+          loc = klass.instance_method(method_name).source_location&.first
+          return loc if app_source?(loc, app_root)
+        end
+
+        # Tier 3: Class/singleton methods defined on this class
+        klass.methods(false).each do |method_name|
+          loc = klass.method(method_name).source_location&.first
+          return loc if app_source?(loc, app_root)
+        end
+
+        fallback
+      rescue StandardError
+        fallback
+      end
+
+      # Extract the primary class name from source or fall back to a file path convention.
+      #
+      # @param file_path [String] Absolute path to the Ruby file
+      # @param source [String] Ruby source code
+      # @param dir_prefix [String] Regex fragment matching the app/ subdirectory to strip
+      #   (e.g., "policies", "validators", "(?:services|interactors|operations|commands|use_cases)")
+      # @return [String] The class name
+      def extract_class_name(file_path, source, dir_prefix)
+        return ::Regexp.last_match(1) if source =~ /^\s*class\s+([\w:]+)/
+
+        file_path.sub("#{Rails.root}/", '').sub(%r{^app/#{dir_prefix}/}, '').sub('.rb', '').camelize
+      end
+
+      # Extract the parent class name from a class definition.
+      #
+      # @param source [String] Ruby source code
+      # @return [String, nil] Parent class name or nil
+      def extract_parent_class(source)
+        match = source.match(/^\s*class\s+[\w:]+\s*<\s*([\w:]+)/)
+        match ? match[1] : nil
+      end
+
+      # Count non-blank, non-comment lines of code.
+      #
+      # @param source [String] Ruby source code
+      # @return [Integer] LOC count
+      def count_loc(source)
+        source.lines.count { |l| l.strip.length.positive? && !l.strip.start_with?('#') }
+      end
+
+      # Skip module-only files (concerns, base modules without a class).
+      #
+      # @param source [String] Ruby source code
+      # @return [Boolean]
+      def skip_file?(source)
+        source.match?(/^\s*module\s+[\w:]+\s*$/) && !source.match?(/^\s*class\s+/)
+      end
+
+      # Extract custom error/exception class names defined inline.
+      #
+      # @param source [String] Ruby source code
+      # @return [Array<String>] Custom error class names
+      def extract_custom_errors(source)
+        source.scan(/class\s+(\w+(?:Error|Exception))\s*</).flatten
+      end
+
+      # Detect common entry point methods in a source file.
+      #
+      # @param source [String] Ruby source code
+      # @return [Array<String>] Entry point method names
+      def detect_entry_points(source)
+        points = []
+        points << 'call'    if source.match?(/def (self\.)?call\b/)
+        points << 'perform' if source.match?(/def (self\.)?perform\b/)
+        points << 'execute' if source.match?(/def (self\.)?execute\b/)
+        points << 'run'     if source.match?(/def (self\.)?run\b/)
+        points << 'process' if source.match?(/def (self\.)?process\b/)
+        points.empty? ? ['unknown'] : points
+      end
+
+      # Extract :only/:except action lists and :if/:unless conditions from a callback.
+      #
+      # Modern Rails (4.2+) stores conditions in @if/@unless ivar arrays.
+      # ActionFilter objects hold action Sets; other conditions are procs/symbols.
+      #
+      # @param callback [ActiveSupport::Callbacks::Callback]
+      # @return [Array(Array<String>, Array<String>, Array<String>, Array<String>)]
+      #   [only_actions, except_actions, if_labels, unless_labels]
+      def extract_callback_conditions(callback)
+        if_conditions = callback.instance_variable_get(:@if) || []
+        unless_conditions = callback.instance_variable_get(:@unless) || []
+
+        only = []
+        except = []
+        if_labels = []
+        unless_labels = []
+
+        if_conditions.each do |cond|
+          actions = extract_action_filter_actions(cond)
+          if actions
+            only.concat(actions)
+          else
+            if_labels << condition_label(cond)
+          end
+        end
+
+        unless_conditions.each do |cond|
+          actions = extract_action_filter_actions(cond)
+          if actions
+            except.concat(actions)
+          else
+            unless_labels << condition_label(cond)
+          end
+        end
+
+        [only, except, if_labels, unless_labels]
+      end
+
+      # Extract action names from an ActionFilter-like condition object.
+      # Duck-types on the @actions ivar being a Set, avoiding dependence
+      # on private class names across Rails versions.
+      #
+      # @param condition [Object] A condition from the callback's @if/@unless array
+      # @return [Array<String>, nil] Action names, or nil if not an ActionFilter
+      def extract_action_filter_actions(condition)
+        return nil unless condition.instance_variable_defined?(:@actions)
+
+        actions = condition.instance_variable_get(:@actions)
+        return nil unless actions.is_a?(Set)
+
+        actions.to_a
+      end
+
+      # Human-readable label for a non-ActionFilter condition.
+      #
+      # @param condition [Object] A proc, symbol, or other condition
+      # @return [String]
+      def condition_label(condition)
+        case condition
+        when Symbol then ":#{condition}"
+        when Proc then 'Proc'
+        when String then condition
+        else condition.class.name
+        end
+      end
+
       # Extract namespace from a class name string or class object.
       #
       # Handles both string input (e.g., "Payments::StripeService")
@@ -70,11 +248,15 @@ module CodebaseIndex
         source.scan(/def\s+self\.(\w+[?!=]?)/).flatten
       end
 
-      # Extract initialize parameters from source code.
+      # Extract initialize parameters from source code via regex.
       #
       # Parses the parameter list of the initialize method to determine
       # parameter names, defaults, and whether they are keyword arguments.
       #
+      # Note: PhlexExtractor and ViewComponentExtractor override this with a
+      # runtime-introspection version that takes a Class object instead of source
+      # text, providing richer type information (:req, :opt, :keyreq, :rest, etc.).
+      #
       # @param source [String] Ruby source code
       # @return [Array<Hash>] Parameter info hashes with :name, :has_default, :keyword
       def extract_initialize_params(source)

data/lib/codebase_index/extractors/validator_extractor.rb

@@ -54,7 +54,7 @@ module CodebaseIndex
       # @return [ExtractedUnit, nil] The extracted unit or nil if not a validator
       def extract_validator_file(file_path)
         source = File.read(file_path)
-        class_name = extract_class_name(file_path, source)
+        class_name = extract_class_name(file_path, source, 'validators')
 
         return nil unless class_name
         return nil unless validator_file?(source)
@@ -82,16 +82,6 @@ module CodebaseIndex
       # Class Discovery
       # ──────────────────────────────────────────────────────────────────────
 
-      def extract_class_name(file_path, source)
-        return ::Regexp.last_match(1) if source =~ /^\s*class\s+([\w:]+)/
-
-        file_path
-          .sub("#{Rails.root}/", '')
-          .sub(%r{^app/validators/}, '')
-          .sub('.rb', '')
-          .camelize
-      end
-
       def validator_file?(source)
         source.match?(/< ActiveModel::Validator/) ||
           source.match?(/< ActiveModel::EachValidator/) ||
@@ -176,7 +166,7 @@ module CodebaseIndex
 
       def extract_error_messages(source)
         # errors.add(:attr, "message") or errors.add(variable, "message")
-        messages = source.scan(/errors\.add\s*\(\s*:?\w+\s*,\s*["']([^"']+)["']/).flatten.map { |m| m }
+        messages = source.scan(/errors\.add\s*\(\s*:?\w+\s*,\s*["']([^"']+)["']/).flatten
 
         # errors.add(:attr, :symbol) or errors.add(variable, :symbol)
         source.scan(/errors\.add\s*\(\s*:?\w+\s*,\s*:(\w+)/).flatten.each { |m| messages << ":#{m}" }
@@ -186,7 +176,7 @@ module CodebaseIndex
 
       def extract_options(source)
         # options[:key] access
-        options = source.scan(/options\[:(\w+)\]/).flatten.map { |o| o }
+        options = source.scan(/options\[:(\w+)\]/).flatten
 
         options.uniq
       end
@@ -200,10 +190,6 @@ module CodebaseIndex
         inferred.empty? ? [] : [inferred]
       end
 
-      def extract_custom_errors(source)
-        source.scan(/class\s+(\w+(?:Error|Exception))\s*</).flatten
-      end
-
       # ──────────────────────────────────────────────────────────────────────
       # Dependency Extraction
       # ──────────────────────────────────────────────────────────────────────

data/lib/codebase_index/extractors/view_component_extractor.rb

@@ -56,8 +56,12 @@ module CodebaseIndex
           file_path: source_file_for(component)
         )
 
-        unit.namespace = extract_namespace(component)
+        # Skip components with no resolvable source file (framework/internal)
+        return nil unless unit.file_path
+
         unit.source_code = read_source(unit.file_path)
+
+        unit.namespace = extract_namespace(component)
         unit.metadata = extract_metadata(component, unit.source_code)
         unit.dependencies = extract_dependencies(component, unit.source_code)
 
@@ -86,7 +90,10 @@ module CodebaseIndex
         defined?(ViewComponent::Preview) && klass < ViewComponent::Preview
       end
 
-      # Locate the source file for a component class
+      # Locate the source file for a component class.
+      #
+      # Convention paths first, then introspection via {#resolve_source_location}
+      # which filters out vendor/node_modules paths.
       #
       # @param component [Class]
       # @return [String, nil]
@@ -99,13 +106,7 @@ module CodebaseIndex
         found = possible_paths.find { |p| File.exist?(p) }
         return found.to_s if found
 
-        # Fall back to method source location
-        if component.instance_methods(false).any?
-          method = component.instance_methods(false).first
-          component.instance_method(method).source_location&.first
-        end
-      rescue StandardError
-        nil
+        resolve_source_location(component, app_root: Rails.root.to_s, fallback: nil)
       end
 
       # @param file_path [String, nil]

data/lib/codebase_index/filename_utils.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module CodebaseIndex
+  # Shared filename helpers for converting unit identifiers to safe filenames.
+  #
+  # Used by Extractor (writing) and IndexValidator (reading) to ensure
+  # filename generation is consistent across both sides.
+  module FilenameUtils
+    # Convert an identifier to a safe filename (legacy format).
+    #
+    # @param identifier [String] The unit identifier (e.g., "Admin::UsersController")
+    # @return [String] A filesystem-safe filename (e.g., "Admin__UsersController.json")
+    def safe_filename(identifier)
+      "#{identifier.gsub('::', '__').gsub(/[^a-zA-Z0-9_-]/, '_')}.json"
+    end
+
+    # Convert an identifier to a collision-safe filename (current format).
+    #
+    # Appends a short SHA256 digest to disambiguate identifiers that normalize
+    # to the same safe_filename.
+    #
+    # @param identifier [String] The unit identifier
+    # @return [String] Collision-safe filename (e.g., "Admin__UsersController_a1b2c3d4.json")
+    def collision_safe_filename(identifier)
+      base = identifier.gsub('::', '__').gsub(/[^a-zA-Z0-9_-]/, '_')
+      digest = Digest::SHA256.hexdigest(identifier)[0, 8]
+      "#{base}_#{digest}.json"
+    end
+  end
+end

data/lib/codebase_index/flow_analysis/operation_extractor.rb

@@ -185,10 +185,7 @@ module CodebaseIndex
 
       # Detect response calls (render, redirect_to, head, render_*).
       def response_call?(node)
-        return true if RESPONSE_METHODS.include?(node.method_name)
-        return true if node.method_name&.start_with?('render')
-
-        false
+        RESPONSE_METHODS.include?(node.method_name) || node.method_name&.start_with?('render')
       end
 
       # Detect dynamic dispatch (send, public_send).

data/lib/codebase_index/formatting/base.rb

@@ -25,16 +25,6 @@ module CodebaseIndex
       def format(_assembled_context)
         raise NotImplementedError, "#{self.class}#format must be implemented"
       end
-
-      private
-
-      # Estimate token count using the project convention.
-      #
-      # @param text [String]
-      # @return [Integer]
-      def estimate_tokens(text)
-        (text.length / 4.0).ceil
-      end
     end
   end
 end

data/lib/codebase_index/graph_analyzer.rb

@@ -89,7 +89,7 @@ module CodebaseIndex
           dependents: dependents
         }
       end
-      identifiers_with_dependents.sort_by { |h| -h[:dependent_count] }.first(limit)
+      identifiers_with_dependents.max_by(limit) { |h| h[:dependent_count] }
     end
 
     # Detect circular dependency chains in the graph.

data/lib/codebase_index/mcp/bootstrapper.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module MCP
+    # Shared setup logic for MCP server executables.
+    #
+    # Validates the index directory, checks for a manifest, and builds
+    # an optional retriever for semantic search — all duplicated between
+    # the stdio and HTTP server entry points.
+    #
+    module Bootstrapper
+      # Resolve and validate the index directory from CLI args or environment.
+      #
+      # @param argv [Array<String>] Command-line arguments
+      # @return [String] Validated index directory path
+      def self.resolve_index_dir(argv)
+        dir = argv[0] || ENV['CODEBASE_INDEX_DIR'] || Dir.pwd
+
+        unless Dir.exist?(dir)
+          warn "Error: Index directory does not exist: #{dir}"
+          exit 1
+        end
+
+        unless File.exist?(File.join(dir, 'manifest.json'))
+          warn "Error: No manifest.json found in: #{dir}"
+          warn 'Run `bundle exec rake codebase_index:extract` in your Rails app first.'
+          exit 1
+        end
+
+        dir
+      end
+
+      # Attempt to build a retriever for semantic search.
+      #
+      # Auto-configures from environment variables when no explicit configuration
+      # exists. Returns nil if embedding is unavailable or setup fails.
+      #
+      # @return [CodebaseIndex::Retriever, nil]
+      def self.build_retriever
+        config = CodebaseIndex.configuration
+
+        openai_key = ENV.fetch('OPENAI_API_KEY', nil)
+        if !config.embedding_provider && openai_key
+          config.vector_store = :in_memory
+          config.metadata_store = :in_memory
+          config.graph_store = :in_memory
+          config.embedding_provider = :openai
+          config.embedding_options = { api_key: openai_key }
+        end
+
+        CodebaseIndex::Builder.new(config).build_retriever if config.embedding_provider
+      rescue StandardError => e
+        warn "Note: Semantic search unavailable (#{e.message}). Using pattern-based search only."
+        nil
+      end
+    end
+  end
+end

data/lib/codebase_index/mcp/renderers/markdown_renderer.rb

@@ -53,18 +53,20 @@ module CodebaseIndex
         # @param data [Hash] Search results with :query, :result_count, :results
         # @return [String] Markdown search results
         def render_search(data, **)
+          query = fetch_key(data, :query)
+          count = fetch_key(data, :result_count, 0)
+          results = fetch_key(data, :results, [])
+
           lines = []
-          lines << "## Search: \"#{data[:query] || data['query']}\""
-          count = data[:result_count] || data['result_count'] || 0
+          lines << "## Search: \"#{query}\""
           lines << ''
           lines << "#{count} result#{'s' unless count == 1} found."
           lines << ''
 
-          results = data[:results] || data['results'] || []
           results.each do |r|
-            ident = r['identifier'] || r[:identifier]
-            type = r['type'] || r[:type]
-            match = r['match_field'] || r[:match_field]
+            ident = fetch_key(r, :identifier)
+            type = fetch_key(r, :type)
+            match = fetch_key(r, :match_field)
             line = "- **#{ident}** (#{type})"
             line += " — matched in #{match}" if match
             lines << line
@@ -92,7 +94,7 @@ module CodebaseIndex
         # @param data [Hash] Structure data with :manifest and optional :summary
         # @return [String] Markdown structure overview
         def render_structure(data, **)
-          manifest = data[:manifest] || data['manifest'] || {}
+          manifest = fetch_key(data, :manifest, {})
           lines = []
           lines << '## Codebase Structure'
           lines << ''
@@ -113,7 +115,7 @@ module CodebaseIndex
             lines << ''
           end
 
-          summary = data[:summary] || data['summary']
+          summary = fetch_key(data, :summary)
           if summary
             lines << '### Summary'
             lines << ''
@@ -132,14 +134,14 @@ module CodebaseIndex
           lines << '## Graph Analysis'
           lines << ''
 
-          stats = data['stats'] || data[:stats]
+          stats = fetch_key(data, :stats)
           if stats.is_a?(Hash)
             stats.each { |k, v| lines << "- **#{k}:** #{v}" }
             lines << ''
           end
 
           %w[orphans dead_ends hubs cycles bridges].each do |section|
-            items = data[section] || data[section.to_sym]
+            items = fetch_key(data, section)
             next unless items.is_a?(Array) && items.any?
 
             lines << "### #{section.tr('_', ' ').capitalize}"
@@ -171,17 +173,14 @@ module CodebaseIndex
           lines = []
           lines << '## PageRank Scores'
           lines << ''
-          lines << "#{data[:total_nodes] || data['total_nodes']} nodes in graph."
+          lines << "#{fetch_key(data, :total_nodes)} nodes in graph."
           lines << ''
           lines << '| Rank | Identifier | Type | Score |'
           lines << '|------|-----------|------|-------|'
 
-          results = data[:results] || data['results'] || []
+          results = fetch_key(data, :results, [])
           results.each_with_index do |r, i|
-            ident = r[:identifier] || r['identifier']
-            type = r[:type] || r['type']
-            score = r[:score] || r['score']
-            lines << "| #{i + 1} | #{ident} | #{type} | #{score} |"
+            lines << "| #{i + 1} | #{fetch_key(r, :identifier)} | #{fetch_key(r, :type)} | #{fetch_key(r, :score)} |"
           end
 
           lines.join("\n").rstrip
@@ -192,19 +191,20 @@ module CodebaseIndex
         # @param data [Hash] Framework search results
         # @return [String] Markdown framework results
         def render_framework(data, **)
+          keyword = fetch_key(data, :keyword)
+          count = fetch_key(data, :result_count, 0)
+          results = fetch_key(data, :results, [])
+
           lines = []
-          keyword = data[:keyword] || data['keyword']
-          count = data[:result_count] || data['result_count'] || 0
           lines << "## Framework: \"#{keyword}\""
           lines << ''
           lines << "#{count} result#{'s' unless count == 1} found."
           lines << ''
 
-          results = data[:results] || data['results'] || []
           results.each do |r|
-            ident = r['identifier'] || r[:identifier]
-            type = r['type'] || r[:type]
-            file = r['file_path'] || r[:file_path]
+            ident = fetch_key(r, :identifier)
+            type = fetch_key(r, :type)
+            file = fetch_key(r, :file_path)
             line = "- **#{ident}** (#{type})"
             line += " — `#{file}`" if file
             lines << line
@@ -218,8 +218,10 @@ module CodebaseIndex
         # @param data [Hash] Recent changes with :result_count and :results
         # @return [String] Markdown table of recent changes
         def render_recent_changes(data, **)
+          count = fetch_key(data, :result_count, 0)
+          results = fetch_key(data, :results, [])
+
           lines = []
-          count = data[:result_count] || data['result_count'] || 0
           lines << '## Recent Changes'
           lines << ''
           lines << "#{count} recently modified unit#{'s' unless count == 1}."
@@ -227,12 +229,11 @@ module CodebaseIndex
           lines << '| Identifier | Type | Last Modified | Author |'
           lines << '|-----------|------|---------------|--------|'
 
-          results = data[:results] || data['results'] || []
           results.each do |r|
-            ident = r['identifier'] || r[:identifier]
-            type = r['type'] || r[:type]
-            modified = r['last_modified'] || r[:last_modified] || '-'
-            author = r['author'] || r[:author] || '-'
+            ident = fetch_key(r, :identifier)
+            type = fetch_key(r, :type)
+            modified = fetch_key(r, :last_modified) || '-'
+            author = fetch_key(r, :author) || '-'
             lines << "| #{ident} | #{type} | #{modified} | #{author} |"
           end
 
@@ -257,23 +258,23 @@ module CodebaseIndex
         private
 
         def render_traversal(label, data)
-          root = data[:root] || data['root']
+          root = fetch_key(data, :root)
           found = data[:found] || data['found']
-          nodes = data[:nodes] || data['nodes'] || {}
-          message = data[:message] || data['message']
+          nodes = fetch_key(data, :nodes, {})
+          message = fetch_key(data, :message)
 
           lines = []
           lines << "## #{label} of #{root}"
           lines << ''
 
           if found == false
-            (lines << message) || "Identifier '#{root}' not found in the index."
+            lines << (message || "Identifier '#{root}' not found in the index.")
             return lines.join("\n").rstrip
           end
 
           nodes.each do |id, info|
-            depth = info['depth'] || info[:depth] || 0
-            deps = info['deps'] || info[:deps] || []
+            depth = fetch_key(info, :depth) || 0
+            deps = fetch_key(info, :deps, [])
             indent = '  ' * depth
             lines << "#{indent}- **#{id}**"
             deps.each { |d| lines << "#{indent}  - #{d}" }