codebase_index 0.2.1 → 0.3.1

data/lib/codebase_index/resilience/index_validator.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'digest'
+require_relative '../filename_utils'
 
 module CodebaseIndex
   module Resilience
@@ -18,6 +18,8 @@ module CodebaseIndex
     #   report = validator.validate
     #   puts report.errors if !report.valid?
     class IndexValidator
+      include CodebaseIndex::FilenameUtils
+
       # Report produced by {#validate}.
       #
       # @!attribute [r] valid?
@@ -160,26 +162,6 @@ module CodebaseIndex
           warnings << "Stale file not in index: #{type_name}/#{basename}"
         end
       end
-
-      # Convert an identifier to a safe filename (legacy format, mirrors Extractor#safe_filename).
-      #
-      # @param identifier [String] The unit identifier (e.g., "Admin::UsersController")
-      # @return [String] A filesystem-safe filename (e.g., "Admin__UsersController.json")
-      def safe_filename(identifier)
-        "#{identifier.gsub('::', '__').gsub(/[^a-zA-Z0-9_-]/, '_')}.json"
-      end
-
-      # Convert an identifier to a collision-safe filename (current format).
-      # Mirrors {Extractor#collision_safe_filename} — appends a short SHA256 digest
-      # to disambiguate identifiers that normalize to the same safe_filename.
-      #
-      # @param identifier [String] The unit identifier
-      # @return [String] Collision-safe filename (e.g., "Admin__UsersController_a1b2c3d4.json")
-      def collision_safe_filename(identifier)
-        base = identifier.gsub('::', '__').gsub(/[^a-zA-Z0-9_-]/, '_')
-        digest = Digest::SHA256.hexdigest(identifier)[0, 8]
-        "#{base}_#{digest}.json"
-      end
     end
   end
 end

data/lib/codebase_index/retrieval/context_assembler.rb

@@ -54,6 +54,9 @@ module CodebaseIndex
         sources = []
         tokens_used = 0
 
+        # Pre-fetch all candidate metadata in one batch query
+        @unit_cache = @metadata_store.find_batch(candidates.map(&:identifier))
+
         # 1. Structural context (always first if provided)
         tokens_used = add_structural_section(sections, structural_context, tokens_used, effective_budget)
 
@@ -141,7 +144,7 @@ module CodebaseIndex
 
       # Append a single candidate to the section. Returns updated tokens_used, or nil to stop.
       def append_candidate(parts, sources, candidate, budget, tokens_used)
-        unit = @metadata_store.find(candidate.identifier)
+        unit = @unit_cache[candidate.identifier]
         return tokens_used unless unit
 
         text = format_unit(unit, candidate)
@@ -165,10 +168,10 @@ module CodebaseIndex
       # @param candidate [Candidate] The search candidate
       # @return [String]
       def format_unit(unit, _candidate)
-        identifier = unit[:identifier] || unit['identifier']
-        type = unit[:type] || unit['type']
-        file_path = unit[:file_path] || unit['file_path']
-        source = unit[:source_code] || unit['source_code'] || ''
+        identifier = unit_field(unit, :identifier)
+        type = unit_field(unit, :type)
+        file_path = unit_field(unit, :file_path)
+        source = unit_field(unit, :source_code) || ''
 
         <<~UNIT.strip
           ## #{identifier} (#{type})
@@ -184,14 +187,23 @@ module CodebaseIndex
       def build_source_attribution(candidate, unit, truncated: false)
         attribution = {
           identifier: candidate.identifier,
-          type: unit[:type] || unit['type'],
+          type: unit_field(unit, :type),
           score: candidate.score,
-          file_path: unit[:file_path] || unit['file_path']
+          file_path: unit_field(unit, :file_path)
         }
         attribution[:truncated] = true if truncated
         attribution
       end
 
+      # Read a field from a unit hash, accepting either symbol or string keys.
+      #
+      # @param unit [Hash]
+      # @param key [Symbol]
+      # @return [Object, nil]
+      def unit_field(unit, key)
+        unit[key] || unit[key.to_s]
+      end
+
       # Check if a candidate is framework source.
       #
       # @param candidate [Candidate]

data/lib/codebase_index/retrieval/query_classifier.rb

@@ -87,28 +87,30 @@ module CodebaseIndex
       # @param query [String]
       # @return [Symbol]
       def detect_intent(query)
-        INTENT_PATTERNS.each do |intent, pattern|
-          return intent if query.match?(pattern)
-        end
-        :understand # default
+        match_first(INTENT_PATTERNS, query, default: :understand)
       end
 
       # @param query [String]
       # @return [Symbol]
       def detect_scope(query)
-        SCOPE_PATTERNS.each do |scope, pattern|
-          return scope if query.match?(pattern)
-        end
-        :focused # default
+        match_first(SCOPE_PATTERNS, query, default: :focused)
       end
 
       # @param query [String]
       # @return [Symbol, nil]
       def detect_target_type(query)
-        TARGET_PATTERNS.each do |type, pattern|
-          return type if query.match?(pattern)
-        end
-        nil # no specific type detected
+        match_first(TARGET_PATTERNS, query, default: nil)
+      end
+
+      # Match query against a hash of {key => pattern}, returning the first matching key.
+      #
+      # @param patterns [Hash{Symbol => Regexp}]
+      # @param query [String]
+      # @param default [Object] value if no pattern matches
+      # @return [Object]
+      def match_first(patterns, query, default:)
+        patterns.each { |key, pattern| return key if query.match?(pattern) }
+        default
       end
 
       # @param query [String]

data/lib/codebase_index/retrieval/ranker.rb

@@ -102,8 +102,9 @@ module CodebaseIndex
       #
       # @return [Array<Candidate>]
       def rebuild_rrf_candidates(candidates, rrf_scores, metadata_map)
+        original_by_id = candidates.index_by(&:identifier)
         rrf_scores.sort_by { |_id, score| -score }.map do |identifier, score|
-          original = candidates.find { |c| c.identifier == identifier }
+          original = original_by_id[identifier]
           build_candidate(
             identifier: identifier,
             score: score,
@@ -119,8 +120,11 @@ module CodebaseIndex
       # @param classification [QueryClassifier::Classification]
       # @return [Array<Hash>]
       def score_candidates(candidates, classification)
+        # Batch-fetch all metadata in one query instead of per-candidate lookups
+        unit_map = @metadata_store.find_batch(candidates.map(&:identifier))
+
         candidates.map do |candidate|
-          unit = @metadata_store.find(candidate.identifier)
+          unit = unit_map[candidate.identifier]
 
           {
             candidate: candidate,

data/lib/codebase_index/retrieval/search_executor.rb

@@ -33,26 +33,11 @@ module CodebaseIndex
 
       # Strategy mapping from (intent, scope) → strategy.
       #
-      # Pinpoint scope always uses :direct for locate/reference.
-      # Comprehensive/exploratory scopes use :hybrid.
-      # Framework intent always uses :keyword against framework sources.
+      # Covers pinpoint overrides for locate/reference (:direct).
+      # Trace and framework intents are handled before this map is consulted.
       STRATEGY_MAP = {
-        # [intent, scope] => strategy
-        # Pinpoint
         %i[locate pinpoint] => :direct,
-        %i[reference pinpoint] => :direct,
-
-        # Trace always uses graph
-        %i[trace pinpoint] => :graph,
-        %i[trace focused] => :graph,
-        %i[trace exploratory] => :graph,
-        %i[trace comprehensive] => :graph,
-
-        # Framework always keyword
-        %i[framework pinpoint] => :keyword,
-        %i[framework focused] => :keyword,
-        %i[framework exploratory] => :keyword,
-        %i[framework comprehensive] => :keyword
+        %i[reference pinpoint] => :direct
       }.freeze
 
       # @param vector_store [Storage::VectorStore::Interface] Vector store adapter
@@ -93,7 +78,11 @@ module CodebaseIndex
         intent = classification.intent
         scope = classification.scope
 
-        # Check explicit mapping first
+        # Intent-level overrides (apply regardless of scope)
+        return :graph if intent == :trace
+        return :keyword if intent == :framework
+
+        # Pinpoint overrides for locate/reference
         mapped = STRATEGY_MAP[[intent, scope]]
         return mapped if mapped
 

data/lib/codebase_index/retriever.rb

@@ -140,7 +140,7 @@ module CodebaseIndex
       return nil if total.zero?
 
       type_counts = STRUCTURAL_TYPES.filter_map do |type|
-        count = count_by_type(type)
+        count = @metadata_store.find_by_type(type).size
         "#{count} #{type}s" if count.positive?
       end
 
@@ -148,13 +148,5 @@ module CodebaseIndex
     rescue StandardError
       nil
     end
-
-    # Count units of a given type in the metadata store.
-    #
-    # @param type [String] The unit type to count
-    # @return [Integer] Number of units of this type
-    def count_by_type(type)
-      @metadata_store.find_by_type(type).size
-    end
   end
 end

data/lib/codebase_index/ruby_analyzer/class_analyzer.rb

@@ -18,6 +18,7 @@ module CodebaseIndex
     #
     class ClassAnalyzer
       include FqnBuilder
+      include Ast::SourceSpan
 
       # @param parser [Ast::Parser, nil] Parser instance (creates default if nil)
       def initialize(parser: nil)
@@ -43,9 +44,9 @@ module CodebaseIndex
 
         case node.type
         when :class
-          process_class(node, source, file_path, namespace_stack, units)
+          process_definition(node, :ruby_class, source, file_path, namespace_stack, units)
         when :module
-          process_module(node, source, file_path, namespace_stack, units)
+          process_definition(node, :ruby_module, source, file_path, namespace_stack, units)
         else
           (node.children || []).each do |child|
             extract_definitions(child, source, file_path, namespace_stack, units)
@@ -53,14 +54,6 @@ module CodebaseIndex
         end
       end
 
-      def process_class(node, source, file_path, namespace_stack, units)
-        process_definition(node, :ruby_class, source, file_path, namespace_stack, units)
-      end
-
-      def process_module(node, source, file_path, namespace_stack, units)
-        process_definition(node, :ruby_module, source, file_path, namespace_stack, units)
-      end
-
       def process_definition(node, type, source, file_path, namespace_stack, units)
         name = node.method_name
         fqn = build_fqn(name, namespace_stack)
@@ -144,13 +137,7 @@ module CodebaseIndex
 
       # Count def and defs nodes in body children (non-recursive — only direct methods).
       def count_methods(body_children)
-        count = 0
-        body_children.each do |child|
-          next unless child.is_a?(Ast::Node)
-
-          count += 1 if %i[def defs].include?(child.type)
-        end
-        count
+        body_children.count { |child| child.is_a?(Ast::Node) && %i[def defs].include?(child.type) }
       end
 
       # Build the constant name from a :const node (may have receiver for namespaced).
@@ -163,14 +150,7 @@ module CodebaseIndex
 
       # Extract source text for a node using line range.
       def extract_source(node, source)
-        return nil unless node.line && node.end_line
-
-        lines = source.lines
-        start_idx = node.line - 1
-        end_idx = node.end_line - 1
-        return nil if start_idx.negative? || end_idx >= lines.length
-
-        lines[start_idx..end_idx].join
+        extract_source_span(source, node.line, node.end_line)
       end
 
       # Build dependency list from superclass, includes, and extends.

data/lib/codebase_index/ruby_analyzer/dataflow_analyzer.rb

@@ -21,6 +21,11 @@ module CodebaseIndex
       CONSTRUCTION_METHODS = %w[new].freeze
       SERIALIZATION_METHODS = %w[to_h to_json to_a serialize as_json].freeze
       DESERIALIZATION_METHODS = %w[from_json parse].freeze
+      CATEGORY_BY_METHOD = [
+        *CONSTRUCTION_METHODS.map { |m| [m, :construction] },
+        *SERIALIZATION_METHODS.map { |m| [m, :serialization] },
+        *DESERIALIZATION_METHODS.map { |m| [m, :deserialization] }
+      ].to_h.freeze
 
       # @param parser [Ast::Parser, nil] Parser instance (creates default if nil)
       def initialize(parser: nil)
@@ -65,13 +70,7 @@ module CodebaseIndex
       end
 
       def categorize(method_name)
-        if CONSTRUCTION_METHODS.include?(method_name)
-          :construction
-        elsif SERIALIZATION_METHODS.include?(method_name)
-          :serialization
-        elsif DESERIALIZATION_METHODS.include?(method_name)
-          :deserialization
-        end
+        CATEGORY_BY_METHOD[method_name]
       end
     end
   end

data/lib/codebase_index/ruby_analyzer/mermaid_renderer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'set'
+
 module CodebaseIndex
   module RubyAnalyzer
     # Renders Mermaid-format diagrams from extracted units, dependency graphs,
@@ -25,31 +27,24 @@ module CodebaseIndex
         lines = ['graph TD']
         return lines.join("\n") if units.nil? || units.empty?
 
-        seen_nodes = {}
-        edges = []
+        seen_nodes = Set.new
+        seen_edges = Set.new
 
         units.each do |unit|
           node_id = sanitize_id(unit.identifier)
-          unless seen_nodes[node_id]
-            seen_nodes[node_id] = true
-            lines << "  #{node_id}[\"#{escape_label(unit.identifier)}\"]"
-          end
+          lines << "  #{node_id}[\"#{escape_label(unit.identifier)}\"]" if seen_nodes.add?(node_id)
 
           (unit.dependencies || []).each do |dep|
             target = dep[:target] || dep['target']
             next unless target
 
             target_id = sanitize_id(target)
-            unless seen_nodes[target_id]
-              seen_nodes[target_id] = true
-              lines << "  #{target_id}[\"#{escape_label(target)}\"]"
-            end
+            lines << "  #{target_id}[\"#{escape_label(target)}\"]" if seen_nodes.add?(target_id)
 
             via = dep[:via] || dep['via']
             edge_key = "#{node_id}->#{target_id}"
-            next if edges.include?(edge_key)
+            next unless seen_edges.add?(edge_key)
 
-            edges << edge_key
             lines << if via
                        "  #{node_id} -->|#{via}| #{target_id}"
                      else
@@ -95,16 +90,14 @@ module CodebaseIndex
         end
 
         # Render edges
-        seen_edges = []
+        seen_edges = Set.new
         edges.each do |source, targets|
-          targets = Array(targets)
-          targets.each do |target|
+          Array(targets).each do |target|
             next unless nodes.key?(target)
 
             edge_key = "#{sanitize_id(source)}->#{sanitize_id(target)}"
-            next if seen_edges.include?(edge_key)
+            next unless seen_edges.add?(edge_key)
 
-            seen_edges << edge_key
             lines << "  #{sanitize_id(source)} --> #{sanitize_id(target)}"
           end
         end
@@ -123,15 +116,14 @@ module CodebaseIndex
         lines = ['flowchart TD']
         return lines.join("\n") if units.nil? || units.empty?
 
-        seen_nodes = {}
+        seen_nodes = Set.new
 
         units.each do |unit|
           transformations = unit.metadata[:data_transformations] || unit.metadata['data_transformations']
           next unless transformations.is_a?(Array) && transformations.any?
 
           node_id = sanitize_id(unit.identifier)
-          unless seen_nodes[node_id]
-            seen_nodes[node_id] = true
+          if seen_nodes.add?(node_id)
             shape = dataflow_shape(transformations)
             lines << "  #{node_id}#{shape}"
           end
@@ -144,10 +136,7 @@ module CodebaseIndex
             category = (t[:category] || t['category'])&.to_s
             method_name = t[:method] || t['method']
 
-            unless seen_nodes[receiver_id]
-              seen_nodes[receiver_id] = true
-              lines << "  #{receiver_id}[\"#{escape_label(receiver)}\"]"
-            end
+            lines << "  #{receiver_id}[\"#{escape_label(receiver)}\"]" if seen_nodes.add?(receiver_id)
 
             label = [category, method_name].compact.join(': ')
             lines << "  #{node_id} -->|#{label}| #{receiver_id}"
@@ -200,41 +189,57 @@ module CodebaseIndex
         # Analysis summary
         sections << '## Analysis Summary'
         sections << ''
-        if analysis
-          stats = analysis[:stats] || analysis['stats'] || {}
-          sections << "- **Orphans:** #{stats[:orphan_count] || stats['orphan_count'] || 0}"
-          sections << "- **Dead ends:** #{stats[:dead_end_count] || stats['dead_end_count'] || 0}"
-          sections << "- **Hubs:** #{stats[:hub_count] || stats['hub_count'] || 0}"
-          sections << "- **Cycles:** #{stats[:cycle_count] || stats['cycle_count'] || 0}"
-
-          hubs = analysis[:hubs] || analysis['hubs'] || []
-          if hubs.any?
-            sections << ''
-            sections << '### Top Hubs'
-            sections << ''
-            hubs.first(5).each do |hub|
-              id = hub[:identifier] || hub['identifier']
-              count = hub[:dependent_count] || hub['dependent_count']
-              sections << "- #{id} (#{count} dependents)"
-            end
-          end
-
-          cycles = analysis[:cycles] || analysis['cycles'] || []
-          if cycles.any?
-            sections << ''
-            sections << '### Cycles'
-            sections << ''
-            cycles.each do |cycle|
-              sections << "- #{cycle.join(' -> ')}"
-            end
-          end
-        end
+        sections.concat(render_stats_section(analysis))
 
         sections.join("\n")
       end
 
       private
 
+      # Render the Analysis Summary section lines for a given analysis hash.
+      #
+      # @param analysis [Hash, nil] Graph analysis report from GraphAnalyzer#analyze
+      # @return [Array<String>] Lines to append to the architecture document
+      def render_stats_section(analysis)
+        lines = []
+        return lines unless analysis
+
+        stats = analysis[:stats] || analysis['stats'] || {}
+        lines << "- **Orphans:** #{stats[:orphan_count] || stats['orphan_count'] || 0}"
+        lines << "- **Dead ends:** #{stats[:dead_end_count] || stats['dead_end_count'] || 0}"
+        lines << "- **Hubs:** #{stats[:hub_count] || stats['hub_count'] || 0}"
+        lines << "- **Cycles:** #{stats[:cycle_count] || stats['cycle_count'] || 0}"
+
+        hubs = analysis[:hubs] || analysis['hubs'] || []
+        lines.concat(render_hubs_section(hubs))
+
+        cycles = analysis[:cycles] || analysis['cycles'] || []
+        if cycles.any?
+          lines << ''
+          lines << '### Cycles'
+          lines << ''
+          cycles.each { |cycle| lines << "- #{cycle.join(' -> ')}" }
+        end
+
+        lines
+      end
+
+      # Render the Top Hubs subsection lines.
+      #
+      # @param hubs [Array<Hash>] Hub entries with :identifier and :dependent_count keys
+      # @return [Array<String>] Lines to append, or empty array if no hubs
+      def render_hubs_section(hubs)
+        return [] unless hubs.any?
+
+        lines = ['', '### Top Hubs', '']
+        hubs.first(5).each do |hub|
+          id = hub[:identifier] || hub['identifier']
+          count = hub[:dependent_count] || hub['dependent_count']
+          lines << "- #{id} (#{count} dependents)"
+        end
+        lines
+      end
+
       # Sanitize an identifier for use as a Mermaid node ID.
       #
       # Replaces characters that Mermaid cannot use in node IDs with underscores.

data/lib/codebase_index/ruby_analyzer/trace_enricher.rb

@@ -64,19 +64,19 @@ module CodebaseIndex
 
           traces = grouped[key]
 
-          calls = traces.select { |t| t['event'] == 'call' || t[:event] == 'call' }
-          returns = traces.select { |t| t['event'] == 'return' || t[:event] == 'return' }
+          calls = traces.select { |t| fetch_key(t, :event) == 'call' }
+          returns = traces.select { |t| fetch_key(t, :event) == 'return' }
 
           callers = calls.filter_map do |t|
-            caller_class = t['caller_class'] || t[:caller_class]
-            caller_method = t['caller_method'] || t[:caller_method]
+            caller_class = fetch_key(t, :caller_class)
+            caller_method = fetch_key(t, :caller_method)
             next unless caller_class
 
             { 'caller_class' => caller_class, 'caller_method' => caller_method }
           end
 
           return_types = returns.filter_map do |t|
-            t['return_class'] || t[:return_class]
+            fetch_key(t, :return_class)
           end.uniq
 
           unit.metadata[:trace] = {
@@ -90,11 +90,15 @@ module CodebaseIndex
       class << self
         private
 
+        def fetch_key(hash, key)
+          hash[key.to_s] || hash[key.to_sym]
+        end
+
         def group_traces(trace_data)
           grouped = Hash.new { |h, k| h[k] = [] }
           trace_data.each do |trace|
-            class_name = trace['class_name'] || trace[:class_name]
-            method_name = trace['method_name'] || trace[:method_name]
+            class_name = fetch_key(trace, :class_name)
+            method_name = fetch_key(trace, :method_name)
             next unless class_name && method_name
 
             key = "#{class_name}##{method_name}"

data/lib/codebase_index/session_tracer/file_store.rb

@@ -71,14 +71,7 @@ module CodebaseIndex
 
         files.first(limit).map do |file|
           session_id = File.basename(file, '.jsonl')
-          requests = read(session_id)
-
-          {
-            'session_id' => session_id,
-            'request_count' => requests.size,
-            'first_request' => requests.first&.fetch('timestamp', nil),
-            'last_request' => requests.last&.fetch('timestamp', nil)
-          }
+          session_summary(session_id, read(session_id))
         end
       end
 

data/lib/codebase_index/session_tracer/redis_store.rb

@@ -72,13 +72,7 @@ module CodebaseIndex
         expired.each { |id| @redis.srem(SESSIONS_KEY, id) } if expired.any?
 
         active.first(limit).map do |session_id|
-          requests = read(session_id)
-          {
-            'session_id' => session_id,
-            'request_count' => requests.size,
-            'first_request' => requests.first&.fetch('timestamp', nil),
-            'last_request' => requests.last&.fetch('timestamp', nil)
-          }
+          session_summary(session_id, read(session_id))
         end
       end
 

data/lib/codebase_index/session_tracer/session_flow_assembler.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'set'
+require_relative '../token_utils'
 require_relative 'session_flow_document'
 
 module CodebaseIndex
@@ -220,10 +221,10 @@ module CodebaseIndex
           source = unit[:source_code]
           next unless source
 
-          source_tokens = estimate_token_count(source)
+          source_tokens = TokenUtils.estimate_tokens(source)
           unit[:source_code] = "# source truncated (#{source_tokens} tokens)"
           total -= source_tokens
-          total += estimate_token_count(unit[:source_code])
+          total += TokenUtils.estimate_tokens(unit[:source_code])
         end
 
         [total, 0].max
@@ -236,20 +237,10 @@ module CodebaseIndex
       def estimate_tokens(context_pool)
         context_pool.values.sum do |unit|
           source = unit[:source_code] || ''
-          estimate_token_count(source) + 20 # overhead for tags/metadata
+          TokenUtils.estimate_tokens(source) + 20 # overhead for tags/metadata
         end
       end
 
-      # Estimate token count for a string.
-      # Uses project convention: (string.length / 4.0).ceil
-      # See docs/TOKEN_BENCHMARK.md — conservative floor (~10.6% overestimate).
-      #
-      # @param text [String] Text to estimate
-      # @return [Integer] Estimated token count
-      def estimate_token_count(text)
-        (text.length / 4.0).ceil
-      end
-
       # Build an empty document for sessions with no requests.
       #
       # @param session_id [String]

data/lib/codebase_index/session_tracer/solid_cache_store.rb

@@ -73,13 +73,7 @@ module CodebaseIndex
         write_index(active) if active.size != index.size
 
         active.first(limit).map do |session_id|
-          requests = read(session_id)
-          {
-            'session_id' => session_id,
-            'request_count' => requests.size,
-            'first_request' => requests.first&.fetch('timestamp', nil),
-            'last_request' => requests.last&.fetch('timestamp', nil)
-          }
+          session_summary(session_id, read(session_id))
         end
       end
 

data/lib/codebase_index/session_tracer/store.rb

@@ -62,6 +62,20 @@ module CodebaseIndex
       def sanitize_session_id(session_id)
         session_id.to_s.gsub(/[^a-zA-Z0-9_-]/, '_')
       end
+
+      # Build a session summary hash from a session ID and its requests.
+      #
+      # @param session_id [String]
+      # @param requests [Array<Hash>]
+      # @return [Hash]
+      def session_summary(session_id, requests)
+        {
+          'session_id' => session_id,
+          'request_count' => requests.size,
+          'first_request' => requests.first&.fetch('timestamp', nil),
+          'last_request' => requests.last&.fetch('timestamp', nil)
+        }
+      end
     end
   end
 end