codebase_index 0.1.0

data/lib/codebase_index/extractors/view_template_extractor.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Extractors
+    # ViewTemplateExtractor handles ERB view template extraction.
+    #
+    # Scans `app/views/` for `.html.erb` and `.erb` files and produces
+    # one ExtractedUnit per template. Extracts render calls (partials),
+    # instance variables, and helper method usage. Links partials via
+    # dependencies and infers the owning controller from directory structure.
+    #
+    # This is an ERB-only MVP — HAML, Slim, and layout inheritance
+    # are not yet supported.
+    #
+    # @example
+    #   extractor = ViewTemplateExtractor.new
+    #   units = extractor.extract_all
+    #   index = units.find { |u| u.identifier == "users/index.html.erb" }
+    #
+    class ViewTemplateExtractor
+      # Directories to scan for view templates
+      VIEW_DIRECTORIES = %w[
+        app/views
+      ].freeze
+
+      # Common Rails view helper methods to detect
+      COMMON_HELPERS = %w[
+        link_to
+        button_to
+        form_for
+        form_with
+        form_tag
+        image_tag
+        stylesheet_link_tag
+        javascript_include_tag
+        content_for
+        yield
+        render
+        redirect_to
+        truncate
+        pluralize
+        number_to_currency
+        number_to_percentage
+        number_with_delimiter
+        time_ago_in_words
+        distance_of_time_in_words
+        simple_format
+        sanitize
+        raw
+        safe_join
+        content_tag
+        tag
+        mail_to
+        url_for
+        asset_path
+        asset_url
+      ].freeze
+
+      def initialize
+        @directories = VIEW_DIRECTORIES.map { |d| Rails.root.join(d) }
+                                       .select(&:directory?)
+      end
+
+      # Extract all ERB view templates
+      #
+      # @return [Array<ExtractedUnit>] List of view template units
+      def extract_all
+        @directories.flat_map do |dir|
+          erb_files = Dir[dir.join('**/*.html.erb')] + Dir[dir.join('**/*.erb')]
+          erb_files.uniq.filter_map do |file|
+            extract_view_template_file(file)
+          end
+        end
+      end
+
+      # Extract a single view template file
+      #
+      # @param file_path [String] Path to the ERB template file
+      # @return [ExtractedUnit, nil] The extracted unit or nil if not ERB
+      def extract_view_template_file(file_path)
+        return nil unless file_path.end_with?('.erb')
+
+        source = File.read(file_path)
+        identifier = build_identifier(file_path)
+        namespace = extract_view_namespace(file_path)
+
+        unit = ExtractedUnit.new(
+          type: :view_template,
+          identifier: identifier,
+          file_path: file_path
+        )
+
+        unit.namespace = namespace
+        unit.source_code = source
+        unit.metadata = build_metadata(source, file_path)
+        unit.dependencies = build_dependencies(source, file_path, identifier)
+
+        unit
+      rescue StandardError => e
+        Rails.logger.error("Failed to extract view template #{file_path}: #{e.message}")
+        nil
+      end
+
+      private
+
+      # Build a readable identifier from the file path.
+      #
+      # @param file_path [String] Absolute path to the template
+      # @return [String] Relative identifier like "users/index.html.erb"
+      def build_identifier(file_path)
+        relative = file_path.sub("#{Rails.root}/", '')
+        relative.sub(%r{^app/views/}, '')
+      end
+
+      # Extract namespace from directory structure.
+      #
+      # @param file_path [String] Absolute path
+      # @return [String, nil] Namespace like "users" or "admin/users"
+      def extract_view_namespace(file_path)
+        identifier = build_identifier(file_path)
+        dir = File.dirname(identifier)
+        dir == '.' ? nil : dir
+      end
+
+      # Build metadata hash for the template.
+      #
+      # @param source [String] Template source code
+      # @param file_path [String] Path to the template
+      # @return [Hash]
+      def build_metadata(source, file_path)
+        {
+          template_engine: 'erb',
+          is_partial: partial?(file_path),
+          partials_rendered: extract_rendered_partials(source),
+          instance_variables: extract_instance_variables(source),
+          helpers_called: extract_helpers(source),
+          loc: source.lines.count { |l| l.strip.length.positive? }
+        }
+      end
+
+      # Check if a template is a partial (filename starts with _).
+      #
+      # @param file_path [String] Path to the template
+      # @return [Boolean]
+      def partial?(file_path)
+        File.basename(file_path).start_with?('_')
+      end
+
+      # Extract partial names from render calls.
+      #
+      # Matches:
+      # - render partial: 'foo/bar'
+      # - render 'foo/bar'
+      # - render :foo
+      #
+      # @param source [String] Template source code
+      # @return [Array<String>] Partial names
+      def extract_rendered_partials(source)
+        partials = Set.new
+
+        # render partial: 'path/to/partial'
+        source.scan(/render\s+partial:\s*['"]([^'"]+)['"]/).each do |match|
+          partials << match[0]
+        end
+
+        # render 'path/to/partial' (string without keyword)
+        source.scan(/render\s+['"]([^'"]+)['"]/).each do |match|
+          partials << match[0]
+        end
+
+        # render :symbol
+        source.scan(/render\s+:(\w+)/).each do |match|
+          partials << match[0]
+        end
+
+        partials.to_a
+      end
+
+      # Extract instance variables used in the template.
+      #
+      # @param source [String] Template source code
+      # @return [Array<String>] Instance variable names
+      def extract_instance_variables(source)
+        source.scan(/@[a-zA-Z_]\w*/).uniq.sort
+      end
+
+      # Extract common Rails helper calls from the template.
+      #
+      # @param source [String] Template source code
+      # @return [Array<String>] Helper method names
+      def extract_helpers(source)
+        found = Set.new
+        COMMON_HELPERS.each do |helper|
+          found << helper if source.match?(/\b#{Regexp.escape(helper)}\b/)
+        end
+        found.to_a.sort
+      end
+
+      # Build dependencies for the template.
+      #
+      # @param source [String] Template source code
+      # @param file_path [String] Path to the template
+      # @param identifier [String] Template identifier
+      # @return [Array<Hash>]
+      def build_dependencies(source, file_path, identifier)
+        deps = []
+
+        # Rendered partials
+        extract_rendered_partials(source).each do |partial_name|
+          partial_identifier = resolve_partial_identifier(partial_name, identifier)
+          deps << { type: :view_template, target: partial_identifier, via: :render }
+        end
+
+        # Inferred controller
+        controller = infer_controller(file_path)
+        deps << { type: :controller, target: controller, via: :view_render } if controller
+
+        deps
+      end
+
+      # Resolve a partial name to its file identifier.
+      #
+      # Given a render call like `render 'comments/comment'`, resolves to
+      # `comments/_comment.html.erb`.
+      #
+      # @param partial_name [String] The partial name from the render call
+      # @param current_identifier [String] The current template's identifier
+      # @return [String] Resolved partial identifier
+      def resolve_partial_identifier(partial_name, current_identifier)
+        if partial_name.include?('/')
+          dir = File.dirname(partial_name)
+          base = File.basename(partial_name)
+          "#{dir}/_#{base}.html.erb"
+        else
+          dir = File.dirname(current_identifier)
+          if dir == '.'
+            "_#{partial_name}.html.erb"
+          else
+            "#{dir}/_#{partial_name}.html.erb"
+          end
+        end
+      end
+
+      # Infer the controller class from the template's directory path.
+      #
+      # @param file_path [String] Path to the template
+      # @return [String, nil] Controller class name
+      def infer_controller(file_path)
+        namespace = extract_view_namespace(file_path)
+        return nil unless namespace
+
+        # Skip layout-only directories
+        return nil if namespace == 'layouts'
+
+        parts = namespace.split('/')
+        controller_name = parts.map { |p| p.split('_').map(&:capitalize).join }.join('::')
+        "#{controller_name}Controller"
+      end
+    end
+  end
+end

data/lib/codebase_index/feedback/gap_detector.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module CodebaseIndex
+  module Feedback
+    # Detects patterns in retrieval feedback that suggest coverage gaps.
+    #
+    # Analyzes ratings and gap reports to find:
+    # - Repeated low-score queries with common keywords
+    # - Frequently reported missing units
+    #
+    # @example
+    #   detector = GapDetector.new(feedback_store: store)
+    #   issues = detector.detect
+    #   issues.each { |i| puts "#{i[:type]}: #{i[:description]}" }
+    #
+    class GapDetector
+      LOW_SCORE_THRESHOLD = 2
+      MIN_PATTERN_COUNT = 2
+      MIN_GAP_COUNT = 2
+
+      # @param feedback_store [Feedback::Store]
+      def initialize(feedback_store:)
+        @feedback_store = feedback_store
+      end
+
+      # Detect coverage gaps from accumulated feedback.
+      #
+      # @return [Array<Hash>] List of detected issues with :type, :description, and details
+      def detect
+        issues = []
+        issues.concat(detect_low_score_patterns)
+        issues.concat(detect_frequently_missing)
+        issues
+      end
+
+      private
+
+      # Find keyword patterns in low-scoring queries.
+      #
+      # @return [Array<Hash>]
+      def detect_low_score_patterns
+        low_ratings = @feedback_store.ratings.select { |r| r['score'] <= LOW_SCORE_THRESHOLD }
+        return [] if low_ratings.size < MIN_PATTERN_COUNT
+
+        keyword_counts = count_keywords(low_ratings)
+        keyword_counts.select { |_, count| count >= MIN_PATTERN_COUNT }.map do |keyword, count|
+          {
+            type: :repeated_low_scores,
+            pattern: keyword,
+            count: count,
+            description: "#{count} low-score queries mention '#{keyword}'"
+          }
+        end
+      end
+
+      # Count keyword occurrences across low-scoring query texts.
+      #
+      # @param ratings [Array<Hash>] Low-score rating entries
+      # @return [Hash<String, Integer>] Keyword => occurrence count
+      def count_keywords(ratings)
+        counts = Hash.new(0)
+        ratings.each do |rating|
+          words = rating['query'].to_s.downcase.split(/\W+/).reject { |w| w.length < 3 }
+          words.each { |w| counts[w] += 1 }
+        end
+        counts
+      end
+
+      # Find units that are frequently reported as missing.
+      #
+      # @return [Array<Hash>]
+      def detect_frequently_missing
+        unit_counts = Hash.new(0)
+        @feedback_store.gaps.each do |gap|
+          unit_counts[gap['missing_unit']] += 1
+        end
+
+        unit_counts.select { |_, count| count >= MIN_GAP_COUNT }.map do |unit, count|
+          {
+            type: :frequently_missing,
+            unit: unit,
+            count: count,
+            description: "#{unit} reported missing #{count} times"
+          }
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/feedback/store.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module CodebaseIndex
+  module Feedback
+    # Append-only JSONL file for retrieval feedback: ratings and gap reports.
+    #
+    # Each line is a JSON object with a `type` field ("rating" or "gap")
+    # plus type-specific fields.
+    #
+    # @example
+    #   store = Store.new(path: '/tmp/feedback.jsonl')
+    #   store.record_rating(query: "How does User work?", score: 4)
+    #   store.record_gap(query: "payments", missing_unit: "PaymentService", unit_type: "service")
+    #   store.average_score  # => 4.0
+    #
+    class Store
+      # @param path [String] Path to the JSONL file
+      def initialize(path:)
+        @path = path
+      end
+
+      # Record a retrieval quality rating.
+      #
+      # @param query [String] The original query
+      # @param score [Integer] Rating 1-5
+      # @param comment [String, nil] Optional comment
+      # @return [void]
+      def record_rating(query:, score:, comment: nil)
+        unless score.is_a?(Integer) && (1..5).cover?(score)
+          raise ArgumentError, "score must be an Integer between 1 and 5, got: #{score.inspect}"
+        end
+
+        entry = {
+          type: 'rating',
+          query: query,
+          score: score,
+          comment: comment,
+          timestamp: Time.now.iso8601
+        }
+        append(entry)
+      end
+
+      # Record a missing unit gap report.
+      #
+      # @param query [String] The query that had poor results
+      # @param missing_unit [String] Identifier of the expected but missing unit
+      # @param unit_type [String] Expected type of the missing unit
+      # @return [void]
+      def record_gap(query:, missing_unit:, unit_type:)
+        entry = {
+          type: 'gap',
+          query: query,
+          missing_unit: missing_unit,
+          unit_type: unit_type,
+          timestamp: Time.now.iso8601
+        }
+        append(entry)
+      end
+
+      # Read all feedback entries.
+      #
+      # @param limit [Integer, nil] Maximum number of entries to return. Returns all if nil.
+      # @return [Array<Hash>]
+      def all_entries(limit: nil)
+        return [] unless File.exist?(@path)
+
+        entries = []
+        File.foreach(@path) do |line|
+          entry = JSON.parse(line.strip)
+          entries << entry
+          break if limit && entries.size >= limit
+        rescue JSON::ParserError
+          next
+        end
+        entries
+      end
+
+      # Filter to rating entries only.
+      #
+      # @return [Array<Hash>]
+      def ratings
+        all_entries.select { |e| e['type'] == 'rating' }
+      end
+
+      # Filter to gap report entries only.
+      #
+      # @return [Array<Hash>]
+      def gaps
+        all_entries.select { |e| e['type'] == 'gap' }
+      end
+
+      # Average score across all ratings.
+      #
+      # @return [Float, nil] Average score, or nil if no ratings
+      def average_score
+        scores = ratings.map { |r| r['score'] }
+        return nil if scores.empty?
+
+        scores.sum.to_f / scores.size
+      end
+
+      private
+
+      # Append a JSON entry as a new line.
+      #
+      # @param entry [Hash]
+      # @return [void]
+      def append(entry)
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.open(@path, 'a') do |f|
+          f.puts(JSON.generate(entry))
+        end
+      end
+    end
+  end
+end

data/lib/codebase_index/flow_analysis/operation_extractor.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require_relative '../ast/parser'
+require_relative '../ast/node'
+require_relative '../ast/call_site_extractor'
+require_relative 'response_code_mapper'
+
+module CodebaseIndex
+  module FlowAnalysis
+    # Extracts operations from a method body AST in source line order.
+    #
+    # Uses Ast::CallSiteExtractor for raw call sites, then classifies each
+    # into domain-meaningful operation types: calls, transactions, async
+    # enqueues, responses, conditionals, cycles, and dynamic dispatch.
+    #
+    # @example Extracting operations from a method body
+    #   parser = Ast::Parser.new
+    #   root = parser.parse(source)
+    #   method_node = root.find_all(:def).find { |n| n.method_name == "create" }
+    #   ops = OperationExtractor.new.extract(method_node)
+    #   ops.first[:type] #=> :call
+    #
+    class OperationExtractor
+      TRANSACTION_METHODS = %w[transaction with_lock].freeze
+      ASYNC_METHODS = %w[perform_async perform_later perform_in perform_at].freeze
+      RESPONSE_METHODS = %w[redirect_to head respond_with].freeze
+      DYNAMIC_DISPATCH_METHODS = %w[send public_send].freeze
+
+      # Extract operations from a method definition node in source line order.
+      #
+      # @param method_node [Ast::Node] A :def or :defs node
+      # @return [Array<Hash>] Operations ordered by source line
+      def extract(method_node)
+        return [] unless method_node.is_a?(Ast::Node)
+
+        operations = []
+        walk(method_node, operations)
+        operations
+      end
+
+      private
+
+      # Recursively walk the AST and extract operations.
+      def walk(node, operations)
+        return unless node.is_a?(Ast::Node)
+
+        case node.type
+        when :block
+          handle_block(node, operations)
+        when :send
+          handle_send(node, operations)
+        when :if
+          handle_conditional(node, operations)
+        when :case
+          handle_case(node, operations)
+        else
+          walk_children(node, operations)
+        end
+      end
+
+      # Walk all children of a node.
+      def walk_children(node, operations)
+        return unless node.children
+
+        node.children.each { |child| walk(child, operations) }
+      end
+
+      # Handle :block nodes - check for transaction/with_lock wrappers.
+      def handle_block(node, operations)
+        send_child = node.children&.first
+        if send_child.is_a?(Ast::Node) && send_child.type == :send && transaction_call?(send_child)
+          nested = []
+          # Walk block body (children after the send node)
+          node.children&.drop(1)&.each { |child| walk(child, nested) }
+
+          operations << {
+            type: :transaction,
+            receiver: send_child.receiver,
+            line: send_child.line,
+            nested: nested
+          }
+        else
+          # Non-transaction block: emit the send as a normal call, walk body
+          handle_send(send_child, operations) if send_child.is_a?(Ast::Node) && send_child.type == :send
+          node.children&.drop(1)&.each { |child| walk(child, operations) }
+        end
+      end
+
+      # Handle :send nodes - classify into operation types.
+      def handle_send(node, operations)
+        return unless node.is_a?(Ast::Node) && node.type == :send
+
+        if async_call?(node)
+          operations << {
+            type: :async,
+            target: node.receiver,
+            method: node.method_name,
+            args_hint: node.arguments || [],
+            line: node.line
+          }
+        elsif dynamic_dispatch?(node)
+          operations << {
+            type: :dynamic_dispatch,
+            target: node.receiver,
+            method: node.method_name,
+            args_hint: node.arguments || [],
+            line: node.line
+          }
+        elsif response_call?(node)
+          operations << {
+            type: :response,
+            status_code: ResponseCodeMapper.resolve_method(node.method_name, arguments: node.arguments || []),
+            render_method: node.method_name,
+            line: node.line
+          }
+        elsif significant_call?(node)
+          operations << {
+            type: :call,
+            target: node.receiver,
+            method: node.method_name,
+            line: node.line
+          }
+        end
+
+        # Do NOT recurse into send node children — the walker handles
+        # children at the statement level. Recursing here would double-count
+        # chained calls and pick up receiver lvars as spurious method calls.
+      end
+
+      # Handle :if nodes - extract conditional with then/else branches.
+      def handle_conditional(node, operations)
+        then_ops = []
+        else_ops = []
+
+        # children[0] = condition, children[1] = then, children[2] = else
+        children = node.children || []
+        walk(children[1], then_ops) if children[1].is_a?(Ast::Node)
+        walk(children[2], else_ops) if children[2].is_a?(Ast::Node)
+
+        return if then_ops.empty? && else_ops.empty?
+
+        condition_text = if children[0].is_a?(Ast::Node)
+                           children[0].to_source
+                         elsif children[0].is_a?(String)
+                           children[0]
+                         end
+
+        operations << {
+          type: :conditional,
+          kind: 'if',
+          condition: condition_text || node.source,
+          line: node.line,
+          then_ops: then_ops,
+          else_ops: else_ops
+        }
+      end
+
+      # Handle :case nodes as a conditional variant.
+      def handle_case(node, operations)
+        # Treat case as a conditional - extract ops from all branches
+        branch_ops = []
+        walk_children(node, branch_ops)
+
+        return if branch_ops.empty?
+
+        operations << {
+          type: :conditional,
+          kind: 'case',
+          condition: node.source,
+          line: node.line,
+          then_ops: branch_ops,
+          else_ops: []
+        }
+      end
+
+      # Detect transaction/with_lock calls.
+      def transaction_call?(node)
+        TRANSACTION_METHODS.include?(node.method_name)
+      end
+
+      # Detect async enqueue calls.
+      def async_call?(node)
+        ASYNC_METHODS.include?(node.method_name)
+      end
+
+      # Detect response calls (render, redirect_to, head, render_*).
+      def response_call?(node)
+        return true if RESPONSE_METHODS.include?(node.method_name)
+        return true if node.method_name&.start_with?('render')
+
+        false
+      end
+
+      # Detect dynamic dispatch (send, public_send).
+      def dynamic_dispatch?(node)
+        DYNAMIC_DISPATCH_METHODS.include?(node.method_name)
+      end
+
+      # Determine if a call is significant enough to include.
+      def significant_call?(node)
+        return false if node.method_name.nil?
+        return false if Ast::INSIGNIFICANT_METHODS.include?(node.method_name)
+        return false if transaction_call?(node) # Handled by block wrapper
+
+        true
+      end
+    end
+  end
+end