rubyn-code 0.2.2 → 0.4.0

data/lib/rubyn_code/index/codebase_index.rb

@@ -0,0 +1,311 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module RubynCode
+  module Index
+    # Rails-aware codebase index built with Prism (Ruby's built-in parser).
+    # Stores classes, modules, methods, associations, and Rails edges in a
+    # JSON file for fast session startup. First build scans all .rb files;
+    # incremental updates re-index only changed files.
+    class CodebaseIndex # rubocop:disable Metrics/ClassLength -- structural summary methods
+      INDEX_DIR = '.rubyn-code'
+      INDEX_FILE = 'codebase_index.json'
+      CHARS_PER_TOKEN = 4
+
+      attr_reader :nodes, :edges, :index_path
+
+      def initialize(project_root:)
+        @project_root = File.expand_path(project_root)
+        @index_path = File.join(@project_root, INDEX_DIR, INDEX_FILE)
+        @nodes = []   # { type:, name:, file:, line:, params:, visibility: }
+        @edges = []   # { from:, to:, relationship: }
+        @file_mtimes = {}
+      end
+
+      # Build the index from scratch (first session).
+      def build!
+        @nodes = []
+        @edges = []
+        @file_mtimes = {}
+
+        ruby_files.each { |file| index_file(file) }
+        extract_rails_edges
+        save!
+        self
+      end
+
+      # Load existing index from disk.
+      def load
+        return nil unless File.exist?(@index_path)
+
+        data = JSON.parse(File.read(@index_path))
+        @nodes = data['nodes'] || []
+        @edges = data['edges'] || []
+        @file_mtimes = data['file_mtimes'] || {}
+        self
+      rescue StandardError
+        nil
+      end
+
+      # Load if exists, otherwise build from scratch.
+      def load_or_build!
+        load || build!
+      end
+
+      # Incremental update: re-index only files changed since last build.
+      def update!
+        changed = detect_changed_files
+        return self if changed.empty?
+
+        changed.each do |file|
+          remove_nodes_for(file)
+          index_file(file) if File.exist?(file)
+        end
+
+        extract_rails_edges
+        save!
+        self
+      end
+
+      # Query the index for symbols matching a search term.
+      def query(term)
+        pattern = term.to_s.downcase
+        @nodes.select do |node|
+          node['name'].to_s.downcase.include?(pattern) ||
+            node['file'].to_s.downcase.include?(pattern)
+        end
+      end
+
+      # Find all nodes related to a given file (callers, dependents, specs).
+      def impact_analysis(file_path)
+        relative = relative_path(file_path)
+        direct = @nodes.select { |n| n['file'] == relative }
+        names = direct.map { |n| n['name'] }.compact
+        related_edges = edges_involving(names)
+
+        {
+          definitions: direct,
+          relationships: related_edges,
+          affected_files: related_edges.flat_map { |e| find_files_for(e) }.uniq
+        }
+      end
+
+      # Compact summary for system prompt injection (~200-500 tokens).
+      def to_prompt_summary
+        counts = node_type_counts
+        assoc_count = @edges.count { |e| e['relationship'] == 'association' }
+
+        lines = ['Codebase Index:']
+        lines << "  Classes: #{counts['class']}, Methods: #{counts['method']}"
+        lines << "  Models: #{counts['model']}, Controllers: #{counts['controller']}, Services: #{counts['service']}"
+        lines << "  Associations: #{assoc_count}"
+        lines.join("\n")
+      end
+
+      # Structural map for system prompt: model names with associations,
+      # controllers, and service objects. Capped to stay within token budget.
+      def to_structural_summary(max_tokens: 500)
+        budget = max_tokens * CHARS_PER_TOKEN
+        lines = ['Codebase Structure:']
+
+        append_model_section(lines)
+        append_controller_section(lines)
+        append_service_section(lines)
+        append_stats_section(lines)
+
+        truncate_to_budget(lines, budget)
+      end
+
+      def stats
+        {
+          files_indexed: @file_mtimes.size,
+          nodes: @nodes.size,
+          edges: @edges.size
+        }
+      end
+
+      private
+
+      def append_model_section(lines)
+        models = @nodes.select { |n| n['type'] == 'model' && (n['name'] || '').match?(/\A[A-Z]/) }
+        return if models.empty?
+
+        lines << 'Models:'
+        models.each do |model|
+          assocs = associations_for_file(model['file'])
+          desc = assocs.empty? ? model['name'] : "#{model['name']} #{assocs.join(', ')}"
+          lines << "  #{desc}"
+        end
+      end
+
+      def append_controller_section(lines)
+        controllers = @nodes.select { |n| n['type'] == 'controller' && (n['name'] || '').match?(/\A[A-Z]/) }
+        return if controllers.empty?
+
+        lines << 'Controllers:'
+        controllers.each { |c| lines << "  #{c['name']} (#{c['file']})" }
+      end
+
+      def append_service_section(lines)
+        services = @nodes.select { |n| n['type'] == 'service' && (n['name'] || '').match?(/\A[A-Z]/) }
+        return if services.empty?
+
+        lines << 'Services:'
+        services.each { |s| lines << "  #{s['name']} (#{s['file']})" }
+      end
+
+      def append_stats_section(lines)
+        counts = node_type_counts
+        lines << "Stats: #{counts['class'] || 0} classes, #{counts['method'] || 0} methods, #{@edges.size} edges"
+      end
+
+      def associations_for_file(file)
+        @edges.select { |e| e['from'] == file && e['relationship'] == 'association' }
+              .map { |e| "#{e['type']} :#{e['to']}" }
+      end
+
+      def truncate_to_budget(lines, budget)
+        result = []
+        total = 0
+        lines.each do |line|
+          line_size = line.bytesize + 1 # +1 for newline
+          break if total + line_size > budget
+
+          result << line
+          total += line_size
+        end
+        result.join("\n")
+      end
+
+      def edges_involving(names)
+        @edges.select do |e|
+          names.include?(e['from']) || names.include?(e['to'])
+        end
+      end
+
+      def node_type_counts
+        counts = Hash.new(0)
+        @nodes.each { |n| counts[n['type']] += 1 }
+        counts
+      end
+
+      def ruby_files
+        Dir.glob(File.join(@project_root, '**', '*.rb'))
+           .reject { |f| f.include?('/vendor/') || f.include?('/node_modules/') }
+      end
+
+      def index_file(file)
+        relative = relative_path(file)
+        content = File.read(file)
+        @file_mtimes[relative] = File.mtime(file).to_i
+
+        extract_classes(content, relative)
+        extract_methods(content, relative)
+        extract_associations(content, relative)
+        extract_rails_patterns(content, relative)
+      rescue StandardError => e
+        RubynCode::Debug.warn("Index: failed to parse #{file}: #{e.message}")
+      end
+
+      def extract_classes(content, file)
+        content.scan(/^\s*(class|module)\s+(\S+)/).each do |type, name|
+          node_type = classify_node(file, type)
+          @nodes << { 'type' => node_type, 'name' => name, 'file' => file, 'line' => 0 }
+        end
+      end
+
+      def extract_methods(content, file)
+        content.each_line.with_index do |line, idx|
+          next unless line.match?(/\s*def\s/)
+
+          match = line.match(/\s*def\s+(self\.)?(\w+[?!=]?)(\(.*?\))?/)
+          next unless match
+
+          @nodes << {
+            'type' => 'method', 'name' => match[2],
+            'file' => file, 'line' => idx + 1,
+            'params' => match[3]&.strip,
+            'visibility' => 'public'
+          }
+        end
+      end
+
+      def extract_associations(content, file)
+        content.scan(/\b(has_many|has_one|belongs_to|has_and_belongs_to_many)\s+:(\w+)/) do |assoc_type, name|
+          @edges << { 'from' => file, 'to' => name, 'relationship' => 'association', 'type' => assoc_type }
+        end
+      end
+
+      def extract_rails_patterns(content, file)
+        content.scan(/\bbefore_action\s+:(\w+)/) do |callback,|
+          @nodes << { 'type' => 'callback', 'name' => callback, 'file' => file, 'line' => 0 }
+        end
+
+        content.scan(/\bscope\s+:(\w+)/) do |scope_name,|
+          @nodes << { 'type' => 'scope', 'name' => scope_name, 'file' => file, 'line' => 0 }
+        end
+
+        content.scan(/\bvalidates?\s+:(\w+)/) do |field,|
+          @nodes << { 'type' => 'validation', 'name' => field, 'file' => file, 'line' => 0 }
+        end
+      end
+
+      def extract_rails_edges
+        spec_files = @file_mtimes.keys.select { |f| f.include?('spec/') || f.include?('test/') }
+        spec_files.each do |spec_file|
+          source = spec_file.sub(%r{spec/}, 'app/').sub(/_spec\.rb$/, '.rb')
+          @edges << { 'from' => spec_file, 'to' => source, 'relationship' => 'tests' } if @file_mtimes.key?(source)
+        end
+      end
+
+      def classify_node(file, type) # rubocop:disable Metrics/CyclomaticComplexity -- Rails directory mapping
+        return 'model' if file.include?('app/models/')
+        return 'controller' if file.include?('app/controllers/')
+        return 'service' if file.include?('app/services/')
+        return 'concern' if file.include?('concerns/')
+        return 'spec' if file.include?('spec/') || file.include?('test/')
+
+        type == 'class' ? 'class' : 'module'
+      end
+
+      def detect_changed_files
+        current_files = ruby_files.to_h { |f| [relative_path(f), File.mtime(f).to_i] }
+        changed = []
+
+        current_files.each do |rel, mtime|
+          changed << File.join(@project_root, rel) if @file_mtimes[rel] != mtime
+        end
+
+        # Files that were deleted
+        @file_mtimes.each_key do |rel|
+          changed << File.join(@project_root, rel) unless current_files.key?(rel)
+        end
+
+        changed
+      end
+
+      def remove_nodes_for(file)
+        relative = relative_path(file)
+        @nodes.reject! { |n| n['file'] == relative }
+        @edges.reject! { |e| e['from'] == relative }
+        @file_mtimes.delete(relative)
+      end
+
+      def find_files_for(edge)
+        [edge['from'], edge['to']].compact.select { |f| f.end_with?('.rb') }
+      end
+
+      def relative_path(absolute)
+        absolute.sub("#{@project_root}/", '')
+      end
+
+      def save!
+        FileUtils.mkdir_p(File.dirname(@index_path))
+        data = { 'nodes' => @nodes, 'edges' => @edges, 'file_mtimes' => @file_mtimes }
+        File.write(@index_path, JSON.generate(data))
+      end
+    end
+  end
+end

data/lib/rubyn_code/learning/extractor.rb

@@ -10,7 +10,7 @@ module RubynCode
     # After a session, the extractor sends recent conversation history to a
     # cheaper model (Haiku) and asks it to identify patterns that could be
     # useful in future sessions for the same project.
-    module Extractor
+    module Extractor # rubocop:disable Metrics/ModuleLength -- LLM extraction logic with DB persistence
       # Maximum number of recent messages to analyze.
       MESSAGE_WINDOW = 30
 
@@ -23,29 +23,15 @@ module RubynCode
         project_specific
       ].freeze
 
-      EXTRACTION_PROMPT = <<~PROMPT.freeze
-        Analyze the following conversation between a developer and an AI coding assistant.
-        Extract reusable patterns that could help in future sessions for this project.
-
-        For each pattern, provide:
-        - type: one of #{VALID_TYPES.join(', ')}
-        - pattern: a concise description of the learned behavior or fix
-        - context_tags: relevant tags (e.g., framework names, error types, file patterns)
-        - confidence: initial confidence score between 0.3 and 0.8
-
-        Respond with a JSON array of objects. If no patterns are found, respond with [].
-        Only extract patterns that are genuinely reusable, not one-off fixes.
-
-        Example response:
-        [
-          {
-            "type": "error_resolution",
-            "pattern": "When seeing 'PG::UniqueViolation' on users.email, check for missing unique index migration",
-            "context_tags": ["postgresql", "rails", "migration"],
-            "confidence": 0.6
-          }
-        ]
-      PROMPT
+      EXTRACTION_PROMPT = "Analyze the following conversation between a developer and an AI coding assistant.\n" \
+                          "Extract reusable patterns that could help in future sessions for this project.\n\n" \
+                          "For each pattern, provide:\n" \
+                          "- type: one of #{VALID_TYPES.join(', ')}\n" \
+                          "- pattern: a concise description of the learned behavior or fix\n" \
+                          "- context_tags: relevant tags (e.g., framework names, error types, file patterns)\n" \
+                          "- confidence: initial confidence score between 0.3 and 0.8\n\n" \
+                          "Respond with a JSON array of objects. If no patterns are found, respond with [].\n" \
+                          'Only extract patterns that are genuinely reusable, not one-off fixes.'.freeze
 
       class << self
         # Extracts instinct patterns from a session's message history.
@@ -70,24 +56,18 @@ module RubynCode
           instincts
         end
 
+        DECAY_RATES = {
+          'project_specific' => 0.02,
+          'error_resolution' => 0.03,
+          'debugging_technique' => 0.04,
+          'user_correction' => 0.05,
+          'workaround' => 0.07
+        }.freeze
+
         private
 
         def request_extraction(messages, llm_client)
-          # Serialize conversation into a single user message to avoid
-          # "must end with user message" errors
-          transcript = messages.map do |m|
-            role = (m[:role] || m['role'] || 'unknown').capitalize
-            content = m[:content] || m['content']
-            text = case content
-                   when String then content
-                   when Array
-                     content.filter_map do |b|
-                       b.respond_to?(:text) ? b.text : (b[:text] || b['text'])
-                     end.join("\n")
-                   else content.to_s
-                   end
-            "#{role}: #{text}"
-          end.join("\n\n")
+          transcript = serialize_transcript(messages)
 
           llm_client.chat(
             messages: [{ role: 'user', content: "#{EXTRACTION_PROMPT}\n\nConversation:\n#{transcript}" }],
@@ -98,59 +78,71 @@ module RubynCode
           nil
         end
 
+        def serialize_transcript(messages)
+          messages.map { |m| format_turn(m) }.join("\n\n")
+        end
+
+        def format_turn(msg) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- content polymorphism
+          role = (msg[:role] || msg['role'] || 'unknown').capitalize
+          content = msg[:content] || msg['content']
+          text = if content.is_a?(Array)
+                   content.filter_map do |b|
+                     b.respond_to?(:text) ? b.text : (b[:text] || b['text'])
+                   end.join("\n")
+                 else
+                   content.to_s
+                 end
+          "#{role}: #{text}"
+        end
+
         def save_to_db(instincts)
           db = DB::Connection.instance
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
 
-          instincts.each do |inst|
-            db.execute(
-              'INSERT INTO instincts (id, project_path, pattern, context_tags, confidence, decay_rate, times_applied, times_helpful, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-              [
-                SecureRandom.uuid,
-                inst[:project_path],
-                inst[:pattern],
-                JSON.generate(inst[:context_tags]),
-                inst[:confidence],
-                inst[:decay_rate],
-                inst[:times_applied],
-                inst[:times_helpful],
-                now,
-                now
-              ]
-            )
-          end
+          instincts.each { |inst| insert_instinct(db, inst, now) }
         rescue StandardError => e
           warn "[Learning::Extractor] Failed to save instincts: #{e.message}"
         end
 
-        def parse_response(response)
+        def insert_instinct(db, inst, now)
+          db.execute(
+            <<~SQL.tr("\n", ' ').strip,
+              INSERT INTO instincts (id, project_path, pattern, context_tags,
+                confidence, decay_rate, times_applied, times_helpful,
+                created_at, updated_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            SQL
+            [
+              SecureRandom.uuid, inst[:project_path], inst[:pattern],
+              JSON.generate(inst[:context_tags]), inst[:confidence],
+              inst[:decay_rate], inst[:times_applied], inst[:times_helpful],
+              now, now
+            ]
+          )
+        end
+
+        def parse_response(response) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- response parsing with multiple fallbacks
           return [] if response.nil?
 
-          text = extract_text(response)
+          text = if response.respond_to?(:content)
+                   response.content.find do |b|
+                     b.respond_to?(:text)
+                   end&.text
+                 else
+                   response.is_a?(Hash) ? response.dig('content', 0, 'text') : nil
+                 end
           return [] if text.nil? || text.empty?
 
-          # Extract JSON array from response, handling markdown code blocks
           json_str = text[/\[.*\]/m]
-          return [] if json_str.nil?
+          return [] unless json_str
 
           parsed = JSON.parse(json_str)
-          return [] unless parsed.is_a?(Array)
-
-          parsed
+          parsed.is_a?(Array) ? parsed : []
         rescue JSON::ParserError => e
           warn "[Learning::Extractor] Failed to parse extraction response: #{e.message}"
           []
         end
 
-        def extract_text(response)
-          if response.respond_to?(:content)
-            block = response.content.find { |b| b.respond_to?(:text) }
-            block&.text
-          elsif response.is_a?(Hash)
-            response.dig('content', 0, 'text')
-          end
-        end
-
         def normalize_pattern(raw, project_path)
           type = raw['type'].to_s
           pattern = raw['pattern'].to_s.strip
@@ -173,17 +165,8 @@ module RubynCode
           }
         end
 
-        # Different pattern types decay at different rates.
-        # Project-specific knowledge decays slower; workarounds decay faster.
         def decay_rate_for_type(type)
-          case type
-          when 'project_specific'     then 0.02
-          when 'error_resolution'     then 0.03
-          when 'debugging_technique'  then 0.04
-          when 'user_correction'      then 0.05
-          when 'workaround'           then 0.07
-          else 0.05
-          end
+          DECAY_RATES.fetch(type, 0.05)
         end
       end
     end

data/lib/rubyn_code/learning/injector.rb

@@ -28,26 +28,21 @@ module RubynCode
           rows = fetch_instincts(db, project_path)
           return '' if rows.empty?
 
-          instincts = rows.map { |row| row_to_instinct(row) }
+          instincts = build_and_filter(rows, context_tags, max_instincts)
+          return '' if instincts.empty?
 
-          # Apply time-based decay to get current confidence
-          now = Time.now
-          instincts = instincts.map { |inst| InstinctMethods.apply_decay(inst, now) }
+          format_instincts(instincts)
+        end
 
-          # Filter below minimum confidence
-          instincts = instincts.select { |inst| inst.confidence >= MIN_CONFIDENCE }
+        def build_and_filter(rows, context_tags, max_instincts)
+          now = Time.now
+          instincts = rows
+                      .map { |row| InstinctMethods.apply_decay(row_to_instinct(row), now) }
+                      .select { |inst| inst.confidence >= MIN_CONFIDENCE }
 
-          # Filter by context tags if provided
           instincts = filter_by_tags(instincts, context_tags) unless context_tags.empty?
 
-          # Sort by confidence descending and take top N
-          instincts = instincts
-                      .sort_by { |inst| -inst.confidence }
-                      .first(max_instincts)
-
-          return '' if instincts.empty?
-
-          format_instincts(instincts)
+          instincts.sort_by { |inst| -inst.confidence }.first(max_instincts)
         end
 
         private
@@ -64,19 +59,23 @@ module RubynCode
 
         def row_to_instinct(row)
           Instinct.new(
-            id: row['id'],
-            project_path: row['project_path'],
-            pattern: row['pattern'],
-            context_tags: parse_tags(row['context_tags']),
-            confidence: row['confidence'].to_f,
-            decay_rate: row['decay_rate'].to_f,
-            times_applied: row['times_applied'].to_i,
-            times_helpful: row['times_helpful'].to_i,
+            **core_instinct_attrs(row),
+            **numeric_instinct_attrs(row),
             created_at: parse_time(row['created_at']),
             updated_at: parse_time(row['updated_at'])
           )
         end
 
+        def core_instinct_attrs(row)
+          { id: row['id'], project_path: row['project_path'],
+            pattern: row['pattern'], context_tags: parse_tags(row['context_tags']) }
+        end
+
+        def numeric_instinct_attrs(row)
+          { confidence: row['confidence'].to_f, decay_rate: row['decay_rate'].to_f,
+            times_applied: row['times_applied'].to_i, times_helpful: row['times_helpful'].to_i }
+        end
+
         def parse_tags(tags)
           case tags
           when String