rails_lens 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b17142784a27f1ffd45512665eea05a135e1c97b0e1dcc38505f8a07b067ad0
-  data.tar.gz: 9ae11ad7619a74b5662a8b6bb2e767c33ad1523f5c6a2cf590cb809389d0ab9e
+  metadata.gz: b150328b6771f2de4015820d40e3942c10b41903687f800fd4b068f79605deeb
+  data.tar.gz: 2d67fc7fd815b1af80c3cb9e1ad8b94e1166129fc3a1eb24e28958ad3e625b47
 SHA512:
-  metadata.gz: 7ebe5bfc79a4415138fc65f03f393796780e514a32f5226c0fbf66acb2d4d6dd2ac5e1b8878429f3383023f7489b5b40a35d9930876d8ffecf563a14f8bdbfb5
-  data.tar.gz: 2b40b476f42536db544130081063e3a0e992bbeda51d4ff993b4b3be3d0b8073a5ea00c3df98d57321de995d3dca6120f335fc9c2fca652abe3794cc4c704cdd
+  metadata.gz: 0b9f944f8e7be0f11789817a49f168f7be310e6742207897029c20540b701c525da861a1b49cdcbeee0690a9bd6424fbace720e6bb42fd5d9d81b46cd244ed88
+  data.tar.gz: 22f3bc6e6223ad726c2fc3cf617c37af1e587e466119fd1138965e38ff51cb9128b441b5081214e79f9e2d334b8ec7990e437bdfb20257783a40a194f572ebfc

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.2.5](https://github.com/seuros/rails_lens/compare/rails_lens/v0.2.4...rails_lens/v0.2.5) (2025-07-31)
+
+
+### Bug Fixes
+
+* change strategy to cover database with hundrends of connections pool ([#12](https://github.com/seuros/rails_lens/issues/12)) ([7054d35](https://github.com/seuros/rails_lens/commit/7054d3582bfee41f0050725c2bd23e80c5898486))
+
 ## [0.2.4](https://github.com/seuros/rails_lens/compare/rails_lens/v0.2.3...rails_lens/v0.2.4) (2025-07-31)
 
 

data/lib/rails_lens/model_detector.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'concurrent'
-
 module RailsLens
   class ModelDetector
     class << self
@@ -198,7 +196,10 @@ module RailsLens
 
         # Exclude abstract models and models without valid tables
         before_count = models.size
-        models = filter_models_concurrently(models, trace_filtering, options)
+
+        # Use connection management during model filtering to prevent connection exhaustion
+        models = filter_models_with_connection_management(models, trace_filtering, options)
+
         log_filter_step('Abstract/invalid table removal', before_count, models.size, trace_filtering)
 
         # Exclude tables from configuration
@@ -257,59 +258,178 @@ module RailsLens
       end
 
       def filter_models_concurrently(models, trace_filtering, options = {})
-        # Use concurrent futures to check table existence in parallel
-        futures = models.map do |model|
-          Concurrent::Future.execute do
-            should_exclude = false
-            reason = nil
-
-            begin
-              # Skip abstract models unless explicitly included
-              if model.abstract_class? && !options[:include_abstract]
-                should_exclude = true
-                reason = 'abstract class'
-              # For abstract models that are included, skip table checks
-              elsif model.abstract_class? && options[:include_abstract]
-                reason = 'abstract class (included)'
-              # Skip models without configured tables
-              elsif !model.table_name
-                should_exclude = true
-                reason = 'no table name'
-              # Skip models whose tables don't exist
-              elsif !model.table_exists?
-                should_exclude = true
-                reason = "table '#{model.table_name}' does not exist"
-              # Additional check: Skip models that don't have any columns
-              elsif model.columns.empty?
-                should_exclude = true
-                reason = "table '#{model.table_name}' has no columns"
-              else
-                reason = "table '#{model.table_name}' exists with #{model.columns.size} columns"
-              end
-            rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotDefined => e
+        puts "ModelDetector: Sequential filtering #{models.size} models..." if options[:verbose]
+        # Process models sequentially to prevent concurrent database connections
+        results = models.map do |model|
+          should_exclude = false
+          reason = nil
+
+          begin
+            # Skip abstract models unless explicitly included
+            if model.abstract_class? && !options[:include_abstract]
+              should_exclude = true
+              reason = 'abstract class'
+            # For abstract models that are included, skip table checks
+            elsif model.abstract_class? && options[:include_abstract]
+              reason = 'abstract class (included)'
+            # Skip models without configured tables
+            elsif !model.table_name
               should_exclude = true
-              reason = "database error checking model - #{e.message}"
-            rescue NameError, NoMethodError => e
+              reason = 'no table name'
+            # Skip models whose tables don't exist
+            elsif !model.table_exists?
               should_exclude = true
-              reason = "method error checking model - #{e.message}"
-            rescue StandardError => e
-              # Catch any other errors and exclude the model to prevent ERD corruption
+              reason = "table '#{model.table_name}' does not exist"
+            # Additional check: Skip models that don't have any columns
+            elsif model.columns.empty?
               should_exclude = true
-              reason = "unexpected error checking model - #{e.message}"
+              reason = "table '#{model.table_name}' has no columns"
+            else
+              reason = "table '#{model.table_name}' exists with #{model.columns.size} columns"
             end
+          rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotDefined => e
+            should_exclude = true
+            reason = "database error checking model - #{e.message}"
+          rescue NameError, NoMethodError => e
+            should_exclude = true
+            reason = "method error checking model - #{e.message}"
+          rescue StandardError => e
+            # Catch any other errors and exclude the model to prevent ERD corruption
+            should_exclude = true
+            reason = "unexpected error checking model - #{e.message}"
+          end
 
-            if trace_filtering
-              action = should_exclude ? 'Excluding' : 'Keeping'
-              Rails.logger.debug { "[ModelDetector] #{action} #{model.name}: #{reason}" }
+          if trace_filtering
+            action = should_exclude ? 'Excluding' : 'Keeping'
+            Rails.logger.debug { "[ModelDetector] #{action} #{model.name}: #{reason}" }
+          end
+
+          { model: model, exclude: should_exclude }
+        end
+
+        # Filter out excluded models
+        results.reject { |result| result[:exclude] }.pluck(:model)
+      end
+
+      def filter_models_with_connection_management(models, trace_filtering, options = {})
+        # Group models by connection pool first
+        models_by_pool = models.group_by do |model|
+          model.connection_pool
+        rescue StandardError
+          nil
+        end
+
+        # Assign orphaned models to primary pool
+        if models_by_pool[nil]&.any?
+          begin
+            primary_pool = ApplicationRecord.connection_pool
+            models_by_pool[primary_pool] ||= []
+            models_by_pool[primary_pool].concat(models_by_pool[nil])
+            models_by_pool.delete(nil)
+          rescue StandardError
+            # Keep orphaned models for individual processing
+          end
+        end
+
+        all_pools = models_by_pool.keys.compact
+        valid_models = []
+
+        models_by_pool.each do |pool, pool_models|
+          if pool
+            # Disconnect other pools before processing this one
+            all_pools.each do |p|
+              next if p == pool
+
+              begin
+                p.disconnect! if p.connected?
+              rescue StandardError
+                # Ignore disconnect errors
+              end
             end
 
-            { model: model, exclude: should_exclude }
+            # Process models with managed connection
+            pool.with_connection do |connection|
+              pool_models.each do |model|
+                result = filter_single_model(model, connection, trace_filtering, options)
+                valid_models << model unless result[:exclude]
+              end
+            end
+          else
+            # Fallback for models without pools
+            pool_models.each do |model|
+              result = filter_single_model(model, nil, trace_filtering, options)
+              valid_models << model unless result[:exclude]
+            end
           end
         end
 
-        # Wait for all futures to complete and filter results
-        results = futures.map(&:value!)
-        results.reject { |result| result[:exclude] }.pluck(:model)
+        valid_models
+      end
+
+      def filter_single_model(model, connection, trace_filtering, options)
+        should_exclude = false
+        reason = nil
+
+        begin
+          # Skip abstract models unless explicitly included
+          if model.abstract_class? && !options[:include_abstract]
+            should_exclude = true
+            reason = 'abstract class'
+          # For abstract models that are included, skip table checks
+          elsif model.abstract_class? && options[:include_abstract]
+            reason = 'abstract class (included)'
+          # Skip models without configured tables
+          elsif !model.table_name
+            should_exclude = true
+            reason = 'no table name'
+          # Skip models whose tables don't exist (use connection if available)
+          elsif connection ? !table_exists_with_connection?(model, connection) : !model.table_exists?
+            should_exclude = true
+            reason = "table '#{model.table_name}' does not exist"
+          # Additional check: Skip models that don't have any columns
+          elsif connection ? columns_empty_with_connection?(model, connection) : model.columns.empty?
+            should_exclude = true
+            reason = "table '#{model.table_name}' has no columns"
+          else
+            column_count = connection ? get_column_count_with_connection(model, connection) : model.columns.size
+            reason = "table '#{model.table_name}' exists with #{column_count} columns"
+          end
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotDefined => e
+          should_exclude = true
+          reason = "database error checking model - #{e.message}"
+        rescue NameError, NoMethodError => e
+          should_exclude = true
+          reason = "method error checking model - #{e.message}"
+        rescue StandardError => e
+          # Catch any other errors and exclude the model to prevent corruption
+          should_exclude = true
+          reason = "unexpected error checking model - #{e.message}"
+        end
+
+        if trace_filtering
+          action = should_exclude ? 'Excluding' : 'Keeping'
+          Rails.logger.debug { "[ModelDetector] #{action} #{model.name}: #{reason}" }
+        end
+
+        { model: model, exclude: should_exclude }
+      end
+
+      def table_exists_with_connection?(model, connection)
+        connection.table_exists?(model.table_name)
+      rescue StandardError
+        false
+      end
+
+      def columns_empty_with_connection?(model, connection)
+        connection.columns(model.table_name).empty?
+      rescue StandardError
+        true
+      end
+
+      def get_column_count_with_connection(model, connection)
+        connection.columns(model.table_name).size
+      rescue StandardError
+        0
       end
 
       def has_sti_column?(model)

data/lib/rails_lens/providers/schema_provider.rb

@@ -12,8 +12,12 @@ module RailsLens
       end
 
       def process(model_class, connection = nil)
-        # Use passed connection or fall back to model's connection
-        conn = connection || model_class.connection
+        # Always require a connection to be passed to prevent connection pool exhaustion
+        if connection.nil?
+          raise ArgumentError, 'SchemaProvider requires a connection to be passed to prevent connection pool exhaustion'
+        end
+
+        conn = connection
 
         if model_class.abstract_class?
           # For abstract classes, show database connection information in TOML format

data/lib/rails_lens/schema/annotation_manager.rb

@@ -85,8 +85,33 @@ module RailsLens
             end
           end
         else
-          # Fall back to normal processing without connection management
-          results = pipeline.process(model_class)
+          # Fallback: Use the model's connection pool with proper management
+          # This path is used when annotating individual models
+          warn "Using fallback connection management for #{model_class.name}" if RailsLens.config.verbose
+
+          # Force connection management even in fallback mode
+          results = { schema: nil, sections: [], notes: [] }
+
+          model_class.connection_pool.with_connection do |connection|
+            pipeline.instance_variable_get(:@providers).each do |provider|
+              next unless provider.applicable?(model_class)
+
+              begin
+                result = provider.process(model_class, connection)
+
+                case provider.type
+                when :schema
+                  results[:schema] = result
+                when :section
+                  results[:sections] << result if result
+                when :notes
+                  results[:notes].concat(Array(result))
+                end
+              rescue StandardError => e
+                warn "Provider #{provider.class} error for #{model_class}: #{e.message}"
+              end
+            end
+          end
         end
 
         annotation = Annotation.new
@@ -117,7 +142,13 @@ module RailsLens
       end
 
       def self.annotate_all(options = {})
+        # Convert models option to include option for ModelDetector
+        if options[:models]
+          options[:include] = options[:models]
+        end
+
         models = ModelDetector.detect_models(options)
+        puts "Detected #{models.size} models for annotation" if options[:verbose]
 
         # Filter abstract classes based on options
         if options[:include_abstract]
@@ -133,13 +164,35 @@ module RailsLens
 
         # Group models by their connection pool to process each database separately
         models_by_connection_pool = models.group_by do |model|
-          model.connection_pool
-        rescue StandardError
-          nil # Models without connection pools will be processed separately
+          pool = model.connection_pool
+          pool
+        rescue StandardError => e
+          puts "Model #{model.name} -> NO POOL (#{e.message})" if options[:verbose]
+          nil # Models without connection pools will use primary pool
         end
 
+        # Force models without connection pools to use the primary connection pool
+        if models_by_connection_pool[nil]&.any?
+          begin
+            primary_pool = ApplicationRecord.connection_pool
+            models_by_connection_pool[primary_pool] ||= []
+            models_by_connection_pool[primary_pool].concat(models_by_connection_pool[nil])
+            models_by_connection_pool.delete(nil)
+          rescue StandardError => e
+            puts "Failed to assign to primary pool: #{e.message}" if options[:verbose]
+          end
+        end
+
+        # Get all connection pools first
+        all_pools = get_all_connection_pools(models_by_connection_pool)
+
+        # Log initial connection status (removed verbose output)
+
         models_by_connection_pool.each do |connection_pool, pool_models|
           if connection_pool
+            # Disconnect all OTHER connection pools before processing this one
+            disconnect_other_pools(connection_pool, all_pools, options)
+
             # Process all models for this database using a single connection
             connection_pool.with_connection do |connection|
               pool_models.each do |model|
@@ -147,9 +200,20 @@ module RailsLens
               end
             end
           else
-            # Process models without connection pools individually
-            pool_models.each do |model|
-              process_model_with_connection(model, nil, results, options)
+            # This should not happen anymore since we assign orphaned models to primary pool
+            # Use primary connection pool as fallback to avoid creating new connections
+            begin
+              primary_pool = ApplicationRecord.connection_pool
+              primary_pool.with_connection do |connection|
+                pool_models.each do |model|
+                  process_model_with_connection(model, connection, results, options)
+                end
+              end
+            rescue StandardError => e
+              # Last resort: process without connection management (will create multiple connections)
+              pool_models.each do |model|
+                process_model_with_connection(model, nil, results, options)
+              end
             end
           end
         end
@@ -167,7 +231,6 @@ module RailsLens
         # Skip models without tables or with missing tables (but not abstract classes)
         unless model.abstract_class? || model.table_exists?
           results[:skipped] << model.name
-          warn "Skipping #{model.name} - table does not exist" if options[:verbose]
           return
         end
 
@@ -191,10 +254,9 @@ module RailsLens
         else
           results[:skipped] << model.name
         end
-      rescue ActiveRecord::StatementInvalid => e
+      rescue ActiveRecord::StatementInvalid
         # Handle database-related errors (missing tables, schemas, etc.)
         results[:skipped] << model.name
-        warn "Skipping #{model.name} - database error: #{e.message}" if options[:verbose]
       rescue StandardError => e
         model_name = if model.is_a?(Class) && model.respond_to?(:name)
                        model.name
@@ -222,6 +284,45 @@ module RailsLens
         results
       end
 
+      def self.disconnect_other_pools(current_pool, all_pools, options = {})
+        all_pools.each do |pool|
+          next if pool == current_pool || pool.nil?
+
+          begin
+            if pool.connected?
+              pool.disconnect!
+            end
+          rescue StandardError => e
+            warn "Failed to disconnect pool: #{e.message}"
+          end
+        end
+      end
+
+      def self.get_all_connection_pools(models_by_pool)
+        models_by_pool.keys.compact
+      end
+
+      def self.log_connection_status(all_pools, options = {})
+        return unless options[:verbose]
+
+        puts "\n=== Connection Pool Status ==="
+        all_pools.each do |pool|
+          next unless pool
+
+          begin
+            name = pool.db_config&.name || 'unknown'
+            connected = pool.connected?
+            size = pool.size
+            checked_out = pool.stat[:busy]
+
+            puts "Pool #{name}: connected=#{connected}, size=#{size}, busy=#{checked_out}"
+          rescue StandardError => e
+            puts "Pool status error: #{e.message}"
+          end
+        end
+        puts "================================\n"
+      end
+
       private
 
       def add_annotation(content, _file_path = nil)

data/lib/rails_lens/tasks/schema.rake

@@ -9,7 +9,14 @@ namespace :rails_lens do
       options = {}
       options[:include_abstract] = true if ENV['INCLUDE_ABSTRACT'] == 'true'
 
-      puts 'Annotating models with schema information...'
+      # Support model filtering via environment variable
+      if ENV['MODELS']
+        model_list = ENV['MODELS'].split(',').map(&:strip)
+        options[:models] = model_list
+        puts "Filtering to specific models: #{model_list.join(', ')}"
+      end
+
+      options[:verbose] = true # Force verbose mode to see connection management
       results = RailsLens.annotate_models(options)
 
       if results[:annotated].any?

data/lib/rails_lens/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsLens
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_lens
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Abdelkader Boudih