activerecord-graph-extractor 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f64bcc21e756d5c43f0718c2e582f1f39a71f2a612f1a1c15103690fd6c82989
-  data.tar.gz: 8b0a59753bf91dab1f7fe04859edeb67c9bd920cbb0f515ac5c422a806a8eeaa
+  metadata.gz: 34558c2d8251545d4a78aa2db20225c01c4118fdd35246d5c27110913b7c70fc
+  data.tar.gz: 12dc7966fc513148aadf1ac79767078d7f05c2b66c56747df4c217385a8f3638
 SHA512:
-  metadata.gz: bf60e230bd435b0fa9b7463181dde2591a85a73711b9928cc7a35649999afaf28f3b0e93c40a34b4ede516be364e4a6557876e8e58760825c38b9aa3f8ba490d
-  data.tar.gz: 37e91e7e9006a1a133819e98155adfac51ce2d8685f79591103183045390a9cf5796b68ae4fc8264566d4ec949733183f2f7b5db49c1081fd9593b600c9ed7c0
+  metadata.gz: 0ebfe420b66bbdb6eae26124fff14e22d259df0f381bec4faaef8b3d1113701fe540803b77021792a28d82ce0226358002515f3bc54329ef94d6028709335f17
+  data.tar.gz: fd48bb0cc84f0e6917ea10d8be396002dde9e514c36c5b7b830f69cb6e497a4e2b672ee8acd134c22dcf2ae90c164326456340e2e1a8b18f56fd13736aee3452

data/lib/activerecord_graph_extractor/configuration.rb

@@ -6,7 +6,7 @@ module ActiveRecordGraphExtractor
                   :validate_records, :use_transactions, :handle_circular_references,
                   :skip_missing_models, :included_models, :excluded_models,
                   :included_relationships, :excluded_relationships,
-                  :custom_serializers, :primary_key_strategy
+                  :custom_serializers, :primary_key_strategy, :skip_non_primary_database_models
 
     def initialize
       reset!
@@ -21,6 +21,7 @@ module ActiveRecordGraphExtractor
       @use_transactions = true
       @handle_circular_references = true
       @skip_missing_models = true
+      @skip_non_primary_database_models = true
       @included_models = []
       @excluded_models = []
       @included_relationships = []

data/lib/activerecord_graph_extractor/dry_run_analyzer.rb

@@ -2,11 +2,14 @@
 
 module ActiveRecordGraphExtractor
   class DryRunAnalyzer
-    attr_reader :config, :relationship_analyzer
+    attr_reader :config, :relationship_analyzer, :model_count_cache, :relationship_count_cache, :sample_record_cache
 
     def initialize(config = ActiveRecordGraphExtractor.configuration)
       @config = config
       @relationship_analyzer = RelationshipAnalyzer.new(config)
+      @model_count_cache = {}        # Cache for model.count queries
+      @relationship_count_cache = {} # Cache for relationship count estimates
+      @sample_record_cache = {}      # Cache for sample records
     end
 
     def analyze(root_objects, options = {})
@@ -134,10 +137,61 @@ module ActiveRecordGraphExtractor
     end
 
     def estimate_relationship_count(model_class, relationship_name, relationship_info)
-      # Try to get a sample record to estimate relationship sizes
-      sample_record = model_class.first
-      return 0 unless sample_record
+      # Create cache key for this specific relationship
+      cache_key = "#{model_class.name}##{relationship_name}"
+      
+      # Return cached result if available
+      return @relationship_count_cache[cache_key] if @relationship_count_cache.key?(cache_key)
+
+      # Get cached sample record or fetch and cache it
+      sample_record = get_cached_sample_record(model_class)
+      
+      result = if sample_record.nil?
+        0
+      else
+        calculate_relationship_estimate(model_class, relationship_name, relationship_info, sample_record)
+      end
+
+      # Cache the result
+      @relationship_count_cache[cache_key] = result
+      result
+    end
+
+    def get_cached_sample_record(model_class)
+      model_name = model_class.name
+      
+      # Return cached sample record if available
+      return @sample_record_cache[model_name] if @sample_record_cache.key?(model_name)
+      
+      # Fetch and cache sample record (or nil if none exists)
+      sample_record = begin
+        model_class.first
+      rescue StandardError
+        nil
+      end
+      
+      @sample_record_cache[model_name] = sample_record
+      sample_record
+    end
 
+    def get_cached_model_count(model_class)
+      model_name = model_class.name
+      
+      # Return cached count if available
+      return @model_count_cache[model_name] if @model_count_cache.key?(model_name)
+      
+      # Fetch and cache model count
+      count = begin
+        model_class.count
+      rescue StandardError
+        0
+      end
+      
+      @model_count_cache[model_name] = count
+      count
+    end
+
+    def calculate_relationship_estimate(model_class, relationship_name, relationship_info, sample_record)
       begin
         case relationship_info['type']
         when 'has_many', 'has_and_belongs_to_many'
@@ -145,8 +199,8 @@ module ActiveRecordGraphExtractor
           related_records = sample_record.public_send(relationship_name)
           if related_records.respond_to?(:count)
             sample_count = related_records.limit(100).count
-            # Estimate total based on sample (with some reasonable assumptions)
-            total_records = model_class.count
+            # Use cached total record count
+            total_records = get_cached_model_count(model_class)
             return 0 if total_records == 0
             
             # Use sample count as average, but cap at reasonable limits
@@ -155,17 +209,36 @@ module ActiveRecordGraphExtractor
           end
         when 'has_one', 'belongs_to'
           # For singular relationships, estimate 1 per parent record
-          total_records = model_class.count
+          total_records = get_cached_model_count(model_class)
           return (total_records * 0.9).to_i # 90% factor assuming some records might not have the relationship
         end
       rescue StandardError
         # If we can't estimate, return a conservative estimate
-        return model_class.count > 0 ? [model_class.count / 10, 1].max : 0
+        total_records = get_cached_model_count(model_class)
+        return total_records > 0 ? [total_records / 10, 1].max : 0
       end
 
       0
     end
 
+    # Clear all caches (useful for testing or memory management)
+    def clear_cache!
+      @model_count_cache.clear
+      @relationship_count_cache.clear
+      @sample_record_cache.clear
+    end
+
+    # Get cache statistics for debugging
+    def cache_stats
+      {
+        model_count_cache_size: @model_count_cache.size,
+        relationship_count_cache_size: @relationship_count_cache.size,
+        sample_record_cache_size: @sample_record_cache.size,
+        cached_models: @model_count_cache.keys,
+        cached_relationships: @relationship_count_cache.keys
+      }
+    end
+
     def estimate_file_size(model_counts, relationship_map)
       total_size = 0
       

data/lib/activerecord_graph_extractor/relationship_analyzer.rb

@@ -8,6 +8,7 @@ module ActiveRecordGraphExtractor
       @config = config
       @visited_models = Set.new
       @circular_paths = []
+      @primary_connection = ActiveRecord::Base.connection
     end
 
     def analyze_model(model_class)
@@ -25,12 +26,21 @@ module ActiveRecordGraphExtractor
           relationship_name = association.name.to_s
           next unless config.relationship_included?(relationship_name)
 
-          model_name = association.klass.name
+          # Check if the associated model uses the primary database
+          associated_model = association.klass
+          model_name = associated_model.name
+          
+          # Skip if model is excluded
           next unless config.model_included?(model_name)
+          
+          # Skip if the associated model doesn't use the primary database (if enabled)
+          if config.skip_non_primary_database_models
+            next unless uses_primary_database?(associated_model)
+          end
 
           relationships[relationship_name] = {
             'type' => association.macro.to_s,
-            'model_class' => association.klass,
+            'model_class' => associated_model,
             'model_name' => model_name,
             'foreign_key' => association.foreign_key,
             'polymorphic' => association.options[:polymorphic] || false,
@@ -42,6 +52,13 @@ module ActiveRecordGraphExtractor
           else
             raise e
           end
+        rescue StandardError => e
+          # Skip associations that cause database connection errors
+          if e.message.include?('Unknown database') || e.message.include?('database connection')
+            next
+          else
+            raise e unless config.skip_missing_models
+          end
         end
       end
 
@@ -101,7 +118,7 @@ module ActiveRecordGraphExtractor
           end
         end
 
-        dependency_graph[model_class] = dependencies
+        dependency_graph[model_class] = dependencies.uniq
       end
 
       dependency_graph
@@ -113,6 +130,61 @@ module ActiveRecordGraphExtractor
 
     private
 
+    def uses_primary_database?(model_class)
+      begin
+        # Check if the model inherits from a non-primary database class
+        return false if inherits_from_non_primary_database?(model_class)
+        
+        # Get the model's connection
+        model_connection = model_class.connection
+        
+        # Compare connection configurations
+        primary_config = @primary_connection.instance_variable_get(:@config) || {}
+        model_config = model_connection.instance_variable_get(:@config) || {}
+        
+        # Check if they're the same connection object
+        return true if model_connection == @primary_connection
+        
+        # Check if they have the same database configuration
+        if primary_config.any? && model_config.any?
+          # Compare key connection parameters
+          same_database = primary_config[:database] == model_config[:database]
+          same_host = (primary_config[:host] || 'localhost') == (model_config[:host] || 'localhost')
+          same_port = (primary_config[:port] || 5432) == (model_config[:port] || 5432)
+          
+          return same_database && same_host && same_port
+        end
+        
+        # If we can't determine, check if the connection pool is the same
+        primary_pool = ActiveRecord::Base.connection_pool
+        model_pool = model_class.connection_pool
+        
+        return primary_pool == model_pool
+        
+      rescue StandardError => e
+        # If any error occurs (including database connection errors), assume it's not the primary database
+        return false
+      end
+    end
+
+    def inherits_from_non_primary_database?(model_class)
+      # Check if the model inherits from known non-primary database classes
+      non_primary_classes = [
+        'Ops::ReplicaDatabase',
+        'Analytics::Base',
+        'Reporting::Base',
+        'DataWarehouse::Base',
+        'External::Base',
+        'ThirdParty::Base'
+      ]
+      
+      ancestors = model_class.ancestors.map(&:name).compact
+      non_primary_classes.any? { |non_primary_class| ancestors.include?(non_primary_class) }
+    rescue StandardError
+      # If we can't determine the ancestry, err on the side of caution
+      false
+    end
+
     def filter_relationships(relationships)
       relationships.select do |name, info|
         config.model_included?(info['model_name']) &&
@@ -128,6 +200,11 @@ module ActiveRecordGraphExtractor
       
       return false unless config.model_included?(association.klass.name)
       
+      # Check if the associated model uses the primary database (if enabled)
+      if config.skip_non_primary_database_models
+        return false unless uses_primary_database?(association.klass)
+      end
+      
       # Skip polymorphic associations that can't be resolved
       return false if association.polymorphic? && association.foreign_type.nil?
       
@@ -135,6 +212,12 @@ module ActiveRecordGraphExtractor
     rescue NameError
       # Skip associations that reference non-existent models
       false
+    rescue StandardError => e
+      # Skip associations that cause database connection errors
+      if e.message.include?('Unknown database') || e.message.include?('database connection')
+        return false
+      end
+      raise e unless config.skip_missing_models
     end
 
     def build_relationship_info(association)

data/lib/activerecord_graph_extractor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordGraphExtractor
-  VERSION = "0.1.0"
-end 
+  VERSION = "0.2.0"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-graph-extractor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Florian Lorrain
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-06-14 00:00:00.000000000 Z
+date: 2025-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -335,7 +335,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- activerecord-graph-extractor.gemspec
 - docs/dry_run.md
 - docs/examples.md
 - docs/s3_integration.md

data/activerecord-graph-extractor.gemspec

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "lib/activerecord_graph_extractor/version"
-
-Gem::Specification.new do |spec|
-  spec.name          = "activerecord-graph-extractor"
-  spec.version       = ActiveRecordGraphExtractor::VERSION
-  spec.authors       = ["Florian Lorrain"]
-  spec.email         = ["lorrain.florian@gmail.com"]
-
-  spec.summary       = "Extract and import complex ActiveRecord object graphs while preserving relationships"
-  spec.description   = "A Ruby gem for extracting and importing complex ActiveRecord object graphs with smart dependency resolution, beautiful CLI progress visualization, and memory-efficient streaming. Perfect for data migration, testing, and environment synchronization."
-  spec.homepage      = "https://github.com/florrain/activerecord-graph-extractor"
-  spec.license       = "MIT"
-  spec.required_ruby_version = ">= 2.7.0"
-
-  spec.metadata["allowed_push_host"] = "https://rubygems.org"
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/florrain/activerecord-graph-extractor"
-  spec.metadata["changelog_uri"] = "https://github.com/florrain/activerecord-graph-extractor/blob/main/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
-    end
-  end
-  
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  # Core dependencies
-  spec.add_dependency "activerecord", ">= 6.0"
-  spec.add_dependency "activesupport", ">= 6.0"
-  
-  # CLI dependencies
-  spec.add_dependency "thor", "~> 1.2"
-  spec.add_dependency "tty-progressbar", "~> 0.18"
-  spec.add_dependency "tty-spinner", "~> 0.9"
-  spec.add_dependency "tty-tree", "~> 0.4"
-  spec.add_dependency "pastel", "~> 0.8"
-  spec.add_dependency "tty-prompt", "~> 0.23"
-  
-  # JSON streaming
-  spec.add_dependency "oj", "~> 3.13"
-  spec.add_dependency "yajl-ruby", ">= 1.3"
-  
-  # S3 support
-  spec.add_dependency "aws-sdk-s3", "~> 1.0"
-
-  # Development dependencies
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.12"
-  spec.add_development_dependency "rubocop", "~> 1.57"
-  spec.add_development_dependency "sqlite3", "~> 1.6"
-  spec.add_development_dependency "database_cleaner", "~> 2.0"
-  spec.add_development_dependency "factory_bot", "~> 6.2"
-  spec.add_development_dependency "simplecov", "~> 0.22"
-  spec.add_development_dependency "rubocop-rspec", "~> 2.25"
-  spec.add_development_dependency "pry", "~> 0.14"
-  spec.add_development_dependency "pry-byebug", "~> 3.10"
-end