active_record_data_loader 1.0.1 → 1.3.0

data/lib/active_record_data_loader/loader.rb

@@ -1,92 +1,82 @@
 # frozen_string_literal: true
 
-require "benchmark"
-
 module ActiveRecordDataLoader
   class Loader
-    class << self
-      def load_data(
-        data_generator:,
-        total_rows:,
-        batch_size:,
-        configuration:
-      )
-        new(
-          logger: configuration.logger,
-          statement_timeout: configuration.statement_timeout,
-          strategy: strategy_class.new(data_generator)
-        ).load_data(batch_size, total_rows)
-      end
-
-      private
-
-      def strategy_class
-        if ::ActiveRecord::Base.connection.raw_connection.respond_to?(:copy_data)
-          ActiveRecordDataLoader::CopyStrategy
-        else
-          ActiveRecordDataLoader::BulkInsertStrategy
-        end
-      end
-    end
-
-    def initialize(logger:, statement_timeout:, strategy:)
-      @logger = logger
-      @strategy = strategy
-      @statement_timeout = statement_timeout
+    def initialize(configuration, definition)
+      @configuration = configuration
+      @definition = definition
     end
 
-    def load_data(batch_size, total_rows)
-      batch_count = (total_rows / batch_size.to_f).ceil
+    def load_data
+      ActiveRecordDataLoader::ActiveRecord::PerRowValueCache.clear
 
-      logger.info(
-        "[ActiveRecordDataLoader] "\
-        "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
-        "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
-      )
-      total_time = Benchmark.realtime do
-        load_in_batches(batch_size, total_rows, batch_count)
+      file_adapter_class.with_output_options(file_adapter_options) do |file_adapter|
+        definition.models.map { |m| load_model(m, file_adapter) }
       end
-      logger.info(
-        "[ActiveRecordDataLoader] "\
-        "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
-        "in #{total_time} seconds."
-      )
     end
 
     private
 
-    attr_reader :strategy, :statement_timeout, :logger
+    attr_reader :definition, :configuration
 
-    def load_in_batches(batch_size, total_rows, batch_count)
-      with_connection do |connection|
-        total_rows.times.each_slice(batch_size).with_index do |row_numbers, i|
-          time = Benchmark.realtime { strategy.load_batch(row_numbers, connection) }
+    def load_model(model, file_adapter)
+      ActiveRecordDataLoader::TableLoader.load_data(
+        batch_size: model.batch_size,
+        total_rows: model.row_count,
+        connection_handler: connection_handler,
+        strategy: strategy_class.new(generator(model), file_adapter),
+        logger: configuration.logger
+      )
+    end
 
-          logger.debug(
-            "[ActiveRecordDataLoader] "\
-            "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
-          )
-        end
-      end
+    def generator(model)
+      ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
+        model: model.klass,
+        column_settings: model.columns,
+        polymorphic_settings: model.polymorphic_associations,
+        belongs_to_settings: model.belongs_to_associations,
+        connection_factory: configuration.connection_factory,
+        raise_on_duplicates: model.raise_on_duplicates_flag,
+        max_duplicate_retries: model.max_duplicate_retries,
+        logger: configuration.logger
+      )
     end
 
-    def with_connection
-      if ::ActiveRecord::Base.connection.adapter_name.downcase.to_sym == :postgresql
-        original_timeout = retrieve_statement_timeout
-        update_statement_timeout(statement_timeout)
-        yield ::ActiveRecord::Base.connection
-        update_statement_timeout(original_timeout)
+    def file_adapter_class
+      if configuration.output.present?
+        ActiveRecordDataLoader::FileOutputAdapter
       else
-        yield ::ActiveRecord::Base.connection
+        ActiveRecordDataLoader::NullOutputAdapter
       end
     end
 
-    def retrieve_statement_timeout
-      ::ActiveRecord::Base.connection.execute("SHOW statement_timeout").first["statement_timeout"]
+    def file_adapter_options
+      timeout_commands =
+        if connection_handler.supports_timeout?
+          {
+            pre_command: connection_handler.timeout_set_command,
+            post_command: connection_handler.reset_timeout_command,
+          }
+        else
+          {}
+        end
+
+      timeout_commands.merge(filename: configuration.output)
     end
 
-    def update_statement_timeout(timeout)
-      ::ActiveRecord::Base.connection.execute("SET statement_timeout = \"#{timeout}\"")
+    def strategy_class
+      @strategy_class ||= if connection_handler.supports_copy?
+                            ActiveRecordDataLoader::CopyStrategy
+                          else
+                            ActiveRecordDataLoader::BulkInsertStrategy
+                          end
+    end
+
+    def connection_handler
+      @connection_handler ||= ActiveRecordDataLoader::ConnectionHandler.new(
+        connection_factory: configuration.connection_factory,
+        statement_timeout: configuration.statement_timeout
+      )
     end
   end
 end

data/lib/active_record_data_loader/null_output_adapter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  class NullOutputAdapter
+    def self.with_output_options(_options)
+      yield new
+    end
+
+    def copy(table:, columns:, data:, row_numbers:); end
+
+    def insert(command); end
+
+    def write_command(command); end
+  end
+end

data/lib/active_record_data_loader/table_loader.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class TableLoader
+    def self.load_data(
+      total_rows:,
+      batch_size:,
+      logger:,
+      connection_handler:,
+      strategy:
+    )
+      new(logger: logger, connection_handler: connection_handler, strategy: strategy)
+        .load_data(batch_size, total_rows)
+    end
+
+    def initialize(logger:, connection_handler:, strategy:)
+      @logger = logger
+      @connection_handler = connection_handler
+      @strategy = strategy
+    end
+
+    def load_data(batch_size, total_rows)
+      batch_count = (total_rows / batch_size.to_f).ceil
+
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
+        "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
+      )
+      total_time = Benchmark.realtime do
+        load_in_batches(batch_size, total_rows, batch_count)
+      end
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
+        "in #{total_time} seconds."
+      )
+    end
+
+    private
+
+    attr_reader :strategy, :connection_handler, :logger
+
+    def load_in_batches(batch_size, total_rows, batch_count)
+      connection_handler.with_connection do |connection|
+        total_rows.times.each_slice(batch_size).with_index do |row_numbers, i|
+          time = Benchmark.realtime { strategy.load_batch(row_numbers, connection) }
+
+          logger.debug(
+            "[ActiveRecordDataLoader] "\
+            "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordDataLoader
-  VERSION = "1.0.1"
+  VERSION = "1.3.0"
 end

data/lib/active_record_data_loader.rb

@@ -2,29 +2,36 @@
 
 require "active_record_data_loader/version"
 require "active_record"
+require "active_record_data_loader/errors"
 require "active_record_data_loader/configuration"
+require "active_record_data_loader/connection_handler"
 require "active_record_data_loader/data_faker"
+require "active_record_data_loader/active_record/list"
 require "active_record_data_loader/active_record/per_row_value_cache"
 require "active_record_data_loader/active_record/integer_value_generator"
 require "active_record_data_loader/active_record/text_value_generator"
 require "active_record_data_loader/active_record/enum_value_generator"
 require "active_record_data_loader/active_record/datetime_value_generator"
-require "active_record_data_loader/active_record/column_configuration"
-require "active_record_data_loader/active_record/belongs_to_configuration"
-require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
+require "active_record_data_loader/active_record/column_data_provider"
+require "active_record_data_loader/active_record/belongs_to_data_provider"
+require "active_record_data_loader/active_record/polymorphic_belongs_to_data_provider"
+require "active_record_data_loader/active_record/unique_index_tracker"
 require "active_record_data_loader/active_record/model_data_generator"
 require "active_record_data_loader/dsl/belongs_to_association"
 require "active_record_data_loader/dsl/polymorphic_association"
 require "active_record_data_loader/dsl/model"
 require "active_record_data_loader/dsl/definition"
+require "active_record_data_loader/file_output_adapter"
+require "active_record_data_loader/null_output_adapter"
 require "active_record_data_loader/copy_strategy"
 require "active_record_data_loader/bulk_insert_strategy"
+require "active_record_data_loader/table_loader"
 require "active_record_data_loader/loader"
 
 module ActiveRecordDataLoader
   def self.define(config = ActiveRecordDataLoader.configuration, &block)
-    LoaderProxy.new(
-      configuration,
+    ActiveRecordDataLoader::Loader.new(
+      config,
       ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
     )
   end
@@ -36,35 +43,4 @@ module ActiveRecordDataLoader
   def self.configuration
     @configuration ||= ActiveRecordDataLoader::Configuration.new
   end
-
-  class LoaderProxy
-    def initialize(configuration, definition)
-      @configuration = configuration
-      @definition = definition
-    end
-
-    def load_data
-      ActiveRecordDataLoader::ActiveRecord::PerRowValueCache.clear
-
-      definition.models.map do |m|
-        generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
-          model: m.klass,
-          column_settings: m.columns,
-          polymorphic_settings: m.polymorphic_associations,
-          belongs_to_settings: m.belongs_to_associations
-        )
-
-        ActiveRecordDataLoader::Loader.load_data(
-          data_generator: generator,
-          batch_size: m.batch_size,
-          total_rows: m.row_count,
-          configuration: configuration
-        )
-      end
-    end
-
-    private
-
-    attr_reader :definition, :configuration
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_record_data_loader
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Alejandro Beiderman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-17 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +150,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -180,17 +208,19 @@ dependencies:
         version: '0'
 description: A utility to bulk load test data for performance testing.
 email:
-- abeiderman@gmail.com
+- active_record_data_loader@ossprojects.dev
 executables:
 - console
 - setup
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/build.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/gem-push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -203,7 +233,7 @@ files:
 - bin/console
 - bin/setup
 - config/database.yml
-- config/database.yml.travis
+- docker-compose.yml
 - gemfiles/.bundle/config
 - gemfiles/activerecord_5.gemfile
 - gemfiles/activerecord_6.gemfile
@@ -211,24 +241,31 @@ files:
 - gemfiles/ffaker.gemfile
 - gemfiles/rails.gemfile
 - lib/active_record_data_loader.rb
-- lib/active_record_data_loader/active_record/belongs_to_configuration.rb
-- lib/active_record_data_loader/active_record/column_configuration.rb
+- lib/active_record_data_loader/active_record/belongs_to_data_provider.rb
+- lib/active_record_data_loader/active_record/column_data_provider.rb
 - lib/active_record_data_loader/active_record/datetime_value_generator.rb
 - lib/active_record_data_loader/active_record/enum_value_generator.rb
 - lib/active_record_data_loader/active_record/integer_value_generator.rb
+- lib/active_record_data_loader/active_record/list.rb
 - lib/active_record_data_loader/active_record/model_data_generator.rb
 - lib/active_record_data_loader/active_record/per_row_value_cache.rb
-- lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb
+- lib/active_record_data_loader/active_record/polymorphic_belongs_to_data_provider.rb
 - lib/active_record_data_loader/active_record/text_value_generator.rb
+- lib/active_record_data_loader/active_record/unique_index_tracker.rb
 - lib/active_record_data_loader/bulk_insert_strategy.rb
 - lib/active_record_data_loader/configuration.rb
+- lib/active_record_data_loader/connection_handler.rb
 - lib/active_record_data_loader/copy_strategy.rb
 - lib/active_record_data_loader/data_faker.rb
 - lib/active_record_data_loader/dsl/belongs_to_association.rb
 - lib/active_record_data_loader/dsl/definition.rb
 - lib/active_record_data_loader/dsl/model.rb
 - lib/active_record_data_loader/dsl/polymorphic_association.rb
+- lib/active_record_data_loader/errors.rb
+- lib/active_record_data_loader/file_output_adapter.rb
 - lib/active_record_data_loader/loader.rb
+- lib/active_record_data_loader/null_output_adapter.rb
+- lib/active_record_data_loader/table_loader.rb
 - lib/active_record_data_loader/version.rb
 - log/.keep
 homepage: 
@@ -244,14 +281,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A utility to bulk load test data for performance testing.

data/.travis.yml

@@ -1,23 +0,0 @@
-sudo: false
-language: ruby
-env:
-  - COVERALLS_PARALLEL=true
-rvm:
-  - 2.3.8
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-gemfile:
-  - gemfiles/activerecord_5.gemfile
-  - gemfiles/rails.gemfile
-  - gemfiles/faker.gemfile
-  - gemfiles/ffaker.gemfile
-services:
-  - postgresql
-notifications:
-  webhooks: https://coveralls.io/webhook
-before_install: "gem update --system && gem install bundler"
-before_script:
-  - psql -c 'create database test;' -U postgres
-  - cp config/database.yml.travis config/database.yml
-script: "bundle exec rake"

data/config/database.yml.travis

@@ -1,7 +0,0 @@
-postgres:
-  adapter: "postgresql"
-  database: "test"
-
-sqlite3:
-  adapter: "sqlite3"
-  database: ":memory:"