active_record_data_loader 1.0.2 → 1.3.1

data/lib/active_record_data_loader/table_loader.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class TableLoader
+    def self.load_data(
+      total_rows:,
+      batch_size:,
+      logger:,
+      connection_handler:,
+      strategy:
+    )
+      new(logger: logger, connection_handler: connection_handler, strategy: strategy)
+        .load_data(batch_size, total_rows)
+    end
+
+    def initialize(logger:, connection_handler:, strategy:)
+      @logger = logger
+      @connection_handler = connection_handler
+      @strategy = strategy
+    end
+
+    def load_data(batch_size, total_rows)
+      batch_count = (total_rows / batch_size.to_f).ceil
+
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
+        "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
+      )
+      total_time = Benchmark.realtime do
+        load_in_batches(batch_size, total_rows, batch_count)
+      end
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
+        "in #{total_time} seconds."
+      )
+    end
+
+    private
+
+    attr_reader :strategy, :connection_handler, :logger
+
+    def load_in_batches(batch_size, total_rows, batch_count)
+      connection_handler.with_connection do |connection|
+        total_rows.times.each_slice(batch_size).with_index do |row_numbers, i|
+          time = Benchmark.realtime { strategy.load_batch(row_numbers, connection) }
+
+          logger.debug(
+            "[ActiveRecordDataLoader] "\
+            "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordDataLoader
-  VERSION = "1.0.2"
+  VERSION = "1.3.1"
 end

data/lib/active_record_data_loader.rb

@@ -2,28 +2,35 @@
 
 require "active_record_data_loader/version"
 require "active_record"
+require "active_record_data_loader/errors"
 require "active_record_data_loader/configuration"
+require "active_record_data_loader/connection_handler"
 require "active_record_data_loader/data_faker"
+require "active_record_data_loader/active_record/list"
 require "active_record_data_loader/active_record/per_row_value_cache"
 require "active_record_data_loader/active_record/integer_value_generator"
 require "active_record_data_loader/active_record/text_value_generator"
 require "active_record_data_loader/active_record/enum_value_generator"
 require "active_record_data_loader/active_record/datetime_value_generator"
-require "active_record_data_loader/active_record/column_configuration"
-require "active_record_data_loader/active_record/belongs_to_configuration"
-require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
+require "active_record_data_loader/active_record/column_data_provider"
+require "active_record_data_loader/active_record/belongs_to_data_provider"
+require "active_record_data_loader/active_record/polymorphic_belongs_to_data_provider"
+require "active_record_data_loader/active_record/unique_index_tracker"
 require "active_record_data_loader/active_record/model_data_generator"
 require "active_record_data_loader/dsl/belongs_to_association"
 require "active_record_data_loader/dsl/polymorphic_association"
 require "active_record_data_loader/dsl/model"
 require "active_record_data_loader/dsl/definition"
+require "active_record_data_loader/file_output_adapter"
+require "active_record_data_loader/null_output_adapter"
 require "active_record_data_loader/copy_strategy"
 require "active_record_data_loader/bulk_insert_strategy"
+require "active_record_data_loader/table_loader"
 require "active_record_data_loader/loader"
 
 module ActiveRecordDataLoader
   def self.define(config = ActiveRecordDataLoader.configuration, &block)
-    LoaderProxy.new(
+    ActiveRecordDataLoader::Loader.new(
       config,
       ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
     )
@@ -36,38 +43,4 @@ module ActiveRecordDataLoader
   def self.configuration
     @configuration ||= ActiveRecordDataLoader::Configuration.new
   end
-
-  class LoaderProxy
-    def initialize(configuration, definition)
-      @configuration = configuration
-      @definition = definition
-    end
-
-    def load_data
-      ActiveRecordDataLoader::ActiveRecord::PerRowValueCache.clear
-
-      definition.models.map { |m| load_model(m) }
-    end
-
-    private
-
-    attr_reader :definition, :configuration
-
-    def load_model(model)
-      generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
-        model: model.klass,
-        column_settings: model.columns,
-        polymorphic_settings: model.polymorphic_associations,
-        belongs_to_settings: model.belongs_to_associations,
-        connection_factory: configuration.connection_factory
-      )
-
-      ActiveRecordDataLoader::Loader.load_data(
-        data_generator: generator,
-        batch_size: model.batch_size,
-        total_rows: model.row_count,
-        configuration: configuration
-      )
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_record_data_loader
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.3.1
 platform: ruby
 authors:
 - Alejandro Beiderman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-06 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
-- !ruby/object:Gem::Dependency
-  name: coveralls
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +150,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -194,17 +208,19 @@ dependencies:
         version: '0'
 description: A utility to bulk load test data for performance testing.
 email:
-- abeiderman@gmail.com
+- active_record_data_loader@ossprojects.dev
 executables:
 - console
 - setup
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/build.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/gem-push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -217,7 +233,7 @@ files:
 - bin/console
 - bin/setup
 - config/database.yml
-- config/database.yml.travis
+- docker-compose.yml
 - gemfiles/.bundle/config
 - gemfiles/activerecord_5.gemfile
 - gemfiles/activerecord_6.gemfile
@@ -225,24 +241,31 @@ files:
 - gemfiles/ffaker.gemfile
 - gemfiles/rails.gemfile
 - lib/active_record_data_loader.rb
-- lib/active_record_data_loader/active_record/belongs_to_configuration.rb
-- lib/active_record_data_loader/active_record/column_configuration.rb
+- lib/active_record_data_loader/active_record/belongs_to_data_provider.rb
+- lib/active_record_data_loader/active_record/column_data_provider.rb
 - lib/active_record_data_loader/active_record/datetime_value_generator.rb
 - lib/active_record_data_loader/active_record/enum_value_generator.rb
 - lib/active_record_data_loader/active_record/integer_value_generator.rb
+- lib/active_record_data_loader/active_record/list.rb
 - lib/active_record_data_loader/active_record/model_data_generator.rb
 - lib/active_record_data_loader/active_record/per_row_value_cache.rb
-- lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb
+- lib/active_record_data_loader/active_record/polymorphic_belongs_to_data_provider.rb
 - lib/active_record_data_loader/active_record/text_value_generator.rb
+- lib/active_record_data_loader/active_record/unique_index_tracker.rb
 - lib/active_record_data_loader/bulk_insert_strategy.rb
 - lib/active_record_data_loader/configuration.rb
+- lib/active_record_data_loader/connection_handler.rb
 - lib/active_record_data_loader/copy_strategy.rb
 - lib/active_record_data_loader/data_faker.rb
 - lib/active_record_data_loader/dsl/belongs_to_association.rb
 - lib/active_record_data_loader/dsl/definition.rb
 - lib/active_record_data_loader/dsl/model.rb
 - lib/active_record_data_loader/dsl/polymorphic_association.rb
+- lib/active_record_data_loader/errors.rb
+- lib/active_record_data_loader/file_output_adapter.rb
 - lib/active_record_data_loader/loader.rb
+- lib/active_record_data_loader/null_output_adapter.rb
+- lib/active_record_data_loader/table_loader.rb
 - lib/active_record_data_loader/version.rb
 - log/.keep
 homepage: 
@@ -258,15 +281,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A utility to bulk load test data for performance testing.

data/.travis.yml

@@ -1,24 +0,0 @@
-sudo: false
-language: ruby
-env:
-  - COVERALLS_PARALLEL=true
-rvm:
-  - 2.3.8
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-gemfile:
-  - gemfiles/activerecord_5.gemfile
-  - gemfiles/rails.gemfile
-  - gemfiles/faker.gemfile
-  - gemfiles/ffaker.gemfile
-services:
-  - postgresql
-notifications:
-  webhooks: https://coveralls.io/webhook
-before_install: "gem update --system && gem install bundler"
-before_script:
-  - psql -c 'create database test;' -U postgres
-  - mysql -e 'CREATE DATABASE IF NOT EXISTS test;'
-  - cp config/database.yml.travis config/database.yml
-script: "bundle exec rake"

data/config/database.yml.travis

@@ -1,12 +0,0 @@
-postgres:
-  adapter: "postgresql"
-  database: "test"
-
-sqlite3:
-  adapter: "sqlite3"
-  database: ":memory:"
-
-mysql:
-  adapter: "mysql2"
-  database: "test"
-  username: "travis"