active_record_data_loader 1.2.0 → 1.3.0

data/lib/active_record_data_loader/null_output_adapter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  class NullOutputAdapter
+    def self.with_output_options(_options)
+      yield new
+    end
+
+    def copy(table:, columns:, data:, row_numbers:); end
+
+    def insert(command); end
+
+    def write_command(command); end
+  end
+end

data/lib/active_record_data_loader/table_loader.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class TableLoader
+    def self.load_data(
+      total_rows:,
+      batch_size:,
+      logger:,
+      connection_handler:,
+      strategy:
+    )
+      new(logger: logger, connection_handler: connection_handler, strategy: strategy)
+        .load_data(batch_size, total_rows)
+    end
+
+    def initialize(logger:, connection_handler:, strategy:)
+      @logger = logger
+      @connection_handler = connection_handler
+      @strategy = strategy
+    end
+
+    def load_data(batch_size, total_rows)
+      batch_count = (total_rows / batch_size.to_f).ceil
+
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
+        "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
+      )
+      total_time = Benchmark.realtime do
+        load_in_batches(batch_size, total_rows, batch_count)
+      end
+      logger.info(
+        "[ActiveRecordDataLoader] "\
+        "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
+        "in #{total_time} seconds."
+      )
+    end
+
+    private
+
+    attr_reader :strategy, :connection_handler, :logger
+
+    def load_in_batches(batch_size, total_rows, batch_count)
+      connection_handler.with_connection do |connection|
+        total_rows.times.each_slice(batch_size).with_index do |row_numbers, i|
+          time = Benchmark.realtime { strategy.load_batch(row_numbers, connection) }
+
+          logger.debug(
+            "[ActiveRecordDataLoader] "\
+            "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordDataLoader
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/active_record_data_loader.rb

@@ -2,31 +2,35 @@
 
 require "active_record_data_loader/version"
 require "active_record"
+require "active_record_data_loader/errors"
 require "active_record_data_loader/configuration"
 require "active_record_data_loader/connection_handler"
 require "active_record_data_loader/data_faker"
+require "active_record_data_loader/active_record/list"
 require "active_record_data_loader/active_record/per_row_value_cache"
 require "active_record_data_loader/active_record/integer_value_generator"
 require "active_record_data_loader/active_record/text_value_generator"
 require "active_record_data_loader/active_record/enum_value_generator"
 require "active_record_data_loader/active_record/datetime_value_generator"
-require "active_record_data_loader/active_record/column_configuration"
-require "active_record_data_loader/active_record/belongs_to_configuration"
-require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
+require "active_record_data_loader/active_record/column_data_provider"
+require "active_record_data_loader/active_record/belongs_to_data_provider"
+require "active_record_data_loader/active_record/polymorphic_belongs_to_data_provider"
+require "active_record_data_loader/active_record/unique_index_tracker"
 require "active_record_data_loader/active_record/model_data_generator"
 require "active_record_data_loader/dsl/belongs_to_association"
 require "active_record_data_loader/dsl/polymorphic_association"
 require "active_record_data_loader/dsl/model"
 require "active_record_data_loader/dsl/definition"
-require "active_record_data_loader/connection_output_adapter"
 require "active_record_data_loader/file_output_adapter"
+require "active_record_data_loader/null_output_adapter"
 require "active_record_data_loader/copy_strategy"
 require "active_record_data_loader/bulk_insert_strategy"
+require "active_record_data_loader/table_loader"
 require "active_record_data_loader/loader"
 
 module ActiveRecordDataLoader
   def self.define(config = ActiveRecordDataLoader.configuration, &block)
-    LoaderProxy.new(
+    ActiveRecordDataLoader::Loader.new(
       config,
       ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
     )
@@ -39,40 +43,4 @@ module ActiveRecordDataLoader
   def self.configuration
     @configuration ||= ActiveRecordDataLoader::Configuration.new
   end
-
-  class LoaderProxy
-    def initialize(configuration, definition)
-      @configuration = configuration
-      @definition = definition
-    end
-
-    def load_data
-      ActiveRecordDataLoader::ActiveRecord::PerRowValueCache.clear
-
-      configuration.connection_handler.with_statement_timeout_for_output do
-        definition.models.map { |m| load_model(m) }
-      end
-    end
-
-    private
-
-    attr_reader :definition, :configuration
-
-    def load_model(model)
-      generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
-        model: model.klass,
-        column_settings: model.columns,
-        polymorphic_settings: model.polymorphic_associations,
-        belongs_to_settings: model.belongs_to_associations,
-        connection_factory: configuration.connection_factory
-      )
-
-      ActiveRecordDataLoader::Loader.load_data(
-        data_generator: generator,
-        batch_size: model.batch_size,
-        total_rows: model.row_count,
-        configuration: configuration
-      )
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_record_data_loader
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Alejandro Beiderman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-14 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -208,7 +208,7 @@ dependencies:
         version: '0'
 description: A utility to bulk load test data for performance testing.
 email:
-- abeiderman@gmail.com
+- active_record_data_loader@ossprojects.dev
 executables:
 - console
 - setup
@@ -216,6 +216,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/build.yml"
+- ".github/workflows/codeql-analysis.yml"
 - ".github/workflows/gem-push.yml"
 - ".gitignore"
 - ".rspec"
@@ -240,27 +241,31 @@ files:
 - gemfiles/ffaker.gemfile
 - gemfiles/rails.gemfile
 - lib/active_record_data_loader.rb
-- lib/active_record_data_loader/active_record/belongs_to_configuration.rb
-- lib/active_record_data_loader/active_record/column_configuration.rb
+- lib/active_record_data_loader/active_record/belongs_to_data_provider.rb
+- lib/active_record_data_loader/active_record/column_data_provider.rb
 - lib/active_record_data_loader/active_record/datetime_value_generator.rb
 - lib/active_record_data_loader/active_record/enum_value_generator.rb
 - lib/active_record_data_loader/active_record/integer_value_generator.rb
+- lib/active_record_data_loader/active_record/list.rb
 - lib/active_record_data_loader/active_record/model_data_generator.rb
 - lib/active_record_data_loader/active_record/per_row_value_cache.rb
-- lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb
+- lib/active_record_data_loader/active_record/polymorphic_belongs_to_data_provider.rb
 - lib/active_record_data_loader/active_record/text_value_generator.rb
+- lib/active_record_data_loader/active_record/unique_index_tracker.rb
 - lib/active_record_data_loader/bulk_insert_strategy.rb
 - lib/active_record_data_loader/configuration.rb
 - lib/active_record_data_loader/connection_handler.rb
-- lib/active_record_data_loader/connection_output_adapter.rb
 - lib/active_record_data_loader/copy_strategy.rb
 - lib/active_record_data_loader/data_faker.rb
 - lib/active_record_data_loader/dsl/belongs_to_association.rb
 - lib/active_record_data_loader/dsl/definition.rb
 - lib/active_record_data_loader/dsl/model.rb
 - lib/active_record_data_loader/dsl/polymorphic_association.rb
+- lib/active_record_data_loader/errors.rb
 - lib/active_record_data_loader/file_output_adapter.rb
 - lib/active_record_data_loader/loader.rb
+- lib/active_record_data_loader/null_output_adapter.rb
+- lib/active_record_data_loader/table_loader.rb
 - lib/active_record_data_loader/version.rb
 - log/.keep
 homepage: 

data/lib/active_record_data_loader/connection_output_adapter.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module ActiveRecordDataLoader
-  class ConnectionOutputAdapter
-    def needs_timeout_output?
-      false
-    end
-
-    def copy(connection:, table:, columns:, data:, row_numbers:)
-      raw_connection = connection.raw_connection
-      raw_connection.copy_data("COPY #{table} (#{columns}) FROM STDIN WITH (FORMAT CSV)") do
-        raw_connection.put_copy_data(data.join("\n"))
-      end
-    end
-
-    def insert(connection:, command:)
-      connection.insert(command)
-    end
-  end
-end