active_record_data_loader 0.1.1

data/lib/active_record_data_loader/bulk_insert_strategy.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "csv"
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class BulkInsertStrategy
+    def initialize(data_generator)
+      @data_generator = data_generator
+    end
+
+    def load_batch(row_numbers)
+      connection = ::ActiveRecord::Base.connection
+
+      connection.insert(<<~SQL)
+        INSERT INTO #{quoted_table_name(connection)} (#{column_list(connection)})
+        VALUES #{values(row_numbers, connection)}
+      SQL
+    end
+
+    def table_name
+      data_generator.table
+    end
+
+    def name
+      "BULK INSERT"
+    end
+
+    private
+
+    attr_reader :data_generator
+
+    def quoted_table_name(connection)
+      @quoted_table_name ||= connection.quote_table_name(data_generator.table)
+    end
+
+    def column_list(connection)
+      @column_list ||= data_generator
+                       .column_list
+                       .map { |c| connection.quote_column_name(c) }
+                       .join(",")
+    end
+
+    def values(row_numbers, connection)
+      row_numbers
+        .map { |i| "(#{row_values(i, connection)})" }
+        .join(",")
+    end
+
+    def row_values(row_number, connection)
+      data_generator
+        .generate_row(row_number)
+        .map { |v| connection.quote(v) }
+        .join(",")
+    end
+  end
+end

data/lib/active_record_data_loader/configuration.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  class Configuration
+    attr_accessor :default_batch_size, :default_row_count, :logger
+
+    def initialize(
+      default_batch_size: 100_000,
+      default_row_count: 1,
+      logger: Logger.new(STDOUT, level: :info)
+    )
+      @default_batch_size = default_batch_size
+      @default_row_count = default_row_count
+      @logger = logger
+    end
+  end
+end

data/lib/active_record_data_loader/copy_strategy.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "csv"
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class CopyStrategy
+    def initialize(data_generator)
+      @data_generator = data_generator
+    end
+
+    def load_batch(row_numbers)
+      csv_data = csv_data_batch(row_numbers)
+
+      connection = ::ActiveRecord::Base.connection.raw_connection
+      connection.copy_data(copy_command) { connection.put_copy_data(csv_data) }
+    end
+
+    def table_name
+      data_generator.table
+    end
+
+    def name
+      "COPY"
+    end
+
+    private
+
+    attr_reader :data_generator
+
+    def csv_data_batch(row_numbers)
+      row_numbers.map do |i|
+        data_generator.generate_row(i).map { |d| quote_data(d) }.join(",")
+      end.join("\n")
+    end
+
+    def copy_command
+      @copy_command ||= begin
+        quoted_table_name = ::ActiveRecord::Base.connection.quote_table_name(data_generator.table)
+        columns = data_generator
+                  .column_list
+                  .map { |c| ::ActiveRecord::Base.connection.quote_column_name(c) }
+                  .join(", ")
+
+        <<~SQL
+          COPY #{quoted_table_name} (#{columns})
+            FROM STDIN WITH (FORMAT CSV)
+        SQL
+      end
+    end
+
+    def quote_data(data)
+      return if data.nil?
+
+      "\"#{::ActiveRecord::Base.connection.quote_string(data.to_s)}\""
+    end
+  end
+end

data/lib/active_record_data_loader/data_faker.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module ActiveRecordDataLoader
+  class DataFaker
+    class << self
+      extend Forwardable
+
+      def_delegators :adapter, :person_name, :first_name, :middle_name, :last_name, :company_name
+
+      private
+
+      def adapter
+        @adapter ||=
+          if Gem.loaded_specs.key?("ffaker")
+            require "ffaker"
+            FFakerGemAdapter.new
+          elsif Gem.loaded_specs.key?("faker")
+            require "faker"
+            FakerGemAdapter.new
+          else
+            NoGemAdapter.new
+          end
+      end
+    end
+
+    class FFakerGemAdapter
+      extend Forwardable
+
+      def_delegators :ffaker_name, :first_name, :last_name
+      def_delegator :ffaker_name, :name, :person_name
+      def_delegator :ffaker_name, :first_name, :middle_name
+
+      def company_name
+        FFaker::Company.name
+      end
+
+      def ffaker_name
+        FFaker::Name
+      end
+    end
+
+    class FakerGemAdapter
+      extend Forwardable
+
+      def_delegators :faker_name, :first_name, :middle_name, :last_name
+      def_delegator :faker_name, :name, :person_name
+
+      def company_name
+        Faker::Company.name
+      end
+
+      def faker_name
+        Faker::Name
+      end
+    end
+
+    class NoGemAdapter
+      FIRST_NAMES = %w[John Mary].freeze
+      MIDDLE_NAMES = %w[Madison Ashley].freeze
+      LAST_NAMES = %w[Doe Smith].freeze
+
+      def first_name
+        FIRST_NAMES.sample
+      end
+
+      def middle_name
+        MIDDLE_NAMES.sample
+      end
+
+      def last_name
+        LAST_NAMES.sample
+      end
+
+      def person_name
+        "#{first_name} #{middle_name} #{last_name}"
+      end
+
+      def company_name
+        "Acme"
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/dsl/definition.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module Dsl
+    class Definition
+      attr_reader :models
+
+      def initialize(config = ActiveRecordDataLoader.configuration)
+        @models = []
+        @config = config
+      end
+
+      def model(klass, &block)
+        t = Model.new(klass: klass, configuration: config)
+        block&.call(t)
+        models << t
+        t
+      end
+
+      private
+
+      attr_reader :config
+    end
+  end
+end

data/lib/active_record_data_loader/dsl/model.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module Dsl
+    class Model
+      attr_reader :klass, :columns, :row_count, :polymorphic_associations
+
+      def initialize(klass:, configuration:)
+        @klass = klass
+        @columns = {}
+        @row_count = configuration.default_row_count
+        @batch_size = configuration.default_batch_size
+        @polymorphic_associations = []
+      end
+
+      def count(count)
+        @row_count = count
+      end
+
+      def batch_size(size = nil)
+        @batch_size = (size || @batch_size)
+      end
+
+      def column(name, func)
+        @columns[name.to_sym] = func
+      end
+
+      def polymorphic(assoc_name, &block)
+        @polymorphic_associations << PolymorphicAssociation.new(
+          @klass, assoc_name
+        ).tap { |a| block.call(a) }
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/dsl/polymorphic_association.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module Dsl
+    class PolymorphicAssociation
+      attr_reader :model_class, :name, :models
+
+      def initialize(model_class, name)
+        @model_class = model_class
+        @name = name
+        @models = {}
+      end
+
+      def model(klass, weight: 1)
+        @models[klass] = weight.to_i
+      end
+
+      def weighted_models
+        gcd = models.values.reduce(:gcd)
+
+        models.map { |m, w| [m] * (w / gcd) }.flatten
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/loader.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "benchmark"
+
+module ActiveRecordDataLoader
+  class Loader
+    class << self
+      def load_data(
+        data_generator:,
+        total_rows:,
+        batch_size:,
+        logger:
+      )
+        new(
+          logger: logger,
+          strategy: strategy_class.new(data_generator)
+        ).load_data(batch_size, total_rows)
+      end
+
+      private
+
+      def strategy_class
+        if ::ActiveRecord::Base.connection.raw_connection.respond_to?(:copy_data)
+          ActiveRecordDataLoader::CopyStrategy
+        else
+          ActiveRecordDataLoader::BulkInsertStrategy
+        end
+      end
+    end
+
+    def initialize(logger:, strategy:)
+      @logger = logger
+      @strategy = strategy
+    end
+
+    def load_data(batch_size, total_rows)
+      batch_count = (total_rows / batch_size.to_f).ceil
+
+      logger.info(
+        "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
+        "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
+      )
+      total_time = Benchmark.realtime do
+        load_in_batches(batch_size, total_rows, batch_count)
+      end
+      logger.info(
+        "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
+        "in #{total_time} seconds."
+      )
+    end
+
+    private
+
+    attr_reader :strategy, :logger
+
+    def load_in_batches(batch_size, total_rows, batch_count)
+      total_rows.times.each_slice(batch_size).with_index do |row_numbers, i|
+        time = Benchmark.realtime { strategy.load_batch(row_numbers) }
+
+        logger.debug(
+          "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
+        )
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  VERSION = "0.1.1"
+end

data/script/ci_build.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+bundle exec appraisal
+bundle exec appraisal activerecord-5 rake
+bundle exec appraisal faker rake
+bundle exec appraisal ffaker rake
+bundle exec rake

metadata

@@ -0,0 +1,240 @@
+--- !ruby/object:Gem::Specification
+name: active_record_data_loader
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Alejandro Beiderman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A utility to bulk load test data for performance testing.
+email:
+- abeiderman@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Appraisals
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- active_record_data_loader.gemspec
+- bin/console
+- bin/setup
+- config/database.yml
+- config/database.yml.travis
+- gemfiles/.bundle/config
+- gemfiles/activerecord_5.gemfile
+- gemfiles/activerecord_6.gemfile
+- gemfiles/faker.gemfile
+- gemfiles/ffaker.gemfile
+- lib/active_record_data_loader.rb
+- lib/active_record_data_loader/active_record/belongs_to_configuration.rb
+- lib/active_record_data_loader/active_record/column_configuration.rb
+- lib/active_record_data_loader/active_record/enum_value_generator.rb
+- lib/active_record_data_loader/active_record/integer_value_generator.rb
+- lib/active_record_data_loader/active_record/model_data_generator.rb
+- lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb
+- lib/active_record_data_loader/active_record/text_value_generator.rb
+- lib/active_record_data_loader/bulk_insert_strategy.rb
+- lib/active_record_data_loader/configuration.rb
+- lib/active_record_data_loader/copy_strategy.rb
+- lib/active_record_data_loader/data_faker.rb
+- lib/active_record_data_loader/dsl/definition.rb
+- lib/active_record_data_loader/dsl/model.rb
+- lib/active_record_data_loader/dsl/polymorphic_association.rb
+- lib/active_record_data_loader/loader.rb
+- lib/active_record_data_loader/version.rb
+- log/.keep
+- script/ci_build.sh
+homepage: 
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/abeiderman/active_record_data_loader
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A utility to bulk load test data for performance testing.
+test_files: []