active_record_data_loader 0.1.1

data/config/database.yml

@@ -0,0 +1,10 @@
+postgres:
+  adapter: "postgresql"
+  host: "127.0.0.1"
+  database: "test"
+  username: "test"
+  password: "test"
+
+sqlite3:
+  adapter: "sqlite3"
+  database: ":memory:"

data/config/database.yml.travis

@@ -0,0 +1,7 @@
+postgres:
+  adapter: "postgresql"
+  database: "test"
+
+sqlite3:
+  adapter: "sqlite3"
+  database: ":memory:"

data/gemfiles/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_RETRY: "1"

data/gemfiles/activerecord_5.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 5.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.0.0.rc1"
+
+gemspec path: "../"

data/gemfiles/faker.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "faker", ">= 1.9.3"
+
+gemspec path: "../"

data/gemfiles/ffaker.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "ffaker"
+
+gemspec path: "../"

data/lib/active_record_data_loader.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "active_record_data_loader/version"
+require "active_record"
+require "active_record_data_loader/configuration"
+require "active_record_data_loader/data_faker"
+require "active_record_data_loader/active_record/integer_value_generator"
+require "active_record_data_loader/active_record/text_value_generator"
+require "active_record_data_loader/active_record/enum_value_generator"
+require "active_record_data_loader/active_record/column_configuration"
+require "active_record_data_loader/active_record/belongs_to_configuration"
+require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
+require "active_record_data_loader/active_record/model_data_generator"
+require "active_record_data_loader/dsl/polymorphic_association"
+require "active_record_data_loader/dsl/model"
+require "active_record_data_loader/dsl/definition"
+require "active_record_data_loader/copy_strategy"
+require "active_record_data_loader/bulk_insert_strategy"
+require "active_record_data_loader/loader"
+
+module ActiveRecordDataLoader
+  def self.define(config = ActiveRecordDataLoader.configuration, &block)
+    LoaderProxy.new(
+      configuration,
+      ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
+    )
+  end
+
+  def self.configure(&block)
+    @configuration = ActiveRecordDataLoader::Configuration.new.tap { |c| block.call(c) }
+  end
+
+  def self.configuration
+    @configuration ||= ActiveRecordDataLoader::Configuration.new
+  end
+
+  class LoaderProxy
+    def initialize(configuration, definition)
+      @configuration = configuration
+      @definition = definition
+    end
+
+    def load_data
+      definition.models.map do |m|
+        generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
+          model: m.klass,
+          column_settings: m.columns,
+          polymorphic_settings: m.polymorphic_associations
+        )
+
+        ActiveRecordDataLoader::Loader.load_data(
+          data_generator: generator,
+          batch_size: m.batch_size,
+          total_rows: m.row_count,
+          logger: configuration.logger
+        )
+      end
+    end
+
+    private
+
+    attr_reader :definition, :configuration
+  end
+end

data/lib/active_record_data_loader/active_record/belongs_to_configuration.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class BelongsToConfiguration
+      def self.config_for(ar_association:)
+        raise "#{name} does not support polymorphic associations" if ar_association.polymorphic?
+
+        { ar_association.join_foreign_key.to_sym => new(ar_association).foreign_key_func }
+      end
+
+      def initialize(ar_association)
+        @ar_association = ar_association
+      end
+
+      def foreign_key_func
+        -> { possible_values.sample }
+      end
+
+      private
+
+      def possible_values
+        @possible_values ||= @ar_association.klass.all.pluck(@ar_association.join_primary_key).to_a
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/column_configuration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class ColumnConfiguration
+      class << self
+        VALUE_GENERATORS = {
+          enum: EnumValueGenerator,
+          integer: IntegerValueGenerator,
+          string: TextValueGenerator,
+          text: TextValueGenerator,
+        }.freeze
+
+        def config_for(model_class:, ar_column:)
+          raise_error_if_not_supported(model_class, ar_column)
+
+          {
+            ar_column.name.to_sym => VALUE_GENERATORS[ar_column.type].generator_for(
+              model_class: model_class,
+              ar_column: ar_column
+            ),
+          }
+        end
+
+        def supported?(model_class:, ar_column:)
+          return false if model_class.reflect_on_association(ar_column.name)
+
+          VALUE_GENERATORS.keys.include?(ar_column.type)
+        end
+
+        private
+
+        def raise_error_if_not_supported(model_class, ar_column)
+          return if supported?(model_class: model_class, ar_column: ar_column)
+
+          raise <<~ERROR
+            Column '#{ar_column.name}' of type '#{ar_column.type}' in model '#{model_class.name}' not supported"
+          ERROR
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/enum_value_generator.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class EnumValueGenerator
+      class << self
+        def generator_for(model_class:, ar_column:)
+          values = enum_values_for(model_class, ar_column.sql_type)
+          -> { values.sample }
+        end
+
+        private
+
+        def enum_values_for(model_class, enum_type)
+          model_class
+            .connection
+            .execute("SELECT unnest(enum_range(NULL::#{enum_type}))::text")
+            .map(&:values)
+            .flatten
+            .compact
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/integer_value_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class IntegerValueGenerator
+      class << self
+        def generator_for(model_class:, ar_column:)
+          range_limit = [(256**number_of_bytes(ar_column)) / 2 - 1, 1_000_000_000].min
+
+          -> { rand(0..range_limit) }
+        end
+
+        private
+
+        def number_of_bytes(ar_column)
+          ar_column.limit || 8
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/model_data_generator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class ModelDataGenerator
+      attr_reader :table
+
+      def initialize(model:, column_settings:, polymorphic_settings: [])
+        @model_class = model
+        @table = model.table_name
+        @polymorphic_settings = polymorphic_settings
+        @column_settings = column_settings
+      end
+
+      def column_list
+        columns.keys
+      end
+
+      def generate_row(row_number)
+        column_list.map { |c| column_data(row_number, c) }
+      end
+
+      private
+
+      def column_data(row_number, column)
+        column_value = columns[column]
+        return column_value unless column_value.respond_to?(:call)
+
+        if column_value.arity == 2
+          column_value.call(row_number, column)
+        elsif column_value.arity == 1
+          column_value.call(row_number)
+        else
+          column_value.call
+        end
+      end
+
+      def columns
+        @columns ||= [
+          own_columns_config,
+          belongs_to_config,
+          polymorphic_config,
+          @column_settings,
+        ].reduce(:merge)
+      end
+
+      def own_columns_config
+        @model_class
+          .columns_hash
+          .reject { |name| name == @model_class.primary_key }
+          .select { |_, c| ColumnConfiguration.supported?(model_class: @model_class, ar_column: c) }
+          .map { |_, c| ColumnConfiguration.config_for(model_class: @model_class, ar_column: c) }
+          .reduce({}, :merge)
+      end
+
+      def belongs_to_config
+        @model_class
+          .reflect_on_all_associations
+          .select(&:belongs_to?)
+          .reject(&:polymorphic?)
+          .map { |assoc| BelongsToConfiguration.config_for(ar_association: assoc) }
+          .reduce({}, :merge)
+      end
+
+      def polymorphic_config
+        @polymorphic_settings
+          .map { |s| PolymorphicBelongsToConfiguration.config_for(polymorphic_settings: s) }
+          .reduce({}, :merge)
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class PolymorphicBelongsToConfiguration
+      def self.config_for(polymorphic_settings:)
+        ar_association = polymorphic_settings.model_class.reflect_on_association(
+          polymorphic_settings.name
+        )
+        raise "#{name} only supports polymorphic associations" unless ar_association.polymorphic?
+
+        new(polymorphic_settings, ar_association).polymorphic_config
+      end
+
+      def initialize(settings, ar_association)
+        @settings = settings
+        @ar_association = ar_association
+        @model_count = settings.weighted_models.size
+      end
+
+      def polymorphic_config
+        {
+          @ar_association.foreign_type.to_sym => ->(row_number) { foreign_type(row_number) },
+          @ar_association.foreign_key.to_sym => ->(row_number) { foreign_key(row_number) },
+        }
+      end
+
+      private
+
+      def foreign_type(row_number)
+        possible_values[row_number % @model_count][0]
+      end
+
+      def foreign_key(row_number)
+        possible_values[row_number % @model_count][1].sample
+      end
+
+      def possible_values
+        @possible_values ||= begin
+          values = @settings.models.keys.map do |klass|
+            [klass.name, klass.all.pluck(klass.primary_key).to_a]
+          end.to_h
+
+          @settings.weighted_models.map { |klass| [klass.name, values[klass.name]] }
+        end
+      end
+    end
+  end
+end

data/lib/active_record_data_loader/active_record/text_value_generator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module ActiveRecordDataLoader
+  module ActiveRecord
+    class TextValueGenerator
+      GENERATORS = {
+        likely_a_person_full_name?: -> { ActiveRecordDataLoader::DataFaker.person_name },
+        likely_a_first_name?: -> { ActiveRecordDataLoader::DataFaker.first_name },
+        likely_a_middle_name?: -> { ActiveRecordDataLoader::DataFaker.middle_name },
+        likely_a_last_name?: -> { ActiveRecordDataLoader::DataFaker.last_name },
+        likely_an_organization_name?: -> { ActiveRecordDataLoader::DataFaker.company_name },
+      }.freeze
+
+      class << self
+        def generator_for(model_class:, ar_column:)
+          scenario = GENERATORS.keys.find { |m| send(m, model_class, ar_column) }
+          generator = GENERATORS.fetch(scenario, -> { SecureRandom.uuid })
+
+          -> { truncate_if_needed(generator.call, ar_column.limit) }
+        end
+
+        private
+
+        def truncate_if_needed(value, limit)
+          return value if limit.nil?
+
+          value[0...limit]
+        end
+
+        def likely_a_person_full_name?(model_class, ar_column)
+          ar_column.name.downcase == "name" && likely_a_person?(model_class)
+        end
+
+        def likely_a_first_name?(_, ar_column)
+          ar_column.name.downcase == "first_name" || ar_column.name.downcase == "firstname"
+        end
+
+        def likely_a_middle_name?(_, ar_column)
+          ar_column.name.downcase == "middle_name" || ar_column.name.downcase == "middlename"
+        end
+
+        def likely_a_last_name?(_, ar_column)
+          ar_column.name.downcase == "last_name" || ar_column.name.downcase == "lastname"
+        end
+
+        def likely_an_organization_name?(model_class, ar_column)
+          ar_column.name.downcase == "company_name" ||
+            ar_column.name.downcase == "companyname" ||
+            ar_column.name.downcase == "business_name" ||
+            ar_column.name.downcase == "businessname" ||
+            (ar_column.name.downcase == "name" && likely_an_organization?(model_class))
+        end
+
+        def likely_a_person?(model_class)
+          %w[
+            customer human employee person user
+          ].any? do |word|
+            model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
+          end
+        end
+
+        def likely_an_organization?(model_class)
+          %w[
+            business company enterprise legalentity organization
+          ].any? do |word|
+            model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
+          end
+        end
+      end
+    end
+  end
+end