active_record_data_loader 0.1.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43867d09da0479be24e9a23687d53cd4137f5ff8b47f769183a9bee19d6f8372
-  data.tar.gz: 71ea6ea8d1cbc9821168a0f87e90c7e6e08f2b2053c48905bb0bcfe95f5d37f0
+  metadata.gz: 5c2971599904ef04249cbada28e924103f7a7f37e0d397e047502063bebd4297
+  data.tar.gz: 4b9d6471bb8c01937b459ca21cd8fcff18d2f13950102a63bebb13aa20584fb3
 SHA512:
-  metadata.gz: 2523c1db84ecd726ba333369646a0fe1e9b6ee8e0e0dc5694e2b8a8ef693fe752c8685914d122c7d9561dd05ff3c1d1905a328445a7407c82596b72cf0f5e9ff
-  data.tar.gz: 3c95553f3b39a535739e3fd998f67ca0c5420e37411d229b9284c0ccc99c9b7690d56a835b8fdaadfddc6f8750604f29852f77198b5a178c4cbab00bf3d80a1f
+  metadata.gz: c324c5783427ca9aacb88496e2b76669ade344fdb3d9eb4d640d2579cf9c472eee2a62367478d5211aead5572c155f4beb3ce12c5202503bb80e6f134a565743
+  data.tar.gz: f6f558c48d05973eed5cd9659b5204af780075ee19b8e046ddc1428385e1155a266cdacb052220b1b5354a5e7a50ea04e12c4eefac77002f2fa302bd772ec1e7

data/.travis.yml

@@ -1,11 +1,23 @@
 sudo: false
 language: ruby
+env:
+  - COVERALLS_PARALLEL=true
 rvm:
-  - 2.4.3
+  - 2.3.8
+  - 2.4.6
+  - 2.5.5
+  - 2.6.3
+gemfile:
+  - gemfiles/activerecord_5.gemfile
+  - gemfiles/rails.gemfile
+  - gemfiles/faker.gemfile
+  - gemfiles/ffaker.gemfile
 services:
   - postgresql
-before_install: gem install bundler -v 1.16.1
+notifications:
+  webhooks: https://coveralls.io/webhook
+before_install: "gem update --system && gem install bundler"
 before_script:
   - psql -c 'create database test;' -U postgres
   - cp config/database.yml.travis config/database.yml
-script: ./script/ci_build.sh
+script: "bundle exec rake"

data/Appraisals

@@ -4,6 +4,10 @@ appraise "activerecord-5" do
   gem "activerecord", "~> 5.0"
 end
 
+appraise "rails" do
+  gem "rails"
+end
+
 appraise "faker" do
   gem "faker", ">= 1.9.3"
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    active_record_data_loader (0.1.2)
+    active_record_data_loader (1.0.0)
       activerecord (>= 4.0)
 
 GEM

data/README.md

@@ -1,6 +1,5 @@
 # ActiveRecord Data Loader
 
-[![Gem Version](https://badge.fury.io/rb/active_record_data_loader.svg)](https://badge.fury.io/rb/active_record_data_loader)
 [![Build Status](https://travis-ci.org/abeiderman/active_record_data_loader.svg?branch=master)](https://travis-ci.org/abeiderman/active_record_data_loader)
 [![Coverage Status](https://coveralls.io/repos/github/abeiderman/active_record_data_loader/badge.svg?branch=master&service=github)](https://coveralls.io/github/abeiderman/active_record_data_loader?branch=master)
 [![Maintainability](https://api.codeclimate.com/v1/badges/338904b3f7e8d19a3cb1/maintainability)](https://codeclimate.com/github/abeiderman/active_record_data_loader/maintainability)
@@ -29,7 +28,181 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+The gem will recognize most commonly used column types and attempt to populate with sensible values by default. You can override this behavior as you will see further below.
+
+`belongs_to` associations are recognized automatically. However, data is loaded in the order you define, so you want to make sure the parent model is loaded first.
+
+Polymorphic associations need to be defined explicitly as shown in [Polymorphic associations](#polymorphic-associations).
+
+### Basic usage
+
+Let's say you have the following models:
+```ruby
+class Customer < ApplicationRecord
+end
+
+class Order < ApplicationRecord
+  belongs_to :customer
+end
+```
+
+The following code will create 10,000 customers and 100,000 orders, and will associate the orders to those customers evenly:
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Customer do |m|
+    m.count 10_000
+  end
+
+  model Order do |m|
+    m.count 100_000
+  end
+end
+
+data_loader.load_data
+```
+
+#### Overriding column values
+To provide your own values for columns your can provide a lambda or a constant value:
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Customer do |m|
+    m.count 10_000
+    m.column :name, -> { %w[Jane John Mary Matt].sample }
+    m.column :country, "USA"
+    m.column :terminated_at, nil
+  end
+
+  ...
+end
+
+data_loader.load_data
+```
+
+### Controlling associations
+Let's say that you have certain restrictions on orders depending on country. You would want to test data to follow those restrictions which means orders cannot be randomly associated to any customer. You can control that by providing an `eligible_set` on the association.
+
+In this example, we are creating 25K orders for customers in CAN with a CAD currency, 25K for customers in MEX with a MXN currency, and 50K for those in USA with a USD currency.
+
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Customer do |m|
+    m.count 10_000
+    m.column :country, -> { %w[CAN MXN USA].sample }
+  end
+
+  model Order do |m|
+    m.count 25_000
+    m.column :currency, "CAD"
+    m.belongs_to :customer, eligible_set: -> { Customer.where(country: "CAN") }
+  end
+  
+  model Order do |m|
+    m.count 25_000
+    m.column :currency, "MXN"
+    m.belongs_to :customer, eligible_set: -> { Customer.where(country: "MEX") }
+  end
+  
+   model Order do |m|
+    m.count 50_000
+    m.column :currency, "USD"
+    m.belongs_to :customer, eligible_set: -> { Customer.where(country: "USA") }
+  end
+end
+
+data_loader.load_data
+```
+
+### Polymorphic associations
+
+If you have a polymorphic `belongs_to` association, you will need to define that explicitly for it to be populated.
+
+Let's assume the following models where an order could belong to either a person or a business:
+```ruby
+class Person < ApplicationRecord
+  has_many :orders
+end
+
+class Business < ApplicationRecord
+  has_many :orders
+end
+
+class Order < ApplicationRecord
+  belongs_to :customer, polymorphic: true
+end
+```
+
+In order to populate the `customer` association in orders, you would specify them like this:
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Person do |m|
+    m.count 5_000
+  end
+
+  model Business do |m|
+    m.count 5_000
+  end
+
+  model Order do |m|
+    m.count 100_000
+    
+    m.polymorphic :customer do |c|
+      c.model Person
+      c.model Business
+    end
+  end
+end
+
+data_loader.load_data
+```
+
+You can also provide a `weight` to each of the target models if you want to control how they are distributed. If you wanted to have twice as many orders for `Person` than for `Business`, it would look like this:
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Person do |m|
+    m.count 5_000
+  end
+
+  model Business do |m|
+    m.count 5_000
+  end
+
+  model Order do |m|
+    m.count 100_000
+    
+    m.polymorphic :customer do |c|
+      c.model Person, weight: 2
+      c.model Business, weight: 1
+    end
+  end
+end
+
+data_loader.load_data
+```
+
+Additionaly, you can also provide an `eligible_set` to control which records to limit the association to:
+```ruby
+data_loader = ActiveRecordDataLoader.define do
+  model Person do |m|
+    m.count 5_000
+  end
+
+  model Business do |m|
+    m.count 5_000
+    m.column :country, -> { %w[CAN USA].sample }
+  end
+
+  model Order do |m|
+    m.count 100_000
+    
+    m.polymorphic :customer do |c|
+      c.model Person, weight: 2
+      c.model Business, weight: 1, eligible_set: -> { Business.where(country: "USA") }
+    end
+  end
+end
+
+data_loader.load_data
+```
 
 ## Development
 

data/gemfiles/rails.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails"
+
+gemspec path: "../"

data/lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb

@@ -38,12 +38,20 @@ module ActiveRecordDataLoader
       def possible_values
         @possible_values ||= begin
           values = @settings.models.keys.map do |klass|
-            [klass.name, klass.all.pluck(klass.primary_key).to_a]
+            [klass.name, base_query(klass).pluck(klass.primary_key).to_a]
           end.to_h
 
           @settings.weighted_models.map { |klass| [klass.name, values[klass.name]] }
         end
       end
+
+      def base_query(klass)
+        if @settings.queries[klass]&.respond_to?(:call)
+          @settings.queries[klass].call.all
+        else
+          klass.all
+        end
+      end
     end
   end
 end

data/lib/active_record_data_loader/configuration.rb

@@ -7,13 +7,23 @@ module ActiveRecordDataLoader
     def initialize(
       default_batch_size: 100_000,
       default_row_count: 1,
-      logger: Logger.new(STDOUT, level: :info),
+      logger: nil,
       statement_timeout: "2min"
     )
       @default_batch_size = default_batch_size
       @default_row_count = default_row_count
-      @logger = logger
+      @logger = logger || default_logger
       @statement_timeout = statement_timeout
     end
+
+    private
+
+    def default_logger
+      if defined?(Rails) && Rails.respond_to?(:logger)
+        Rails.logger
+      else
+        Logger.new(STDOUT, level: :info)
+      end
+    end
   end
 end

data/lib/active_record_data_loader/dsl/polymorphic_association.rb

@@ -3,16 +3,18 @@
 module ActiveRecordDataLoader
   module Dsl
     class PolymorphicAssociation
-      attr_reader :model_class, :name, :models
+      attr_reader :model_class, :name, :models, :queries
 
       def initialize(model_class, name)
         @model_class = model_class
         @name = name
         @models = {}
+        @queries = {}
       end
 
-      def model(klass, weight: 1)
+      def model(klass, weight: 1, eligible_set: nil)
         @models[klass] = weight.to_i
+        @queries[klass] = eligible_set if eligible_set
       end
 
       def weighted_models

data/lib/active_record_data_loader/loader.rb

@@ -39,6 +39,7 @@ module ActiveRecordDataLoader
       batch_count = (total_rows / batch_size.to_f).ceil
 
       logger.info(
+        "[ActiveRecordDataLoader] "\
         "Loading #{total_rows} row(s) into '#{strategy.table_name}' via #{strategy.name}. "\
         "#{batch_size} row(s) per batch, #{batch_count} batch(es)."
       )
@@ -46,6 +47,7 @@ module ActiveRecordDataLoader
         load_in_batches(batch_size, total_rows, batch_count)
       end
       logger.info(
+        "[ActiveRecordDataLoader] "\
         "Completed loading #{total_rows} row(s) into '#{strategy.table_name}' "\
         "in #{total_time} seconds."
       )
@@ -61,6 +63,7 @@ module ActiveRecordDataLoader
           time = Benchmark.realtime { strategy.load_batch(row_numbers, connection) }
 
           logger.debug(
+            "[ActiveRecordDataLoader] "\
             "Completed batch #{i + 1}/#{batch_count}, #{row_numbers.count} row(s) in #{time} seconds"
           )
         end

data/lib/active_record_data_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActiveRecordDataLoader
-  VERSION = "0.1.2"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_record_data_loader
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Alejandro Beiderman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-11 00:00:00.000000000 Z
+date: 2019-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -194,6 +194,7 @@ files:
 - gemfiles/activerecord_6.gemfile
 - gemfiles/faker.gemfile
 - gemfiles/ffaker.gemfile
+- gemfiles/rails.gemfile
 - lib/active_record_data_loader.rb
 - lib/active_record_data_loader/active_record/belongs_to_configuration.rb
 - lib/active_record_data_loader/active_record/column_configuration.rb
@@ -213,7 +214,6 @@ files:
 - lib/active_record_data_loader/loader.rb
 - lib/active_record_data_loader/version.rb
 - log/.keep
-- script/ci_build.sh
 homepage: 
 licenses:
 - MIT

data/script/ci_build.sh

@@ -1,4 +0,0 @@
-#! /bin/bash
-bundle exec appraisal
-bundle exec appraisal rake
-bundle exec rake coveralls:push