active_record_data_loader 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2971599904ef04249cbada28e924103f7a7f37e0d397e047502063bebd4297
-  data.tar.gz: 4b9d6471bb8c01937b459ca21cd8fcff18d2f13950102a63bebb13aa20584fb3
+  metadata.gz: 8ae60492b342e71d8a1a2e20aff31794af784f1009346f4d5dc4375dd939bb48
+  data.tar.gz: ab5be5bdf1cb5af69f5f8a1ac65ea8f64ee1338398c7f6cae6e15e6276d2e954
 SHA512:
-  metadata.gz: c324c5783427ca9aacb88496e2b76669ade344fdb3d9eb4d640d2579cf9c472eee2a62367478d5211aead5572c155f4beb3ce12c5202503bb80e6f134a565743
-  data.tar.gz: f6f558c48d05973eed5cd9659b5204af780075ee19b8e046ddc1428385e1155a266cdacb052220b1b5354a5e7a50ea04e12c4eefac77002f2fa302bd772ec1e7
+  metadata.gz: 8b860a06433a0661a765428f37df596ac0b3fd46ff447fbf3edfa5cf50f89dcda0f146b25074959ca5aee792dff153a56eb37888075ae10649d6cde1e598c53d
+  data.tar.gz: dc9b43011f8c1b8c7b663b650b5ba11e43b9dd040a8ef6b7a58b24c7da903b1c963104a06c39f45e5ebaa4f01e815d8d6d565ded2fd9b5a186dcd50ed69b2e7c

data/.github/workflows/build.yml

@@ -0,0 +1,51 @@
+name: Build
+
+on: [push, workflow_dispatch]
+
+jobs:
+  build:
+    name: Build + Test
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:11
+        ports:
+          - "2345:5432"
+        env:
+          POSTGRES_USER: test
+          POSTGRES_PASSWORD: test
+      mysql:
+        image: mysql:5
+        ports:
+          - "3306:3306"
+        env:
+          MYSQL_ROOT_PASSWORD: test
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_DATABASE: test
+    strategy:
+      matrix:
+        ruby: [2.5.9, 2.6.7, 2.7.3]
+        gemfile: [activerecord_5, rails, faker, ffaker]
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Wait for DBs to be ready
+        run: bundle exec rake wait_for_test_db
+
+      - name: Run tests
+        run: bundle exec rake
+
+      - name: Coveralls
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}

data/.github/workflows/gem-push.yml

@@ -0,0 +1,29 @@
+name: Ruby Gem
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: "${{ secrets.RUBYGEMS_API_KEY }}"

data/.rubocop.yml

@@ -1,5 +1,20 @@
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
+  NewCops: enable
+  SuggestExtensions: false
+
+Layout/LineLength:
+  Max: 110
+  Exclude: ["*.gemspec"]
+
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+
+Lint/RaiseException:
+  Enabled: true
+
+Lint/StructNewOverride:
+  Enabled: true
 
 Lint/UnusedMethodArgument:
   AllowUnusedKeywordArguments: true
@@ -11,20 +26,38 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Exclude: ["spec/**/*", "*.gemspec"]
 
-Metrics/LineLength:
-  Max: 110
-  Exclude: ["*.gemspec"]
+Metrics/ClassLength:
+  Exclude: ["spec/**/*", "*.gemspec"]
 
 Metrics/MethodLength:
   Max: 15
   Exclude: ["spec/**/*"]
 
-Style/FrozenStringLiteralComment:
-  Exclude: ["gemfiles/*"]
+Metrics/ParameterLists:
+  Max: 5
+  Exclude: ["lib/active_record_data_loader/configuration.rb"]
+
+Style/CaseLikeIf:
+  Enabled: false
 
 Style/Documentation:
   Enabled: false
 
+Style/ExponentialNotation:
+  Enabled: true
+
+Style/FrozenStringLiteralComment:
+  Exclude: ["gemfiles/*"]
+
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true
+
 Style/StringLiterals:
   EnforcedStyle: double_quotes
 

data/CHANGELOG.md

@@ -0,0 +1,42 @@
+# Change log
+
+## [v1.2.0] - 2021-11-14
+
+[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.1.0...v1.2.0)
+
+### Changes:
+* Add `:file` output option for generating a SQL script instead of loading the data into the database.
+* Fix some connection handling issues when a custom connection factory is provided.
+
+## [v1.1.0] - 2021-05-01
+
+[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.0.2...v1.1.0)
+
+### Changes:
+* Bump ruby version requirement to >= 2.5
+* Bump activerecord requirement to >= 5.0
+
+## [v1.0.2] - 2019-07-05
+
+[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.0.1...v1.0.2)
+
+### Changes:
+* Add support for MySQL enums
+* Accept a connection factory lambda as part of the configuration
+
+## [v1.0.1] - 2019-06-16
+
+[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.0.0...v1.0.1)
+
+### Changes:
+* Generate values for datetime column types. This also fixes the fact that `created_at` and `updated_at` were not being populated by default.
+
+## [v1.0.0] - 2019-06-15
+
+Initial stable release
+
+[v1.0.0]: https://github.com/abeiderman/active_record_data_loader/releases/tag/v1.0.0
+[v1.0.1]: https://github.com/abeiderman/active_record_data_loader/releases/tag/v1.0.1
+[v1.0.2]: https://github.com/abeiderman/active_record_data_loader/releases/tag/v1.0.2
+[v1.1.0]: https://github.com/abeiderman/active_record_data_loader/releases/tag/v1.1.0
+[v1.2.0]: https://github.com/abeiderman/active_record_data_loader/releases/tag/v1.2.0

data/Gemfile.lock

@@ -1,107 +1,109 @@
 PATH
   remote: .
   specs:
-    active_record_data_loader (1.0.0)
-      activerecord (>= 4.0)
+    active_record_data_loader (1.2.0)
+      activerecord (>= 5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.3)
-      activesupport (= 5.2.3)
-    activerecord (5.2.3)
-      activemodel (= 5.2.3)
-      activesupport (= 5.2.3)
-      arel (>= 9.0)
-    activesupport (5.2.3)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     appraisal (2.2.0)
       bundler
       rake
       thor (>= 0.14.0)
-    arel (9.0.0)
-    ast (2.4.0)
+    ast (2.4.2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    coveralls (0.8.23)
-      json (>= 1.8, < 3)
-      simplecov (~> 0.16.1)
-      term-ansicolor (~> 1.3)
-      thor (>= 0.19.4, < 2.0)
-      tins (~> 1.6)
+    concurrent-ruby (1.1.9)
     diff-lcs (1.3)
-    docile (1.3.1)
-    i18n (1.6.0)
+    docile (1.4.0)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.2)
-    json (2.2.0)
-    method_source (0.9.2)
-    minitest (5.11.3)
-    parallel (1.17.0)
-    parser (2.6.3.0)
-      ast (~> 2.4.0)
-    pg (1.1.4)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    mysql2 (0.5.3)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pg (1.2.3)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rainbow (3.0.0)
-    rake (12.3.2)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-collection_matchers (1.1.3)
+    rake (13.0.1)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-collection_matchers (1.2.0)
       rspec-expectations (>= 2.99.0.beta1)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.3)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubocop (0.68.1)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (1.22.3)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.6)
-    ruby-progressbar (1.10.0)
-    simplecov (0.16.1)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov-lcov (0.8.0)
+    simplecov_json_formatter (0.1.3)
     sqlite3 (1.4.1)
-    term-ansicolor (1.7.1)
-      tins (~> 1.0)
     thor (0.20.3)
-    thread_safe (0.3.6)
-    tins (1.20.3)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    unicode-display_width (1.5.0)
+    timecop (0.9.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   ruby
+  x86_64-darwin-19
 
 DEPENDENCIES
   active_record_data_loader!
   appraisal
   bundler (>= 1.16)
-  coveralls
+  mysql2
   pg
   pry
-  rake (~> 12.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rspec-collection_matchers
   rubocop
+  simplecov
+  simplecov-lcov
   sqlite3
+  timecop
 
 BUNDLED WITH
-   2.0.1
+   2.2.31

data/README.md

@@ -1,6 +1,6 @@
-# ActiveRecord Data Loader
+# active_record_data_loader
 
-[![Build Status](https://travis-ci.org/abeiderman/active_record_data_loader.svg?branch=master)](https://travis-ci.org/abeiderman/active_record_data_loader)
+[![Build Status](https://github.com/abeiderman/active_record_data_loader/actions/workflows/build.yml/badge.svg)](https://github.com/abeiderman/active_record_data_loader/actions/workflows/build.yml)
 [![Coverage Status](https://coveralls.io/repos/github/abeiderman/active_record_data_loader/badge.svg?branch=master&service=github)](https://coveralls.io/github/abeiderman/active_record_data_loader?branch=master)
 [![Maintainability](https://api.codeclimate.com/v1/badges/338904b3f7e8d19a3cb1/maintainability)](https://codeclimate.com/github/abeiderman/active_record_data_loader/maintainability)
 
@@ -10,6 +10,10 @@ Efficiently bulk load data for your ActiveRecord models with a simple DSL.
 
 Load, performance, and stress tests often require setting up a realistic amount of data in your database. This gem is intended to help organize that data load and make it more maintainable than having a collection of SQL scripts.
 
+#### How is this different from using _factory_bot_?
+
+This gem is not a replacement for [factory_bot](https://github.com/thoughtbot/factory_bot). It solves a different use case. While _factory_bot_ is great for organizing test data and reducing duplication in your functional tests, _active_record_data_loader_ is focused around bulk loading data for performance tests. The purpose of _active_record_data_loader_ is loading large amounts of data as efficiently as possible while providing a DSL that helps with maintainability.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -95,13 +99,13 @@ data_loader = ActiveRecordDataLoader.define do
     m.column :currency, "CAD"
     m.belongs_to :customer, eligible_set: -> { Customer.where(country: "CAN") }
   end
-  
+
   model Order do |m|
     m.count 25_000
     m.column :currency, "MXN"
     m.belongs_to :customer, eligible_set: -> { Customer.where(country: "MEX") }
   end
-  
+
    model Order do |m|
     m.count 50_000
     m.column :currency, "USD"
@@ -144,7 +148,7 @@ data_loader = ActiveRecordDataLoader.define do
 
   model Order do |m|
     m.count 100_000
-    
+
     m.polymorphic :customer do |c|
       c.model Person
       c.model Business
@@ -168,7 +172,7 @@ data_loader = ActiveRecordDataLoader.define do
 
   model Order do |m|
     m.count 100_000
-    
+
     m.polymorphic :customer do |c|
       c.model Person, weight: 2
       c.model Business, weight: 1
@@ -193,7 +197,7 @@ data_loader = ActiveRecordDataLoader.define do
 
   model Order do |m|
     m.count 100_000
-    
+
     m.polymorphic :customer do |c|
       c.model Person, weight: 2
       c.model Business, weight: 1, eligible_set: -> { Business.where(country: "USA") }
@@ -204,6 +208,85 @@ end
 data_loader.load_data
 ```
 
+### Configuration options
+
+You can define global configuration options like this:
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.logger = ActiveSupport::Logger.new("my_file.log", level: :debug)
+  c.statement_timeout = "5min"
+end
+```
+
+Or you can create a configuration object for the specific data loader instance rather than globally:
+
+```ruby
+config = ActiveRecordDataLoader::Configuration.new(
+  c.logger = ActiveSupport::Logger.new("my_file.log", level: :debug)
+  c.statement_timeout = "5min"
+)
+loader = ActiveRecordDataLoader.define(config) do
+  model Company do |m|
+    m.count 10
+  end
+
+  # ... more definitions
+end
+```
+
+#### statement_timeout
+
+This is currently only used for Postgres connections to adjust the `statement_timeout` value for the connection. The default is `2min`. Depending on the size of the batches you are loading and overall size of the tables you may need to increase this value:
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.statement_timeout = "5min"
+end
+```
+
+#### connection_factory
+
+The `connection_factory` option accepts a lambda that should return a connection object whenever executed. If not specified, the default behavior is to retrieve a connection using `ActiveRecord::Base.connection`. You can configure it like this:
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.connection_factory = -> { MyCustomConnectionHandler.open_connection }
+end
+```
+
+#### output
+
+The `output` option accepts either `:connection` or `:file`. The default behavior is `:connection` which means the data will be loaded into the database using the database connection.
+
+If `:file` is specified, instead of the data being loaded into the database, a script file will be generated. This script file can then be executed manually to load the data. This can be helpful if you need to load the same data multiple times. For example if you are profiling different alternatives in your code and you want to see how each performs with a fully loaded database. In that case you would want to have the same data starting point for each alternative you evaluate. By generating the script file ahead of time, it would be significantly faster to load that data over and over by executing the existing script.
+
+Here are some examples on how to use the `output` option:
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.output = :connection  # This is the default behavior
+end
+```
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.output = :file  # Outputs to a file with a default name
+end
+```
+
+```ruby
+ActiveRecordDataLoader.configure do |c|
+  c.output = { type: :file, filename: "./my_script.sql" }  # Outputs to the provided file
+end
+```
+
+When using the `:file` type with Postgres, the resulting script will have `\COPY` commands which reference CSV files that contain the data batches to be copied. The CSV files will be created along side the SQL script and will have a naming convention of using the table name and the rows range for the given batch. For example `./my_script_customers_1_to_1000.csv`. Each `\COPY` command in the SQL file will reference the corresponding CSV file so all you need to do is execute the SQL file using `psql`:
+
+```bash
+psql -h my-db-host -U my_user -f my_script.sql
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -220,4 +303,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the ActiveRecord Data Loader project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/abeiderman/active_record_data_loader/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the _active_record_data_loader_ project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/abeiderman/active_record_data_loader/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -3,10 +3,16 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require "rubocop/rake_task"
-require "coveralls/rake/task"
 
 RSpec::Core::RakeTask.new(:spec)
 RuboCop::RakeTask.new(:rubocop)
-Coveralls::RakeTask.new
 
 task default: [:spec, :rubocop]
+
+task :wait_for_test_db do
+  require "active_record_data_loader"
+  require "./spec/active_record_helper"
+
+  ActiveRecordHelper.wait_for_mysql
+  ActiveRecordHelper.wait_for_postgres
+end

data/active_record_data_loader.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
     spec.metadata["source_code_uri"] = "https://github.com/abeiderman/active_record_data_loader"
   else
     raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
+          "public gem pushes."
   end
 
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
@@ -30,18 +30,21 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.3.0"
+  spec.required_ruby_version = ">= 2.5.0"
 
-  spec.add_dependency "activerecord", ">= 4.0"
+  spec.add_dependency "activerecord", ">= 5.0"
 
   spec.add_development_dependency "appraisal"
   spec.add_development_dependency "bundler", ">= 1.16"
-  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "mysql2"
   spec.add_development_dependency "pg"
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-collection_matchers"
   spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov-lcov"
   spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "timecop"
 end

data/config/database.yml

@@ -1,6 +1,7 @@
 postgres:
   adapter: "postgresql"
   host: "127.0.0.1"
+  port: "2345"
   database: "test"
   username: "test"
   password: "test"
@@ -8,3 +9,11 @@ postgres:
 sqlite3:
   adapter: "sqlite3"
   database: ":memory:"
+
+mysql:
+  adapter: "mysql2"
+  host: "127.0.0.1"
+  port: "3306"
+  database: "test"
+  username: "test"
+  password: "test"

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.9"
+services:
+  postgres:
+    image: postgres:11
+    ports:
+      - "2345:5432"
+    environment:
+      - POSTGRES_USER=test
+      - POSTGRES_PASSWORD=test
+  mysql:
+    image: mysql:5
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=test
+      - MYSQL_USER=test
+      - MYSQL_PASSWORD=test
+      - MYSQL_DATABASE=test

data/gemfiles/activerecord_6.gemfile

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.0.0.rc1"
+gem "activerecord", "~>6.1"
 
 gemspec path: "../"

data/lib/active_record_data_loader/active_record/belongs_to_configuration.rb

@@ -25,7 +25,7 @@ module ActiveRecordDataLoader
       end
 
       def base_query
-        if @query&.respond_to?(:call)
+        if @query.respond_to?(:call)
           @query.call.all
         else
           @ar_association.klass.all