evil-seed 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5cdab6653f529f88d18a2bd9edebf9ee64a4694
-  data.tar.gz: 244bfb7f54e1c4ca8c7451b79741a132d0a77dc9
+  metadata.gz: 411501048ad3322c19a3db3c781b9e7e19d48725
+  data.tar.gz: 8587a602745f2d08375c9fb6636412c974df963d
 SHA512:
-  metadata.gz: 24ac68a4ae6aa95bb7ebd65688686f60737e4e4de68fab3ce0d269b1b7b3fe82dbab501b623df5a463d7a2ad62283794e65426f1bfbb75b15d6179c75cbc9622
-  data.tar.gz: f6c0e7be2b76b401f4e6ad3c6755cdcab5fb7f568fdc91316889540f20462250e1bf2a4f9b8ce97439b344ab36e3683a8d0ad64a68896f3afb041a5c5ee074bc
+  metadata.gz: bf912a9462601ae9b0b47bf987e1031789a46012a60138db5a5ba9d354739a6044d54839935c3c8366e0fce0daf04f4005270b16bdde4612ebb7798e8f7b721b
+  data.tar.gz: 45219968ddf66748d6d0f40c8a39f86b635b5564d1047266711e14d94ec30679d9a9bd9c02743490ca30de9b7867f83dc7e62cd69018df90910de13458abc395

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 /gemfiles/*.lock
+*.gem

data/.travis.yml

@@ -1,21 +1,6 @@
 cache: bundler
 sudo: false
 language: ruby
-rvm:
-  - 2.4.1
-  - 2.3.4
-  - 2.2.7
-gemfile:
-  - gemfiles/activerecord-5-0.gemfile
-  - gemfiles/activerecord-4-2.gemfile
-env:
-  - DB=postgresql
-  - DB=sqlite
-  - DB=mysql
-before_install:
-  - gem install bundler -v 1.14.6
-  - bundle install
-  - appraisal install
 
 addons:
   apt:
@@ -23,3 +8,22 @@ addons:
       - travis-ci/sqlite3
     packages:
       - sqlite3
+
+matrix:
+  include:
+    - rvm: 2.4.1
+      gemfile: gemfiles/activerecord_5_0.gemfile
+      env: "DB=sqlite"
+    - rvm: 2.4.1
+      gemfile: gemfiles/activerecord_5_0.gemfile
+      env: "DB=postgresql"
+    - rvm: 2.4.1
+      gemfile: gemfiles/activerecord_5_0.gemfile
+      env: "DB=mysql"
+    - rvm: 2.3.4
+      gemfile: gemfiles/activerecord_4_2.gemfile
+      env: "DB=postgresql"
+    - rvm: 2.3.4
+      gemfile: gemfiles/activerecord_4_2.gemfile
+      env: "DB=sqlite"
+    # Please note that gem can't be tested against MySQL on ActiveRecord 4.2 (Dump and restore test doesn't work)!

data/Rakefile

@@ -10,4 +10,16 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
+ADAPTERS = %w[postgresql sqlite mysql].freeze
+
+namespace :test do
+  ADAPTERS.each do |adapter|
+    task adapter => ["#{adapter}:env", :test]
+
+    namespace adapter do
+      task(:env) { ENV['DB'] = adapter }
+    end
+  end
+end
+
 task default: :test

data/gemfiles/activerecord_4_2.gemfile

@@ -7,4 +7,4 @@ gem "pg"
 gem "sqlite3"
 gem "activerecord", "~> 4.2.0"
 
-gemspec path: "../"
+gemspec path: ".."

data/gemfiles/activerecord_5_0.gemfile

@@ -7,4 +7,4 @@ gem "pg"
 gem "sqlite3"
 gem "activerecord", "~> 5.0.0"
 
-gemspec path: "../"
+gemspec path: ".."

data/lib/evil_seed/refinements/in_batches.rb

@@ -0,0 +1,223 @@
+require 'active_record/relation/batches'
+
+module EvilSeed
+  module Refinements
+    # This backports ActiveRecord::Relation#in_batches method for ActiveRecord 4.2
+    # This module contains this method and +BatchEnumerator+ class picked from Ruby on Rails codebase at 2017-05-14
+    # See https://github.com/rails/rails/commit/25cee1f0373aa3b1d893413a959375480e0ac684
+    # The ActiveRecord MIT license is obviously compatible with our license (MIT also)
+    module InBatches
+      refine ActiveRecord::Relation do
+
+        # This is from active_record/core
+        def arel_attribute(name, table = klass.arel_table) # :nodoc:
+          name = klass.attribute_alias(name) if klass.attribute_alias?(name)
+          table[name]
+        end
+
+        class BatchEnumerator
+          include Enumerable
+
+          def initialize(of: 1000, start: nil, finish: nil, relation:) #:nodoc:
+            @of       = of
+            @relation = relation
+            @start = start
+            @finish = finish
+          end
+
+          # Looping through a collection of records from the database (using the
+          # +all+ method, for example) is very inefficient since it will try to
+          # instantiate all the objects at once.
+          #
+          # In that case, batch processing methods allow you to work with the
+          # records in batches, thereby greatly reducing memory consumption.
+          #
+          #   Person.in_batches.each_record do |person|
+          #     person.do_awesome_stuff
+          #   end
+          #
+          #   Person.where("age > 21").in_batches(of: 10).each_record do |person|
+          #     person.party_all_night!
+          #   end
+          #
+          # If you do not provide a block to #each_record, it will return an Enumerator
+          # for chaining with other methods:
+          #
+          #   Person.in_batches.each_record.with_index do |person, index|
+          #     person.award_trophy(index + 1)
+          #   end
+          def each_record
+            return to_enum(:each_record) unless block_given?
+
+            @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true).each do |relation|
+              relation.records.each { |record| yield record }
+            end
+          end
+
+          # Delegates #delete_all, #update_all, #destroy_all methods to each batch.
+          #
+          #   People.in_batches.delete_all
+          #   People.where('age < 10').in_batches.destroy_all
+          #   People.in_batches.update_all('age = age + 1')
+          [:delete_all, :update_all, :destroy_all].each do |method|
+            define_method(method) do |*args, &block|
+              @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false).each do |relation|
+                relation.send(method, *args, &block)
+              end
+            end
+          end
+
+          # Yields an ActiveRecord::Relation object for each batch of records.
+          #
+          #   Person.in_batches.each do |relation|
+          #     relation.update_all(awesome: true)
+          #   end
+          def each
+            enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false)
+            return enum.each { |relation| yield relation } if block_given?
+            enum
+          end
+        end
+
+        # Yields ActiveRecord::Relation objects to work with a batch of records.
+        #
+        #   Person.where("age > 21").in_batches do |relation|
+        #     relation.delete_all
+        #     sleep(10) # Throttle the delete queries
+        #   end
+        #
+        # If you do not provide a block to #in_batches, it will return a
+        # BatchEnumerator which is enumerable.
+        #
+        #   Person.in_batches.with_index do |relation, batch_index|
+        #     puts "Processing relation ##{batch_index}"
+        #     relation.each { |relation| relation.delete_all }
+        #   end
+        #
+        # Examples of calling methods on the returned BatchEnumerator object:
+        #
+        #   Person.in_batches.delete_all
+        #   Person.in_batches.update_all(awesome: true)
+        #   Person.in_batches.each_record(&:party_all_night!)
+        #
+        # ==== Options
+        # * <tt>:of</tt> - Specifies the size of the batch. Default to 1000.
+        # * <tt>:load</tt> - Specifies if the relation should be loaded. Default to false.
+        # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
+        # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
+        # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
+        #                               an order is present in the relation.
+        #
+        # Limits are honored, and if present there is no requirement for the batch
+        # size, it can be less than, equal, or greater than the limit.
+        #
+        # The options +start+ and +finish+ are especially useful if you want
+        # multiple workers dealing with the same processing queue. You can make
+        # worker 1 handle all the records between id 1 and 9999 and worker 2
+        # handle from 10000 and beyond by setting the +:start+ and +:finish+
+        # option on each worker.
+        #
+        #   # Let's process from record 10_000 on.
+        #   Person.in_batches(start: 10_000).update_all(awesome: true)
+        #
+        # An example of calling where query method on the relation:
+        #
+        #   Person.in_batches.each do |relation|
+        #     relation.update_all('age = age + 1')
+        #     relation.where('age > 21').update_all(should_party: true)
+        #     relation.where('age <= 21').delete_all
+        #   end
+        #
+        # NOTE: If you are going to iterate through each record, you should call
+        # #each_record on the yielded BatchEnumerator:
+        #
+        #   Person.in_batches.each_record(&:party_all_night!)
+        #
+        # NOTE: It's not possible to set the order. That is automatically set to
+        # ascending on the primary key ("id ASC") to make the batch ordering
+        # consistent. Therefore the primary key must be orderable, e.g an integer
+        # or a string.
+        #
+        # NOTE: By its nature, batch processing is subject to race conditions if
+        # other processes are modifying the database.
+        def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
+          relation = self
+          unless block_given?
+            return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
+          end
+
+          if arel.orders.present?
+            act_on_ignored_order(error_on_ignore)
+          end
+
+          batch_limit = of
+          if limit_value
+            remaining   = limit_value
+            batch_limit = remaining if remaining < batch_limit
+          end
+
+          relation = relation.reorder(batch_order).limit(batch_limit)
+          relation = apply_limits(relation, start, finish)
+          batch_relation = relation
+
+          loop do
+            if load
+              records = batch_relation.records
+              ids = records.map(&:id)
+              yielded_relation = where(primary_key => ids)
+              yielded_relation.load_records(records)
+            else
+              ids = batch_relation.pluck(primary_key)
+              yielded_relation = where(primary_key => ids)
+            end
+
+            break if ids.empty?
+
+            primary_key_offset = ids.last
+            raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
+
+            yield yielded_relation
+
+            break if ids.length < batch_limit
+
+            if limit_value
+              remaining -= ids.length
+
+              if remaining == 0
+                # Saves a useless iteration when the limit is a multiple of the
+                # batch size.
+                break
+              elsif remaining < batch_limit
+                relation = relation.limit(remaining)
+              end
+            end
+
+            batch_relation = relation.where(arel_attribute(primary_key).gt(primary_key_offset))
+          end
+        end
+
+        private
+
+        def apply_limits(relation, start, finish)
+          relation = relation.where(arel_attribute(primary_key).gteq(start)) if start
+          relation = relation.where(arel_attribute(primary_key).lteq(finish)) if finish
+          relation
+        end
+
+        def batch_order
+          "#{quoted_table_name}.#{quoted_primary_key} ASC"
+        end
+
+        def act_on_ignored_order(error_on_ignore)
+          raise_error = (error_on_ignore.nil? ? klass.error_on_ignored_order : error_on_ignore)
+
+          if raise_error
+            raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
+          elsif logger
+            logger.warn(ORDER_IGNORE_MESSAGE)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/evil_seed/relation_dumper.rb

@@ -1,5 +1,12 @@
 # frozen_string_literal: true
 
+# As method ActiveRecord::Relation#in_batches is available only since ActiveRecord 5.0
+# we will backport it only for us via refinements for ActiveRecord 4.2 compatibility.
+unless ActiveRecord::Batches.instance_methods(false).include?(:in_batches)
+  require_relative 'refinements/in_batches'
+  using EvilSeed::Refinements::InBatches
+end
+
 module EvilSeed
   # This class performs actual dump generation for single relation and all its not yet loaded dependencies
   #

data/lib/evil_seed/root_dumper.rb

@@ -23,7 +23,9 @@ module EvilSeed
     # @param output [IO] Stream to write SQL dump into
     def call
       association_path = model_class.model_name.singular
-      RelationDumper.new(model_class.where(*root.constraints), self, association_path).call
+      relation = model_class.all
+      relation = relation.where(*root.constraints) if root.constraints.any? # without arguments returns not a relation
+      RelationDumper.new(relation, self, association_path).call
     end
 
     # @return [Boolean] +true+ if limits are NOT reached and +false+ otherwise

data/lib/evil_seed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EvilSeed
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: evil-seed
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrey Novikov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -152,6 +152,7 @@ files:
 - lib/evil_seed/configuration/root.rb
 - lib/evil_seed/dumper.rb
 - lib/evil_seed/record_dumper.rb
+- lib/evil_seed/refinements/in_batches.rb
 - lib/evil_seed/relation_dumper.rb
 - lib/evil_seed/root_dumper.rb
 - lib/evil_seed/version.rb