scripterator 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2c3c2f5e186a3e9f766ac33a72903f835ae2f5e
-  data.tar.gz: 196848f11699fc575a26708163e6271babb9a8bd
+  metadata.gz: 42473644a78bcc8e3175774fb6b6783a93ee1ac1
+  data.tar.gz: ca62a4ba9252989df7090cd4a31aa8692b23262d
 SHA512:
-  metadata.gz: 8592a34342273e61e5f23999a5233f4e3ee3d22bc78bd60e3cb640a8ddecf048238b63f823e3ebb874454ad44c00266039e51a14f8c9ee2de6d1b9aab39453f6
-  data.tar.gz: c31e9a4c791c1a18ccdb5245fed0b9d4ffa099c5731614e006b5a1c903d3b2acffa8371c16ef9af64b81e2e92715d85e8c9b4d8cb1c6ada7db0d7c92c6c6e2a4
+  metadata.gz: 76076d599c2f543d563a35c1770056a400fad021cfd8894f87588aec0bdefb268f8cf5d0609cba6dc4308ca6594a4e4aaf55314840ea4c603fad13682fd45ad1
+  data.tar.gz: 7d0c093432672e53539b50f6c7613a37fcb08a20f8a9df168d90fcfe095b74e964b68ff58a8b136d1a0ab66c3e6048d8192e0fc5128504f01e08edfd5819048f

data/CHANGELOG.md

@@ -1,2 +1,6 @@
+# 1.3.0
+- Add support for changing batch size
+- Add support for `after_batch` and `before_batch` callbacks
+
 # 1.2.0
-Add support for ActiveRecord versions 3.0 through 4.2
+- Add support for ActiveRecord versions 3.0 through 4.2

data/README.md

@@ -32,10 +32,18 @@ Scripterator.run "Convert users from legacy auth data" do
     User.skip_some_callbacks_we_want_to_avoid_during_script_running
   end
 
+  before_batch do |batch|
+    # do stuff
+  end
+
   for_each_user do |user|
     user.do_legacy_conversion
   end
 
+  after_batch do |batch|
+    # do stuff
+  end
+
   after do
     # some code to run after everything's finished
   end

data/lib/scripterator/runner.rb

@@ -23,6 +23,7 @@ module Scripterator
       @id_list          = options[:id_list] || []
       @start_id         = options[:start_id] || 1
       @end_id           = options[:end_id]
+      @batch_size       = options[:batch_size] || 1000
       @redis_expiration = options[:redis_expiration]
       @output_stream    = options[:output_stream] || $stdout
 
@@ -34,7 +35,7 @@ module Scripterator
       output_stats
     end
 
-    %w(model before per_record after).each do |callback|
+    %w(model before per_record after before_batch after_batch).each do |callback|
       define_method callback do |&block|
         instance_variable_set "@#{callback}", block
       end
@@ -102,20 +103,25 @@ module Scripterator
     end
 
     def run_loop
-      if @id_list.count > 0
-        run_for_id_list
-      elsif @end_id
-        @id_list = (@start_id..@end_id)
-        run_for_id_list
+      @id_list = (@start_id..@end_id) if @end_id
+      relation = model_finder
+      opts = {
+        batch_size: @batch_size
+      }
+
+      if @id_list.empty?
+        opts[:start] = @start_id
       else
-        model_finder.find_each(start: @start_id) { |record| transform_one_record(record) }
+        relation = model_finder.where(id: @id_list)
       end
 
-      expire_redis_sets
-    end
+      relation.find_in_batches(opts) do |group|
+        self.instance_exec(group, &@before_batch) if @before_batch
+        group.each { |record| transform_one_record(record) }
+        self.instance_exec(group, &@after_batch) if @after_batch
+      end
 
-    def run_for_id_list
-      model_finder.where(id: @id_list).find_each { |record| transform_one_record(record) }
+      expire_redis_sets
     end
 
     def transform_one_record(record)

data/lib/scripterator/version.rb

@@ -1,3 +1,3 @@
 module Scripterator
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/spec/runner_spec.rb

@@ -10,6 +10,7 @@ describe Scripterator::Runner do
     Proc.new do
       before          { Widget.before_stuff }
       for_each_widget { |widget| Widget.transform_a_widget(widget) }
+      after_batch     { |batch| Widget.after_batch_stuff(batch) }
     end
   end
 
@@ -31,6 +32,7 @@ describe Scripterator::Runner do
         model           { Widget.where(name: 'bla') }
         before          { Widget.before_stuff }
         for_each_widget { |widget| Widget.transform_a_widget(widget) }
+        after_batch     { |batch| Widget.after_batch_stuff(batch) }
       end
     end
     let!(:widget1) { Widget.create(name: 'foo') }
@@ -96,6 +98,7 @@ describe Scripterator::Runner do
     it 'runs the given script blocks' do
       Widget.should_receive :before_stuff
       Widget.should_receive(:transform_a_widget).exactly(num_widgets).times
+      Widget.should_receive :after_batch_stuff
       subject
     end
 

data/spec/support/widget.rb

@@ -10,5 +10,6 @@ end
 class Widget < ActiveRecord::Base
   # some dummy methods for setting message expectations in specs
   def self.before_stuff; end
+  def self.after_batch_stuff(batch); end
   def self.transform_a_widget(widget); true; end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scripterator
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Ted Dumitrescu
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: DSL for running operations on each of a set of models
@@ -132,4 +132,3 @@ test_files:
 - spec/runner_spec.rb
 - spec/spec_helper.rb
 - spec/support/widget.rb
-has_rdoc: