scripterator 1.2.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b2c3c2f5e186a3e9f766ac33a72903f835ae2f5e
-  data.tar.gz: 196848f11699fc575a26708163e6271babb9a8bd
+SHA256:
+  metadata.gz: e6b0a2c0390962d5310ff31c294faa74b5d74186c949ed04170c62f0506dc656
+  data.tar.gz: 123f12425a1ee3e941b5cdbdff78d8f7bdfcf49d968dc4c006ba21f240931856
 SHA512:
-  metadata.gz: 8592a34342273e61e5f23999a5233f4e3ee3d22bc78bd60e3cb640a8ddecf048238b63f823e3ebb874454ad44c00266039e51a14f8c9ee2de6d1b9aab39453f6
-  data.tar.gz: c31e9a4c791c1a18ccdb5245fed0b9d4ffa099c5731614e006b5a1c903d3b2acffa8371c16ef9af64b81e2e92715d85e8c9b4d8cb1c6ada7db0d7c92c6c6e2a4
+  metadata.gz: b90ca2a2ef31b7e112c96d3a9560a0d40f27feae22ccea6e79614387d5527a9f891c630467b66cf14dde73315f0b832d075b680efc3f99edb4709a91ac3baf08
+  data.tar.gz: 65f43d3eebeae7b0b5b64fe9e259ac61b6f43984eed5b976e76660ba3fa06d7b1ca567ce3f1b358aa60c305a5334820857ac645ebed52a10d065a2d890053947

data/.gitignore

@@ -3,7 +3,7 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
+Gemfile*.lock
 InstalledFiles
 _yardoc
 coverage

data/.travis.yml

@@ -1,8 +1,8 @@
 language: ruby
 rvm:
-  - ruby-2.1.6
+  - 2.5.8
 script: bundle exec rspec
-
-env:
-  - 'RAILS_VERSION=3.2'
-  - 'RAILS_VERSION=4.0'
+gemfile:
+  - Gemfile-rails-5.0.x
+  - Gemfile-rails-5.1.x
+  - Gemfile-rails-5.2.x

data/CHANGELOG.md

@@ -1,2 +1,17 @@
+# 1.4.0
+- Update redis to 4+
+- Update rspec
+- Update fakeredis
+
+# 1.3.2
+- Rails 5 support
+
+# 1.3.1
+- Fix things from 1.3.0 that didn't work
+
+# 1.3.0
+- Add support for changing batch size
+- Add support for `after_batch` and `before_batch` callbacks
+
 # 1.2.0
-Add support for ActiveRecord versions 3.0 through 4.2
+- Add support for ActiveRecord versions 3.0 through 4.2

data/Gemfile

@@ -1,6 +1,2 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in scripterator.gemspec
-gemspec
-
-gem 'activerecord', ENV['RAILS_VERSION'] || '~> 3.2'
+ENV['BUNDLE_GEMFILE'] = File.expand_path('../Gemfile-rails-5.2.x', __FILE__)
+Bundler.load

data/Gemfile-rails-5.0.x

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'activerecord', '< 5.1'
+gem 'sqlite3',      '~> 1.3.0'

data/Gemfile-rails-5.1.x

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'activerecord', '< 5.2'
+gem 'sqlite3',      '~> 1.4.0'

data/Gemfile-rails-5.2.x

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'activerecord', '< 6.0'
+gem 'sqlite3',      '~> 1.4.0'

data/README.md

@@ -32,10 +32,18 @@ Scripterator.run "Convert users from legacy auth data" do
     User.skip_some_callbacks_we_want_to_avoid_during_script_running
   end
 
+  before_batch do |batch|
+    # do stuff
+  end
+
   for_each_user do |user|
     user.do_legacy_conversion
   end
 
+  after_batch do |batch|
+    # do stuff
+  end
+
   after do
     # some code to run after everything's finished
   end
@@ -93,6 +101,7 @@ Environment variable options:
 
 - `START`: first model ID to scripterate
 - `END`: last model ID to scripterate
+- `BATCH_SIZE`: how many records to process in a batch
 - `ID_LIST`: comma-delimited list of IDs to scripterate (e.g. "ID_LIST=1,99,440,23")
 - `REDIS_EXPIRATION`: amount of time (in seconds) before Redis result sets (checked IDs and failed IDs) are expired
 

data/lib/scripterator.rb

@@ -18,6 +18,7 @@ module Scripterator
       options = {}.tap do |o|
         o[:start_id]         = ENV['START'].try(:to_i)
         o[:end_id]           = ENV['END'].try(:to_i)
+        o[:batch_size]       = ENV['BATCH_SIZE'].try(:to_i)
         o[:id_list]          = ENV['ID_LIST'].try(:split, ',')
         o[:redis_expiration] = ENV['REDIS_EXPIRATION'].try(:to_i) || config.redis_expiration
       end

data/lib/scripterator/runner.rb

@@ -23,6 +23,7 @@ module Scripterator
       @id_list          = options[:id_list] || []
       @start_id         = options[:start_id] || 1
       @end_id           = options[:end_id]
+      @batch_size       = options[:batch_size] || 1000
       @redis_expiration = options[:redis_expiration]
       @output_stream    = options[:output_stream] || $stdout
 
@@ -34,7 +35,7 @@ module Scripterator
       output_stats
     end
 
-    %w(model before per_record after).each do |callback|
+    %w(model before per_record after before_batch after_batch).each do |callback|
       define_method callback do |&block|
         instance_variable_set "@#{callback}", block
       end
@@ -102,20 +103,25 @@ module Scripterator
     end
 
     def run_loop
-      if @id_list.count > 0
-        run_for_id_list
-      elsif @end_id
-        @id_list = (@start_id..@end_id)
-        run_for_id_list
+      @id_list = (@start_id..@end_id) if @end_id
+      relation = model_finder
+      opts = {
+        batch_size: @batch_size
+      }
+
+      if @id_list.count < 1
+        opts[:start] = @start_id
       else
-        model_finder.find_each(start: @start_id) { |record| transform_one_record(record) }
+        relation = model_finder.where(id: @id_list)
       end
 
-      expire_redis_sets
-    end
+      relation.find_in_batches(opts) do |group|
+        self.instance_exec(group, &@before_batch) if @before_batch
+        group.each { |record| transform_one_record(record) }
+        self.instance_exec(group, &@after_batch) if @after_batch
+      end
 
-    def run_for_id_list
-      model_finder.where(id: @id_list).find_each { |record| transform_one_record(record) }
+      expire_redis_sets
     end
 
     def transform_one_record(record)

data/lib/scripterator/version.rb

@@ -1,3 +1,3 @@
 module Scripterator
-  VERSION = '1.2.0'
+  VERSION = '1.4.0'.freeze
 end

data/scripterator.gemspec

@@ -15,10 +15,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Scripterator::VERSION
 
-  gem.add_dependency 'activerecord', '< 5.0'
-  gem.add_dependency 'redis',        '~> 3.0'
+  gem.add_dependency 'activerecord', '< 6'
+  gem.add_dependency 'redis',        '~> 4.0'
 
-  gem.add_development_dependency 'fakeredis', '~> 0.4'
-  gem.add_development_dependency 'rspec',     '~> 2.13'
-  gem.add_development_dependency 'sqlite3',   '~> 1.3.8'
+  gem.add_development_dependency 'fakeredis', '~> 0.8.0'
+  gem.add_development_dependency 'rspec',     '~> 3.0'
+  gem.add_development_dependency 'sqlite3'
 end

data/spec/runner_spec.rb

@@ -10,6 +10,7 @@ describe Scripterator::Runner do
     Proc.new do
       before          { Widget.before_stuff }
       for_each_widget { |widget| Widget.transform_a_widget(widget) }
+      after_batch     { |batch| Widget.after_batch_stuff(batch) }
     end
   end
 
@@ -17,12 +18,12 @@ describe Scripterator::Runner do
 
   shared_examples_for 'raises an error' do
     specify do
-      expect { subject }.to raise_error
+      expect { subject }.to raise_error(StandardError)
     end
   end
 
   it 'infers the model from the for_each block' do
-    runner.send(:model_finder).should == Widget
+    expect(runner.send(:model_finder)).to eq Widget
   end
 
   context 'when a model block is defined' do
@@ -31,14 +32,15 @@ describe Scripterator::Runner do
         model           { Widget.where(name: 'bla') }
         before          { Widget.before_stuff }
         for_each_widget { |widget| Widget.transform_a_widget(widget) }
+        after_batch     { |batch| Widget.after_batch_stuff(batch) }
       end
     end
     let!(:widget1) { Widget.create(name: 'foo') }
     let!(:widget2) { Widget.create(name: 'bla') }
 
     it 'uses the given model finder code' do
-      Widget.should_receive(:transform_a_widget).once.with(widget2)
-      Widget.should_not_receive(:transform_a_widget).with(widget1)
+      expect(Widget).to receive(:transform_a_widget).once.with(widget2)
+      expect(Widget).not_to receive(:transform_a_widget).with(widget1)
       subject
     end
   end
@@ -57,8 +59,8 @@ describe Scripterator::Runner do
 
     it 'transforms each widget in the list' do
       options[:id_list].each do |id|
-        runner.should_receive(:transform_one_record) do |arg1|
-          arg1.id.should == id
+        expect(runner).to receive(:transform_one_record) do |arg1|
+          expect(arg1.id).to eq id
         end
       end
       subject
@@ -76,14 +78,14 @@ describe Scripterator::Runner do
     it_behaves_like 'raises an error'
 
     it 'does not run any other given blocks' do
-      Widget.should_not_receive :before_stuff
+      expect(Widget).to_not receive(:before_stuff)
       subject rescue nil
     end
   end
 
   context 'when there are no records for the specified model' do
     it 'does not run the per-record block' do
-      Widget.should_not_receive :transform_a_widget
+      expect(Widget).to_not receive(:transform_a_widget)
       subject
     end
   end
@@ -94,8 +96,9 @@ describe Scripterator::Runner do
     before { num_widgets.times { Widget.create! } }
 
     it 'runs the given script blocks' do
-      Widget.should_receive :before_stuff
-      Widget.should_receive(:transform_a_widget).exactly(num_widgets).times
+      expect(Widget).to receive(:before_stuff)
+      expect(Widget).to receive(:transform_a_widget).exactly(num_widgets).times
+      expect(Widget).to receive(:after_batch_stuff)
       subject
     end
 
@@ -104,10 +107,10 @@ describe Scripterator::Runner do
 
       it 'marks only the checked IDs as checked' do
         subject
-        Scripterator.already_run_for?(description, Widget.first.id).should be_false
-        Scripterator.checked_ids(description).should_not include Widget.first.id
-        Scripterator.already_run_for?(description, Widget.last.id).should be_true
-        Scripterator.checked_ids(description).should include Widget.last.id
+        expect(Scripterator.already_run_for?(description, Widget.first.id)).to be_falsey
+        expect(Scripterator.checked_ids(description)).to_not include Widget.first.id
+        expect(Scripterator.already_run_for?(description, Widget.last.id)).to be_truthy
+        expect(Scripterator.checked_ids(description)).to include Widget.last.id
       end
     end
 
@@ -115,20 +118,20 @@ describe Scripterator::Runner do
       let(:checked_ids) { [Widget.first.id] }
 
       before do
-        Scripterator.stub(checked_ids: checked_ids)
-        Scripterator::ScriptRedis.any_instance.stub(already_run_for?: false)
-        Scripterator::ScriptRedis.any_instance.stub(:already_run_for?).with(Widget.first.id).and_return(true)
+        allow(Scripterator).to receive(:checked_ids).and_return(checked_ids)
+        allow_any_instance_of(Scripterator::ScriptRedis).to receive(:already_run_for?).and_return(false)
+        allow_any_instance_of(Scripterator::ScriptRedis).to receive(:already_run_for?).with(Widget.first.id).and_return(true)
       end
 
       it 'only runs the per-record code for unchecked records' do
-        Widget.should_receive(:transform_a_widget).exactly(num_widgets - 1).times
+        expect(Widget).to receive(:transform_a_widget).exactly(num_widgets - 1).times
         subject
       end
     end
 
     context 'when the code for some records fails' do
       before do
-        Widget.stub :transform_a_widget do |widget|
+        allow(Widget).to receive(:transform_a_widget) do |widget|
           raise 'Last widget expl0de' if widget.id == Widget.last.id
           true
         end
@@ -136,8 +139,8 @@ describe Scripterator::Runner do
 
       it 'marks only the failed IDs as failed' do
         subject
-        Scripterator.failed_ids(description).should_not include Widget.first.id
-        Scripterator.failed_ids(description).should include Widget.last.id
+        expect(Scripterator.failed_ids(description)).to_not include Widget.first.id
+        expect(Scripterator.failed_ids(description)).to include Widget.last.id
       end
     end
 
@@ -147,7 +150,7 @@ describe Scripterator::Runner do
 
       it 'runs without Redis' do
         expect { subject }.not_to raise_error
-        Scripterator.checked_ids(description).should be_empty
+        expect(Scripterator.checked_ids(description)).to be_empty
       end
     end
   end

data/spec/support/widget.rb

@@ -10,5 +10,6 @@ end
 class Widget < ActiveRecord::Base
   # some dummy methods for setting message expectations in specs
   def self.before_stuff; end
+  def self.after_batch_stuff(batch); end
   def self.transform_a_widget(widget); true; end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: scripterator
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Ted Dumitrescu
 - Carl Furrow
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2021-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -17,70 +17,70 @@ dependencies:
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: fakeredis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 0.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.13'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.13'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.8
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.8
+        version: '0'
 description: Script iterator for ActiveRecord models
 email:
 - ted@lumoslabs.com
@@ -94,6 +94,9 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
+- Gemfile-rails-5.0.x
+- Gemfile-rails-5.1.x
+- Gemfile-rails-5.2.x
 - LICENSE
 - README.md
 - lib/scripterator.rb
@@ -108,7 +111,7 @@ files:
 homepage: http://lumosity.com
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -123,13 +126,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: DSL for running operations on each of a set of models
 test_files:
 - spec/runner_spec.rb
 - spec/spec_helper.rb
 - spec/support/widget.rb
-has_rdoc: