each_in_batches-RobertAudi 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a482b4eaec5896e31e88f91d3a6348a40440fba5
4
+ data.tar.gz: 6d07ce1a9611975a93f91728c6b571313cd6e56b
5
+ SHA512:
6
+ metadata.gz: 3c0e5364093e000cc1dc77b98055cf241b22703bb9ef1b976fd33d3180c932ab8d6c1ea294e9f376c8bbf7230b29670dc3b1bbfb3cf9a6f191d0b87c92698657
7
+ data.tar.gz: 72f7777c4061f07caa7339dbef4d07cf1e8b59595c1bf2844aec7e9b5a122c1e9c864a9044641b454c77d5d164514b47da822512692cb6e9dc22195b21401629
@@ -0,0 +1,15 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
15
+ /.idea
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --color
2
+ --require spec_helper
3
+ --format documentation
@@ -0,0 +1,13 @@
1
+ language: ruby
2
+ cache: bundler
3
+ sudo: false
4
+ rvm:
5
+ - 1.9
6
+ - 2.0
7
+ - 2.1
8
+ - 2.2
9
+ - ruby-head
10
+ matrix:
11
+ allow_failures:
12
+ - rvm: ruby-head
13
+ - rvm: 1.9.3
@@ -0,0 +1,13 @@
1
+ # Contributor Code of Conduct
2
+
3
+ As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
4
+
5
+ We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
6
+
7
+ Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
8
+
9
+ Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
10
+
11
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
12
+
13
+ This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in each_in_batches.gemspec
4
+ gemspec
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2008 Peter H. Boling
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,185 @@
1
+ # EachInBatches
2
+
3
+ Better than Rails Batching. Can you batch backwards for destructive tasks? `each_in_batches` can.
4
+
5
+ | Project | EachInBatches |
6
+ |------------------------ | ----------------- |
7
+ | gem name | each_in_batches |
8
+ | license | MIT |
9
+ | expert support | [![Get help on Codementor](https://cdn.codementor.io/badges/get_help_github.svg)](https://www.codementor.io/peterboling?utm_source=github&utm_medium=button&utm_term=peterboling&utm_campaign=github) |
10
+ | download rank | [![Total Downloads](https://img.shields.io/gem/rt/each_in_batches.svg)](https://rubygems.org/gems/each_in_batches) |
11
+ | version | [![Gem Version](https://badge.fury.io/rb/each_in_batches.png)](http://badge.fury.io/rb/each_in_batches) |
12
+ | dependencies | [![Dependency Status](https://gemnasium.com/pboling/each_in_batches.png)](https://gemnasium.com/pboling/each_in_batches) |
13
+ | code quality | [![Code Climate](https://codeclimate.com/github/pboling/each_in_batches.png)](https://codeclimate.com/github/pboling/each_in_batches) |
14
+ | inline documenation | [![Inline docs](http://inch-ci.org/github/pboling/each_in_batches.png)](http://inch-ci.org/github/pboling/each_in_batches) |
15
+ | continuous integration | [![Build Status](https://secure.travis-ci.org/pboling/each_in_batches.png?branch=master)](https://travis-ci.org/pboling/each_in_batches) |
16
+ | test coverage | [![Coverage Status](https://coveralls.io/repos/pboling/each_in_batches/badge.png)](https://coveralls.io/r/pboling/each_in_batches) |
17
+ | homepage | [on Github.com][homepage] |
18
+ | documentation | [on Rdoc.info][documentation] |
19
+ | live chat | [![Join the chat at https://gitter.im/pboling/each_in_batches](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pboling/each_in_batches?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) |
20
+ | Spread ~♡ⓛⓞⓥⓔ♡~ | [on Coderbits][coderbits], [on Coderwall][coderwall] |
21
+
22
+ [semver]: http://semver.org/
23
+ [pvc]: http://docs.rubygems.org/read/chapter/16#page74
24
+ [railsbling]: http://www.railsbling.com
25
+ [peterboling]: http://www.peterboling.com
26
+ [coderbits]: https://coderbits.com/pboling
27
+ [coderwall]: http://coderwall.com/pboling
28
+ [documentation]: http://rdoc.info/github/pboling/each_in_batches/frames
29
+ [homepage]: https://github.com/pboling/each_in_batches
30
+
31
+ (Originally BolingForBatches)
32
+
33
+ ### NOTE:
34
+ I am resurrecting this code because I still have this recurring need, and Rail's native batching doesn't cut mustard.
35
+ It is some of my most ancient code, and it isn't pretty, but I hope to improve it over time.
36
+
37
+ I often need to execute really large computations on really large data sets.
38
+ I usually end up writing a rake task to do it, which calls methods in my models.
39
+ But something about the process bugged me. Each time I had to re-implement my
40
+ 'batching code' that allowed me to not chew up GB after GB of memory due to
41
+ klass.find(:all, :include => [:everything_under_the_sun]). Re-implementation of
42
+ the same logic over and over across many projects is not very DRY, so I got out
43
+ my blow torch and lit it up. The difficulty was that the part that was different
44
+ each time I batched was at the center of the code, right in the middle of the
45
+ batch loop. But I didn't let that stop me!
46
+
47
+ ## Why this plugin is way better than standard Rails batching
48
+ 1. I've been doing batching in Rails a lot longer than Rails has.
49
+ 2. Metrics. I measure stuff.
50
+ 3. I can batch from the top down (a.k.a backwards), making it possible to DELETE things in batches.
51
+ A. If you've never tried using the built-in rails batching for deleting millions of records... don't start now. Use this gem instead.
52
+ 4. Merged in the EachInBatches fork (from Brian Kidd):
53
+ I needed to iterate over the results and perform more actions than a single
54
+ method would provide. I didn't want to write a method in my app that performed
55
+ the needed functionality as I felt the plugin should support this directly.
56
+ I modified the original plugin so that it takes a block instead of a method.
57
+ It will pass the object instance to the block. It works pretty much the same
58
+ as Class.find(:all).each {|x| do something}, except in batches n that you
59
+ specify with :batch_size.
60
+
61
+ ## Installation
62
+
63
+ Add this line to your application's Gemfile:
64
+
65
+ ```ruby
66
+ gem 'each_in_batches'
67
+ ```
68
+
69
+ And then execute:
70
+
71
+ $ bundle
72
+
73
+ Or install it yourself as:
74
+
75
+ $ gem install each_in_batches
76
+
77
+ ## Usage
78
+
79
+ To create a new Batch, call `Batch#new` pass it the class and any additional arguments (all as a hash).
80
+
81
+ batch = EachInBatches::Batch.new(:arel => Payment.canceled.order("transaction_id ASC"), :batch_size => 50)
82
+
83
+ To process the batched data, pass a block to `Batch#run` the same way you would to an object in a block like `Klass.all.each {|x| x.do_something }`.
84
+ `Batch#run` will pass the data to your block, one at a time, in batches set by the :batch_size argument.
85
+
86
+ batch.run {|x| puts x.id; puts x.transaction_id}
87
+
88
+ Print the results!
89
+
90
+ batch.print_results
91
+
92
+ Or...
93
+
94
+ Consolidate your code if you prefer
95
+
96
+ EachInBatches::Batch.new(:arel => Payment.canceled.order("transaction_id ASC"), batch_size => 50, :show_results => true).run{|x| puts x.id; puts x.transaction_id}
97
+
98
+ ## Configuration
99
+
100
+ Arguements for the initializer (Batch.new) method are:
101
+
102
+ Required:
103
+
104
+ :arel - Usage: :arel => Payment.canceled.order("transaction_id ASC")
105
+ Required, as this is the class that will be batched
106
+
107
+ Optional:
108
+
109
+ :verbose - Usage: :verbose => true or false
110
+ Sets verbosity of output
111
+ Default: false (if not provided)
112
+
113
+ :batch_size - Usage: :batch_size => x
114
+ Where x is some number.
115
+ How many AR Objects should be processed at once?
116
+ Default: 50 (if not provided)
117
+
118
+ :last_batch - Usage: :last_batch => x
119
+ Where x is some number.
120
+ Only process up to and including batch #x.
121
+ Batch numbers start at 0 for the first batch.
122
+ Default: won't be used (no limit if not provided)
123
+
124
+ :first_batch - Usage: first_batch => x
125
+ Where x is some number.
126
+ Begin processing batches beginning at batch #x.
127
+ Batch numbers start at 0 for the first batch.
128
+ Default: won't be used (no offset if not provided)
129
+
130
+ :show_results - Usage: :show_results => true or false
131
+ Prints statistics about the results of Batch#run.
132
+ Default: true if verbose is set to true and :show_results is not provided, otherwise false
133
+
134
+ ## Output
135
+
136
+ Interpreting the output:
137
+
138
+ '[O]' means the batch was skipped due to an offset.
139
+ '[L]' means the batch was skipped due to a limit.
140
+ '[P]' means the batch is processing.
141
+ '[C]' means the batch is complete.
142
+ and yes... it was a coincidence. This class is not affiliated with 'one laptop per child'
143
+
144
+ ## License
145
+
146
+ Copyright ©2008-2015 Peter H. Boling, Brian Kidd, released under the MIT license
147
+
148
+ ## Development
149
+
150
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
151
+
152
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
153
+
154
+ ## Maintenance
155
+
156
+ To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
157
+
158
+ ## Versioning
159
+
160
+ This library aims to adhere to [Semantic Versioning 2.0.0](http://semver.org/).
161
+ Violations of this scheme should be reported as bugs. Specifically,
162
+ if a minor or patch version is released that breaks backward
163
+ compatibility, a new version should be immediately released that
164
+ restores compatibility. Breaking changes to the public API will
165
+ only be introduced with new major versions.
166
+
167
+ As a result of this policy, you can (and should) specify a
168
+ dependency on this gem using the [Pessimistic Version Constraint](http://docs.rubygems.org/read/chapter/16#page74) with two digits of precision.
169
+
170
+ For example:
171
+
172
+ spec.add_dependency 'each_in_batches', '~> 0.0'
173
+
174
+ ## Contributing
175
+
176
+ 1. Fork it ( https://github.com/[my-github-username]/each_in_batches/fork )
177
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
178
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
179
+ 4. Push to the branch (`git push origin my-new-feature`)
180
+ 5. Make sure to add tests!
181
+ 6. Create a new Pull Request
182
+
183
+ ## Contributors
184
+
185
+ See the [Network View](https://github.com/pboling/each_in_batches/network)
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ begin
4
+ require "rspec/core/rake_task"
5
+ desc 'Run specs'
6
+ RSpec::Core::RakeTask.new(:spec) do |t|
7
+ t.ruby_opts = '-w'
8
+ end
9
+ task :default => :spec
10
+ rescue LoadError
11
+ end
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "each_in_batches"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'each_in_batches/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "each_in_batches-RobertAudi"
8
+ spec.version = EachInBatches::VERSION
9
+ spec.authors = ["Peter Boling"]
10
+ spec.email = ["peter.boling@gmail.com"]
11
+
12
+ spec.summary = "Batch Processing of Records with Blocks in Rails"
13
+ spec.description = "Batch Processing of Records with Blocks in Rails"
14
+ spec.homepage = "https://github.com/pboling/each_in_batches"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = "exe"
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "activerecord", ">= 3.2", "< 5.0"
22
+ spec.add_development_dependency "bundler"
23
+ spec.add_development_dependency "rake", "~> 10.0"
24
+ spec.add_development_dependency "rspec", "~> 3.2"
25
+ end
@@ -0,0 +1,281 @@
1
+ # BolingForBatches
2
+ # Copyright ©2008-2015 Peter H. Boling, Brian Kidd, released under the MIT license
3
+ # Gem Plugin for Rails: A Better Way To Run Heavy Queries
4
+ # License: MIT License
5
+ # Labels: Ruby, Rails, Gem
6
+ # Project owners:
7
+ # Peter Boling, Brian Kidd
8
+ require "each_in_batches/version"
9
+ require "active_record"
10
+
11
+ module EachInBatches
12
+
13
+ class Batch
14
+
15
+ attr_accessor :arel
16
+ attr_accessor :verbose
17
+ attr_accessor :batch_size
18
+ attr_accessor :backwards
19
+ attr_accessor :last_batch
20
+ attr_accessor :first_batch
21
+ attr_accessor :skipped_batches
22
+ attr_accessor :offset_array
23
+ attr_accessor :total_records
24
+ attr_accessor :size_of_last_run
25
+ attr_accessor :extra_run
26
+ attr_accessor :num_runs
27
+ attr_accessor :total_time
28
+ attr_accessor :elapsed_time
29
+ attr_accessor :start_time
30
+ attr_accessor :end_time
31
+ attr_accessor :overhead_time
32
+ attr_accessor :completion_times
33
+ attr_accessor :show_results
34
+
35
+ def print_debug
36
+ print "verbose: #{verbose}\nbatch_size: #{batch_size}\nbackwards: #{backwards}\nlast_batch: #{last_batch}\nfirst_batch: #{first_batch}\noffset_array: #{offset_array}\ntotal_records: #{total_records}\nsize_of_last_run: #{size_of_last_run}\nextra_run: #{extra_run}\nnum_runs: #{num_runs}\ntotal_time: #{total_time}\nelapsed_time: #{elapsed_time}\nstart_time: #{start_time}\nend_time: #{end_time}\noverhead_time: #{overhead_time}\ncompletion_times: #{completion_times.inspect}\nshow_results: #{show_results.inspect}\n"
37
+ end
38
+
39
+ def self.help_text
40
+ <<-HEREDOC
41
+ Arguments for the initializer (Batch.new) method are:
42
+
43
+ Required:
44
+
45
+ :arel - Usage: :arel => MyClass.some_scope.order("some_column ASC")
46
+ Required, as this is the class that will be batched
47
+
48
+ Optional:
49
+
50
+ :backwards - Usage: :backwards => true or false
51
+ Whether or not the batches should be processed in reverse order or not.
52
+ NOTE: deletions must be processed backwards or you eat the set as you process
53
+ and end the run half way through
54
+ Default: false (if not provided)
55
+
56
+ :verbose - Usage: :verbose => true or false
57
+ Sets verbosity of output
58
+ Default: false (if not provided)
59
+
60
+ :batch_size - Usage: :batch_size => x
61
+ Where x is some number.
62
+ How many AR Objects should be processed at once?
63
+ Default: 50 (if not provided)
64
+
65
+ :last_batch - Usage: :last_batch => x
66
+ Where x is some number.
67
+ Only process up to and including batch #x.
68
+ Batch numbers start at 0 for the first batch.
69
+ Default: won't be used (no limit if not provided)
70
+
71
+ :first_batch - Usage: first_batch => x
72
+ Where x is some number.
73
+ Begin processing batches beginning at batch #x.
74
+ Batch numbers start at 0 for the first batch.
75
+ Default: won't be used (no offset if not provided)
76
+
77
+ :show_results - Usage: :show_results => true or false
78
+ Prints statistics about the results of Batch#run.
79
+ Default: true if verbose is set to true and :show_results is not provided, otherwise false
80
+
81
+ EXAMPLE:
82
+
83
+ To create a new Batch, call Batch#new and pass it the class and any additional arguements (all as a hash).
84
+
85
+ batch = EachInBatches::Batch.new(:arel => Payment.canceled.order("transaction_id ASC"), :batch_size => 50)
86
+
87
+ To process the batched data, pass a block to Batch#run the same way you would to an object returned by
88
+
89
+ Klass.all.each {|x| x.method}
90
+
91
+ Batch#run will pass the data to your block, one at a time, in batches set by the :batch_size argument.
92
+
93
+ batch.run {|x| puts x.id; puts x.transaction_id}
94
+
95
+ Print the results!
96
+
97
+ batch.print_results
98
+
99
+ Or...
100
+
101
+ Consolidate your code if you prefer
102
+
103
+ EachInBatches::Batch.new(:arel => Payment.canceled.order("transaction_id ASC"), :batch_size => 50, :show_results => true).run{|x| puts x.id; puts x.transaction_id}
104
+
105
+ Interpreting the output:
106
+ '[O]' means the batch was skipped due to an offset.
107
+ '[L]' means the batch was skipped due to a limit.
108
+ '[P]' means the batch is processing.
109
+ '[C]' means the batch is complete.
110
+ and yes... it was a coincidence. This class is not affiliated with 'one laptop per child'
111
+ HEREDOC
112
+ end
113
+
114
+ def self.check(*args)
115
+ if args.empty?
116
+ puts self.help_text and return false
117
+ #Are the values of these parameters going to be valid integers?
118
+ elsif args.first[:batch_size] && (args.first[:batch_size].to_s.gsub(/\d/,'foo') == args.first[:batch_size].to_s)
119
+ puts self.help_text and return false
120
+ elsif args.first[:last_batch] && (args.first[:last_batch].to_s.gsub(/\d/,'foo') == args.first[:last_batch].to_s)
121
+ puts self.help_text and return false
122
+ elsif args.first[:first_batch] && (args.first[:first_batch].to_s.gsub(/\d/,'foo') == args.first[:first_batch].to_s)
123
+ puts self.help_text and return false
124
+ else
125
+ return true
126
+ end
127
+ end
128
+
129
+ def initialize(*args)
130
+ return false unless Batch.check(*args)
131
+ @arel = args.first[:arel]
132
+ @verbose = args.first[:verbose].blank? ? false : args.first[:verbose]
133
+ @backwards = args.first[:backwards].nil? ? false : !(args.first[:backwards] == 'false' || args.first[:backwards] == false)
134
+ @batch_size = args.first[:batch_size] ? args.first[:batch_size].is_a?(Integer) ? args.first[:batch_size] : args.first[:batch_size].to_i : 50
135
+ @last_batch = args.first[:last_batch] ? args.first[:last_batch].is_a?(Integer) ? args.first[:last_batch] : args.first[:last_batch].to_i : false
136
+ @first_batch = args.first[:first_batch] ? args.first[:first_batch].is_a?(Integer) ? args.first[:first_batch] : args.first[:first_batch].to_i : 0
137
+ @show_results = case
138
+ when args.first[:show_results].blank? && !verbose?; false
139
+ when args.first[:show_results].blank? && verbose?; true
140
+ else args.first[:show_results]
141
+ end
142
+ @total_time = 0
143
+ @skipped_batches = []
144
+
145
+ puts "Counting Records..." if verbose?
146
+ @total_records = @arel.count
147
+ @num_runs = @total_records / @batch_size
148
+ @size_of_last_run = @total_records.modulo(@batch_size)
149
+
150
+ if @size_of_last_run > 0
151
+ @num_runs += 1
152
+ @extra_run = true
153
+ else
154
+ @extra_run = false
155
+ end
156
+
157
+ puts "Records: #{@total_records}, Batches: #{@num_runs}" if verbose?
158
+
159
+ @last_batch = @num_runs - 1 unless @num_runs == 0 || @last_batch #because batch numbers start at 0 like array indexes, but only if it was not set in *args
160
+
161
+ current_batch = 0
162
+ @offset_array = Array.new
163
+ if verbose?
164
+ puts "Batch Numbering Begins With 0 (ZERO) and counts up"
165
+ puts "Batch Size (SQL Limit): #{@batch_size}" #This is the SQL Limit
166
+ puts "First Batch # to run: #{@first_batch}" #This is the number of the first batch to run
167
+ puts "Last Batch # to run: #{@last_batch}" # This is the number of the last batch to run
168
+ puts "Batches Before First and After Last will be skipped."
169
+ puts "Creating Batches:\n"
170
+ end
171
+ while current_batch < @num_runs
172
+ @offset_array << (current_batch * @batch_size)
173
+ print "." if verbose?
174
+ current_batch += 1
175
+ end
176
+ puts " #{@num_runs} Batches Created" if verbose?
177
+ #in order to use batching for record deletion, the offsets need to start with largest first
178
+ if @backwards
179
+ @offset_array.reverse!
180
+ puts "Backwards Mode:" if verbose?
181
+ else
182
+ puts "Normal Mode:" if verbose?
183
+ end
184
+ if verbose?
185
+ puts " First Offset: #{@offset_array.first}"
186
+ puts " Last Offset: #{@offset_array.last}"
187
+ # technically the last run doesn't need a limit, and we don't technically use a limit on the last run,
188
+ # but there are only that many records left to process,
189
+ # so the effect is the same as if a limit were applied.
190
+ # We do need the limit when running the batches backwards, however
191
+ if @extra_run
192
+ if @backwards
193
+ puts " Limit of first run: #{@size_of_last_run}"
194
+ else
195
+ puts " Size of Last Run: #{@size_of_last_run}"
196
+ end
197
+ end
198
+ puts " Limit of all #{@extra_run ? "other" : ""} runs: #{@batch_size}" #This is the SQL Limit
199
+ end
200
+ end
201
+
202
+ def is_first_run?
203
+ #if no batches have been completed then we are in a first run situation
204
+ self.completion_times.empty?
205
+ end
206
+
207
+ def verbose?
208
+ !!verbose
209
+ end
210
+
211
+ def run(&block)
212
+ return false unless block_given?
213
+ self.start_time = Time.current
214
+ unless self.num_runs > 0
215
+ puts "There are no batches to run" if verbose?
216
+ return false
217
+ end
218
+ self.total_time = 0
219
+ self.completion_times = Array.new
220
+ self.offset_array.each_with_index do |offset, current_batch|
221
+ if self.backwards && self.is_first_run?
222
+ limite = self.size_of_last_run
223
+ else
224
+ limite = self.batch_size
225
+ end
226
+ if self.first_batch > current_batch
227
+ print "[O] #{show_status(current_batch, limite)} skipped" if verbose?
228
+ self.skipped_batches << current_batch
229
+ elsif self.last_batch && self.last_batch < current_batch
230
+ print "[L] #{show_status(current_batch, limite)} skipped" if verbose?
231
+ self.skipped_batches << current_batch
232
+ else
233
+ print "[P] #{show_status(current_batch, limite)}" if verbose?
234
+
235
+ #start the timer
236
+ beg_time = Time.current
237
+
238
+ self.arel.limit(limite).offset(offset).each {|obj| yield obj}
239
+
240
+ #stop the timer
241
+ fin_time = Time.current
242
+
243
+ this_time = fin_time.to_i - beg_time.to_i
244
+ self.total_time += this_time unless extra_run && current_batch == self.num_runs
245
+ puts "[C] #{show_status(current_batch, limite)} in #{this_time} seconds" if verbose?
246
+ self.completion_times << [current_batch, {:elapsed => this_time, :begin_time => beg_time, :end_time => fin_time}]
247
+ end
248
+ end
249
+ self.num_runs -= 1 if self.extra_run
250
+ self.end_time = Time.current
251
+ self.elapsed_time = (self.end_time.to_i - self.start_time.to_i)
252
+ self.overhead_time = self.elapsed_time - self.total_time
253
+ print_results if self.show_results
254
+ puts "Process Complete" if verbose?
255
+ return true
256
+ end
257
+
258
+ def show_status(current_batch, limite)
259
+ "{#{current_batch} / #{self.last_batch} / #{limite}}"
260
+ end
261
+
262
+ # Allow caller to override verbosity when called from console
263
+ def print_results(verbose = self.verbose)
264
+ printf "Results..."
265
+ printf "Average time per complete batch was %.1f seconds\n", (self.total_time/Float(self.num_runs)) unless self.num_runs < 1
266
+ printf "Total time elapsed was %.1f seconds, about #{self.elapsed_time/60} minute(s)\n", (self.elapsed_time)
267
+ if self.backwards # When backwards might be deleting records
268
+ puts "Total # of #{self.arel.table} - Before: #{self.total_records}"
269
+ puts "Total # of #{self.arel.table} - After : #{self.arel.count}"
270
+ end
271
+ # With a large number of batches this is far too verbose, but don't want to introduce a more complicated verbosity setting.
272
+ # if verbose?
273
+ # puts "Completion times for each batch:"
274
+ # self.completion_times.each do |x|
275
+ # puts "Batch #{x[0]}: Time Elapsed: #{x[1][:elapsed]}s, Begin: #{x[1][:begin_time].strftime("%m.%d.%Y %I:%M:%S %p")}, End: #{x[1][:end_time].strftime("%m.%d.%Y %I:%M:%S %p")}"
276
+ # end
277
+ # end
278
+ end
279
+
280
+ end
281
+ end
@@ -0,0 +1,3 @@
1
+ module EachInBatches
2
+ VERSION = "0.1.4"
3
+ end
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: each_in_batches-RobertAudi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.4
5
+ platform: ruby
6
+ authors:
7
+ - Peter Boling
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-05-31 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activerecord
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '3.2'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '5.0'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '3.2'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '5.0'
33
+ - !ruby/object:Gem::Dependency
34
+ name: bundler
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :development
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: rake
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '10.0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '10.0'
61
+ - !ruby/object:Gem::Dependency
62
+ name: rspec
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '3.2'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '3.2'
75
+ description: Batch Processing of Records with Blocks in Rails
76
+ email:
77
+ - peter.boling@gmail.com
78
+ executables: []
79
+ extensions: []
80
+ extra_rdoc_files: []
81
+ files:
82
+ - ".gitignore"
83
+ - ".rspec"
84
+ - ".travis.yml"
85
+ - CODE_OF_CONDUCT.md
86
+ - Gemfile
87
+ - MIT-LICENSE
88
+ - README.md
89
+ - Rakefile
90
+ - bin/console
91
+ - bin/setup
92
+ - each_in_batches.gemspec
93
+ - lib/each_in_batches.rb
94
+ - lib/each_in_batches/version.rb
95
+ homepage: https://github.com/pboling/each_in_batches
96
+ licenses: []
97
+ metadata: {}
98
+ post_install_message:
99
+ rdoc_options: []
100
+ require_paths:
101
+ - lib
102
+ required_ruby_version: !ruby/object:Gem::Requirement
103
+ requirements:
104
+ - - ">="
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ required_rubygems_version: !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - ">="
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ requirements: []
113
+ rubyforge_project:
114
+ rubygems_version: 2.4.5.1
115
+ signing_key:
116
+ specification_version: 4
117
+ summary: Batch Processing of Records with Blocks in Rails
118
+ test_files: []
119
+ has_rdoc: