each_in_batches 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +11 -1
- data/README.md +29 -2
- data/each_in_batches.gemspec +2 -2
- data/lib/each_in_batches.rb +25 -17
- data/lib/each_in_batches/version.rb +1 -1
- metadata +15 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 201302a3e132a0ffb2d4093dfe0a031d4b2873d4
|
4
|
+
data.tar.gz: 3dfacc828a9223a375f8e6e83fcdf3eecf17a7a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0da5e0e6b810e34122bd535fde4ddbfdbb86956591d6705f04c33de20a4c39d8bdd00027a9750c7280c04e99d44b9c6d66f650cb6e0f384a39d3493561b3fd7d
|
7
|
+
data.tar.gz: 359791e53ec39f1b1b50e7c8d2f580e8bfa2c031ba2da672aca90ab46953f86c811ba069161b4656da6bff027cbd6b0c0d7d07fc4f2c830ae077f28cf5da3541
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,33 @@
|
|
1
1
|
# EachInBatches
|
2
2
|
|
3
|
+
Better than Rails Batching. Can you batch backwards for destructive tasks? `each_in_batches` can.
|
4
|
+
|
5
|
+
| Project | EachInBatches |
|
6
|
+
|------------------------ | ----------------- |
|
7
|
+
| gem name | each_in_batches |
|
8
|
+
| license | MIT |
|
9
|
+
| expert support | [![Get help on Codementor](https://cdn.codementor.io/badges/get_help_github.svg)](https://www.codementor.io/peterboling?utm_source=github&utm_medium=button&utm_term=peterboling&utm_campaign=github) |
|
10
|
+
| download rank | [![Total Downloads](https://img.shields.io/gem/rt/each_in_batches.svg)](https://rubygems.org/gems/each_in_batches) |
|
11
|
+
| version | [![Gem Version](https://badge.fury.io/rb/each_in_batches.png)](http://badge.fury.io/rb/each_in_batches) |
|
12
|
+
| dependencies | [![Dependency Status](https://gemnasium.com/pboling/each_in_batches.png)](https://gemnasium.com/pboling/each_in_batches) |
|
13
|
+
| code quality | [![Code Climate](https://codeclimate.com/github/pboling/each_in_batches.png)](https://codeclimate.com/github/pboling/each_in_batches) |
|
14
|
+
| inline documenation | [![Inline docs](http://inch-ci.org/github/pboling/each_in_batches.png)](http://inch-ci.org/github/pboling/each_in_batches) |
|
15
|
+
| continuous integration | [![Build Status](https://secure.travis-ci.org/pboling/each_in_batches.png?branch=master)](https://travis-ci.org/pboling/each_in_batches) |
|
16
|
+
| test coverage | [![Coverage Status](https://coveralls.io/repos/pboling/each_in_batches/badge.png)](https://coveralls.io/r/pboling/each_in_batches) |
|
17
|
+
| homepage | [on Github.com][homepage] |
|
18
|
+
| documentation | [on Rdoc.info][documentation] |
|
19
|
+
| live chat | [![Join the chat at https://gitter.im/pboling/each_in_batches](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pboling/each_in_batches?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) |
|
20
|
+
| Spread ~♡ⓛⓞⓥⓔ♡~ | [on Coderbits][coderbits], [on Coderwall][coderwall] |
|
21
|
+
|
22
|
+
[semver]: http://semver.org/
|
23
|
+
[pvc]: http://docs.rubygems.org/read/chapter/16#page74
|
24
|
+
[railsbling]: http://www.railsbling.com
|
25
|
+
[peterboling]: http://www.peterboling.com
|
26
|
+
[coderbits]: https://coderbits.com/pboling
|
27
|
+
[coderwall]: http://coderwall.com/pboling
|
28
|
+
[documentation]: http://rdoc.info/github/pboling/each_in_batches/frames
|
29
|
+
[homepage]: https://github.com/pboling/each_in_batches
|
30
|
+
|
3
31
|
(Originally BolingForBatches)
|
4
32
|
|
5
33
|
### NOTE:
|
@@ -21,8 +49,7 @@ batch loop. But I didn't let that stop me!
|
|
21
49
|
2. Metrics. I measure stuff.
|
22
50
|
3. I can batch from the top down (a.k.a backwards), making it possible to DELETE things in batches.
|
23
51
|
A. If you've never tried using the built-in rails batching for deleting millions of records... don't start now. Use this gem instead.
|
24
|
-
4.
|
25
|
-
5. Merged in the EachInBatches fork (from Brian Kidd):
|
52
|
+
4. Merged in the EachInBatches fork (from Brian Kidd):
|
26
53
|
I needed to iterate over the results and perform more actions than a single
|
27
54
|
method would provide. I didn't want to write a method in my app that performed
|
28
55
|
the needed functionality as I felt the plugin should support this directly.
|
data/each_in_batches.gemspec
CHANGED
@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
-
spec.add_dependency "activerecord", "
|
22
|
-
spec.add_development_dependency "bundler"
|
21
|
+
spec.add_dependency "activerecord", ">= 3.2", "< 5.0"
|
22
|
+
spec.add_development_dependency "bundler"
|
23
23
|
spec.add_development_dependency "rake", "~> 10.0"
|
24
24
|
spec.add_development_dependency "rspec", "~> 3.2"
|
25
25
|
end
|
data/lib/each_in_batches.rb
CHANGED
@@ -135,14 +135,14 @@ module EachInBatches
|
|
135
135
|
@last_batch = args.first[:last_batch] ? args.first[:last_batch].is_a?(Integer) ? args.first[:last_batch] : args.first[:last_batch].to_i : false
|
136
136
|
@first_batch = args.first[:first_batch] ? args.first[:first_batch].is_a?(Integer) ? args.first[:first_batch] : args.first[:first_batch].to_i : 0
|
137
137
|
@show_results = case
|
138
|
-
when args.first[:show_results].blank? &&
|
139
|
-
when args.first[:show_results].blank? &&
|
138
|
+
when args.first[:show_results].blank? && !verbose?; false
|
139
|
+
when args.first[:show_results].blank? && verbose?; true
|
140
140
|
else args.first[:show_results]
|
141
141
|
end
|
142
142
|
@total_time = 0
|
143
143
|
@skipped_batches = []
|
144
144
|
|
145
|
-
puts "Counting Records..." if
|
145
|
+
puts "Counting Records..." if verbose?
|
146
146
|
@total_records = @arel.count
|
147
147
|
@num_runs = @total_records / @batch_size
|
148
148
|
@size_of_last_run = @total_records.modulo(@batch_size)
|
@@ -154,13 +154,13 @@ module EachInBatches
|
|
154
154
|
@extra_run = false
|
155
155
|
end
|
156
156
|
|
157
|
-
puts "Records: #{@total_records}, Batches: #{@num_runs}" if
|
157
|
+
puts "Records: #{@total_records}, Batches: #{@num_runs}" if verbose?
|
158
158
|
|
159
159
|
@last_batch = @num_runs - 1 unless @num_runs == 0 || @last_batch #because batch numbers start at 0 like array indexes, but only if it was not set in *args
|
160
160
|
|
161
161
|
current_batch = 0
|
162
162
|
@offset_array = Array.new
|
163
|
-
if
|
163
|
+
if verbose?
|
164
164
|
puts "Batch Numbering Begins With 0 (ZERO) and counts up"
|
165
165
|
puts "Batch Size (SQL Limit): #{@batch_size}" #This is the SQL Limit
|
166
166
|
puts "First Batch # to run: #{@first_batch}" #This is the number of the first batch to run
|
@@ -170,18 +170,18 @@ module EachInBatches
|
|
170
170
|
end
|
171
171
|
while current_batch < @num_runs
|
172
172
|
@offset_array << (current_batch * @batch_size)
|
173
|
-
print "." if
|
173
|
+
print "." if verbose?
|
174
174
|
current_batch += 1
|
175
175
|
end
|
176
|
-
puts " #{@num_runs} Batches Created" if
|
176
|
+
puts " #{@num_runs} Batches Created" if verbose?
|
177
177
|
#in order to use batching for record deletion, the offsets need to start with largest first
|
178
178
|
if @backwards
|
179
179
|
@offset_array.reverse!
|
180
|
-
puts "Backwards Mode:" if
|
180
|
+
puts "Backwards Mode:" if verbose?
|
181
181
|
else
|
182
|
-
puts "Normal Mode:" if
|
182
|
+
puts "Normal Mode:" if verbose?
|
183
183
|
end
|
184
|
-
if
|
184
|
+
if verbose?
|
185
185
|
puts " First Offset: #{@offset_array.first}"
|
186
186
|
puts " Last Offset: #{@offset_array.last}"
|
187
187
|
# technically the last run doesn't need a limit, and we don't technically use a limit on the last run,
|
@@ -204,10 +204,17 @@ module EachInBatches
|
|
204
204
|
self.completion_times.empty?
|
205
205
|
end
|
206
206
|
|
207
|
+
def verbose?
|
208
|
+
!!verbose
|
209
|
+
end
|
210
|
+
|
207
211
|
def run(&block)
|
208
212
|
return false unless block_given?
|
209
213
|
self.start_time = Time.current
|
210
|
-
|
214
|
+
unless self.num_runs > 0
|
215
|
+
puts "There are no batches to run" if verbose?
|
216
|
+
return false
|
217
|
+
end
|
211
218
|
self.total_time = 0
|
212
219
|
self.completion_times = Array.new
|
213
220
|
self.offset_array.each_with_index do |offset, current_batch|
|
@@ -217,13 +224,13 @@ module EachInBatches
|
|
217
224
|
limite = self.batch_size
|
218
225
|
end
|
219
226
|
if self.first_batch > current_batch
|
220
|
-
print "[O] #{show_status(current_batch, limite)} skipped" if
|
227
|
+
print "[O] #{show_status(current_batch, limite)} skipped" if verbose?
|
221
228
|
self.skipped_batches << current_batch
|
222
229
|
elsif self.last_batch && self.last_batch < current_batch
|
223
|
-
print "[L] #{show_status(current_batch, limite)} skipped" if
|
230
|
+
print "[L] #{show_status(current_batch, limite)} skipped" if verbose?
|
224
231
|
self.skipped_batches << current_batch
|
225
232
|
else
|
226
|
-
print "[P] #{show_status(current_batch, limite)}" if
|
233
|
+
print "[P] #{show_status(current_batch, limite)}" if verbose?
|
227
234
|
|
228
235
|
#start the timer
|
229
236
|
beg_time = Time.current
|
@@ -235,7 +242,7 @@ module EachInBatches
|
|
235
242
|
|
236
243
|
this_time = fin_time.to_i - beg_time.to_i
|
237
244
|
self.total_time += this_time unless extra_run && current_batch == self.num_runs
|
238
|
-
puts "[C] #{show_status(current_batch, limite)} in #{this_time} seconds" if
|
245
|
+
puts "[C] #{show_status(current_batch, limite)} in #{this_time} seconds" if verbose?
|
239
246
|
self.completion_times << [current_batch, {:elapsed => this_time, :begin_time => beg_time, :end_time => fin_time}]
|
240
247
|
end
|
241
248
|
end
|
@@ -244,7 +251,8 @@ module EachInBatches
|
|
244
251
|
self.elapsed_time = (self.end_time.to_i - self.start_time.to_i)
|
245
252
|
self.overhead_time = self.elapsed_time - self.total_time
|
246
253
|
print_results if self.show_results
|
247
|
-
|
254
|
+
puts "Process Complete" if verbose?
|
255
|
+
return true
|
248
256
|
end
|
249
257
|
|
250
258
|
def show_status(current_batch, limite)
|
@@ -261,7 +269,7 @@ module EachInBatches
|
|
261
269
|
puts "Total # of #{self.arel.table} - After : #{self.arel.count}"
|
262
270
|
end
|
263
271
|
# With a large number of batches this is far too verbose, but don't want to introduce a more complicated verbosity setting.
|
264
|
-
# if verbose
|
272
|
+
# if verbose?
|
265
273
|
# puts "Completion times for each batch:"
|
266
274
|
# self.completion_times.each do |x|
|
267
275
|
# puts "Batch #{x[0]}: Time Elapsed: #{x[1][:elapsed]}s, Begin: #{x[1][:begin_time].strftime("%m.%d.%Y %I:%M:%S %p")}, End: #{x[1][:end_time].strftime("%m.%d.%Y %I:%M:%S %p")}"
|
metadata
CHANGED
@@ -1,43 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: each_in_batches
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Boling
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-06-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '3.2'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '5.0'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '3.2'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '5.0'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: bundler
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
|
-
- - "
|
37
|
+
- - ">="
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
39
|
+
version: '0'
|
34
40
|
type: :development
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
|
-
- - "
|
44
|
+
- - ">="
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
46
|
+
version: '0'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: rake
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -105,9 +111,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
111
|
version: '0'
|
106
112
|
requirements: []
|
107
113
|
rubyforge_project:
|
108
|
-
rubygems_version: 2.4.
|
114
|
+
rubygems_version: 2.4.6
|
109
115
|
signing_key:
|
110
116
|
specification_version: 4
|
111
117
|
summary: Batch Processing of Records with Blocks in Rails
|
112
118
|
test_files: []
|
113
|
-
has_rdoc:
|