each_in_batches 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +11 -1
- data/README.md +29 -2
- data/each_in_batches.gemspec +2 -2
- data/lib/each_in_batches.rb +25 -17
- data/lib/each_in_batches/version.rb +1 -1
- metadata +15 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 201302a3e132a0ffb2d4093dfe0a031d4b2873d4
|
4
|
+
data.tar.gz: 3dfacc828a9223a375f8e6e83fcdf3eecf17a7a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0da5e0e6b810e34122bd535fde4ddbfdbb86956591d6705f04c33de20a4c39d8bdd00027a9750c7280c04e99d44b9c6d66f650cb6e0f384a39d3493561b3fd7d
|
7
|
+
data.tar.gz: 359791e53ec39f1b1b50e7c8d2f580e8bfa2c031ba2da672aca90ab46953f86c811ba069161b4656da6bff027cbd6b0c0d7d07fc4f2c830ae077f28cf5da3541
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,33 @@
|
|
1
1
|
# EachInBatches
|
2
2
|
|
3
|
+
Better than Rails Batching. Can you batch backwards for destructive tasks? `each_in_batches` can.
|
4
|
+
|
5
|
+
| Project | EachInBatches |
|
6
|
+
|------------------------ | ----------------- |
|
7
|
+
| gem name | each_in_batches |
|
8
|
+
| license | MIT |
|
9
|
+
| expert support | [](https://www.codementor.io/peterboling?utm_source=github&utm_medium=button&utm_term=peterboling&utm_campaign=github) |
|
10
|
+
| download rank | [](https://rubygems.org/gems/each_in_batches) |
|
11
|
+
| version | [](http://badge.fury.io/rb/each_in_batches) |
|
12
|
+
| dependencies | [](https://gemnasium.com/pboling/each_in_batches) |
|
13
|
+
| code quality | [](https://codeclimate.com/github/pboling/each_in_batches) |
|
14
|
+
| inline documenation | [](http://inch-ci.org/github/pboling/each_in_batches) |
|
15
|
+
| continuous integration | [](https://travis-ci.org/pboling/each_in_batches) |
|
16
|
+
| test coverage | [](https://coveralls.io/r/pboling/each_in_batches) |
|
17
|
+
| homepage | [on Github.com][homepage] |
|
18
|
+
| documentation | [on Rdoc.info][documentation] |
|
19
|
+
| live chat | [](https://gitter.im/pboling/each_in_batches?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) |
|
20
|
+
| Spread ~♡ⓛⓞⓥⓔ♡~ | [on Coderbits][coderbits], [on Coderwall][coderwall] |
|
21
|
+
|
22
|
+
[semver]: http://semver.org/
|
23
|
+
[pvc]: http://docs.rubygems.org/read/chapter/16#page74
|
24
|
+
[railsbling]: http://www.railsbling.com
|
25
|
+
[peterboling]: http://www.peterboling.com
|
26
|
+
[coderbits]: https://coderbits.com/pboling
|
27
|
+
[coderwall]: http://coderwall.com/pboling
|
28
|
+
[documentation]: http://rdoc.info/github/pboling/each_in_batches/frames
|
29
|
+
[homepage]: https://github.com/pboling/each_in_batches
|
30
|
+
|
3
31
|
(Originally BolingForBatches)
|
4
32
|
|
5
33
|
### NOTE:
|
@@ -21,8 +49,7 @@ batch loop. But I didn't let that stop me!
|
|
21
49
|
2. Metrics. I measure stuff.
|
22
50
|
3. I can batch from the top down (a.k.a backwards), making it possible to DELETE things in batches.
|
23
51
|
A. If you've never tried using the built-in rails batching for deleting millions of records... don't start now. Use this gem instead.
|
24
|
-
4.
|
25
|
-
5. Merged in the EachInBatches fork (from Brian Kidd):
|
52
|
+
4. Merged in the EachInBatches fork (from Brian Kidd):
|
26
53
|
I needed to iterate over the results and perform more actions than a single
|
27
54
|
method would provide. I didn't want to write a method in my app that performed
|
28
55
|
the needed functionality as I felt the plugin should support this directly.
|
data/each_in_batches.gemspec
CHANGED
@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
-
spec.add_dependency "activerecord", "
|
22
|
-
spec.add_development_dependency "bundler"
|
21
|
+
spec.add_dependency "activerecord", ">= 3.2", "< 5.0"
|
22
|
+
spec.add_development_dependency "bundler"
|
23
23
|
spec.add_development_dependency "rake", "~> 10.0"
|
24
24
|
spec.add_development_dependency "rspec", "~> 3.2"
|
25
25
|
end
|
data/lib/each_in_batches.rb
CHANGED
@@ -135,14 +135,14 @@ module EachInBatches
|
|
135
135
|
@last_batch = args.first[:last_batch] ? args.first[:last_batch].is_a?(Integer) ? args.first[:last_batch] : args.first[:last_batch].to_i : false
|
136
136
|
@first_batch = args.first[:first_batch] ? args.first[:first_batch].is_a?(Integer) ? args.first[:first_batch] : args.first[:first_batch].to_i : 0
|
137
137
|
@show_results = case
|
138
|
-
when args.first[:show_results].blank? &&
|
139
|
-
when args.first[:show_results].blank? &&
|
138
|
+
when args.first[:show_results].blank? && !verbose?; false
|
139
|
+
when args.first[:show_results].blank? && verbose?; true
|
140
140
|
else args.first[:show_results]
|
141
141
|
end
|
142
142
|
@total_time = 0
|
143
143
|
@skipped_batches = []
|
144
144
|
|
145
|
-
puts "Counting Records..." if
|
145
|
+
puts "Counting Records..." if verbose?
|
146
146
|
@total_records = @arel.count
|
147
147
|
@num_runs = @total_records / @batch_size
|
148
148
|
@size_of_last_run = @total_records.modulo(@batch_size)
|
@@ -154,13 +154,13 @@ module EachInBatches
|
|
154
154
|
@extra_run = false
|
155
155
|
end
|
156
156
|
|
157
|
-
puts "Records: #{@total_records}, Batches: #{@num_runs}" if
|
157
|
+
puts "Records: #{@total_records}, Batches: #{@num_runs}" if verbose?
|
158
158
|
|
159
159
|
@last_batch = @num_runs - 1 unless @num_runs == 0 || @last_batch #because batch numbers start at 0 like array indexes, but only if it was not set in *args
|
160
160
|
|
161
161
|
current_batch = 0
|
162
162
|
@offset_array = Array.new
|
163
|
-
if
|
163
|
+
if verbose?
|
164
164
|
puts "Batch Numbering Begins With 0 (ZERO) and counts up"
|
165
165
|
puts "Batch Size (SQL Limit): #{@batch_size}" #This is the SQL Limit
|
166
166
|
puts "First Batch # to run: #{@first_batch}" #This is the number of the first batch to run
|
@@ -170,18 +170,18 @@ module EachInBatches
|
|
170
170
|
end
|
171
171
|
while current_batch < @num_runs
|
172
172
|
@offset_array << (current_batch * @batch_size)
|
173
|
-
print "." if
|
173
|
+
print "." if verbose?
|
174
174
|
current_batch += 1
|
175
175
|
end
|
176
|
-
puts " #{@num_runs} Batches Created" if
|
176
|
+
puts " #{@num_runs} Batches Created" if verbose?
|
177
177
|
#in order to use batching for record deletion, the offsets need to start with largest first
|
178
178
|
if @backwards
|
179
179
|
@offset_array.reverse!
|
180
|
-
puts "Backwards Mode:" if
|
180
|
+
puts "Backwards Mode:" if verbose?
|
181
181
|
else
|
182
|
-
puts "Normal Mode:" if
|
182
|
+
puts "Normal Mode:" if verbose?
|
183
183
|
end
|
184
|
-
if
|
184
|
+
if verbose?
|
185
185
|
puts " First Offset: #{@offset_array.first}"
|
186
186
|
puts " Last Offset: #{@offset_array.last}"
|
187
187
|
# technically the last run doesn't need a limit, and we don't technically use a limit on the last run,
|
@@ -204,10 +204,17 @@ module EachInBatches
|
|
204
204
|
self.completion_times.empty?
|
205
205
|
end
|
206
206
|
|
207
|
+
def verbose?
|
208
|
+
!!verbose
|
209
|
+
end
|
210
|
+
|
207
211
|
def run(&block)
|
208
212
|
return false unless block_given?
|
209
213
|
self.start_time = Time.current
|
210
|
-
|
214
|
+
unless self.num_runs > 0
|
215
|
+
puts "There are no batches to run" if verbose?
|
216
|
+
return false
|
217
|
+
end
|
211
218
|
self.total_time = 0
|
212
219
|
self.completion_times = Array.new
|
213
220
|
self.offset_array.each_with_index do |offset, current_batch|
|
@@ -217,13 +224,13 @@ module EachInBatches
|
|
217
224
|
limite = self.batch_size
|
218
225
|
end
|
219
226
|
if self.first_batch > current_batch
|
220
|
-
print "[O] #{show_status(current_batch, limite)} skipped" if
|
227
|
+
print "[O] #{show_status(current_batch, limite)} skipped" if verbose?
|
221
228
|
self.skipped_batches << current_batch
|
222
229
|
elsif self.last_batch && self.last_batch < current_batch
|
223
|
-
print "[L] #{show_status(current_batch, limite)} skipped" if
|
230
|
+
print "[L] #{show_status(current_batch, limite)} skipped" if verbose?
|
224
231
|
self.skipped_batches << current_batch
|
225
232
|
else
|
226
|
-
print "[P] #{show_status(current_batch, limite)}" if
|
233
|
+
print "[P] #{show_status(current_batch, limite)}" if verbose?
|
227
234
|
|
228
235
|
#start the timer
|
229
236
|
beg_time = Time.current
|
@@ -235,7 +242,7 @@ module EachInBatches
|
|
235
242
|
|
236
243
|
this_time = fin_time.to_i - beg_time.to_i
|
237
244
|
self.total_time += this_time unless extra_run && current_batch == self.num_runs
|
238
|
-
puts "[C] #{show_status(current_batch, limite)} in #{this_time} seconds" if
|
245
|
+
puts "[C] #{show_status(current_batch, limite)} in #{this_time} seconds" if verbose?
|
239
246
|
self.completion_times << [current_batch, {:elapsed => this_time, :begin_time => beg_time, :end_time => fin_time}]
|
240
247
|
end
|
241
248
|
end
|
@@ -244,7 +251,8 @@ module EachInBatches
|
|
244
251
|
self.elapsed_time = (self.end_time.to_i - self.start_time.to_i)
|
245
252
|
self.overhead_time = self.elapsed_time - self.total_time
|
246
253
|
print_results if self.show_results
|
247
|
-
|
254
|
+
puts "Process Complete" if verbose?
|
255
|
+
return true
|
248
256
|
end
|
249
257
|
|
250
258
|
def show_status(current_batch, limite)
|
@@ -261,7 +269,7 @@ module EachInBatches
|
|
261
269
|
puts "Total # of #{self.arel.table} - After : #{self.arel.count}"
|
262
270
|
end
|
263
271
|
# With a large number of batches this is far too verbose, but don't want to introduce a more complicated verbosity setting.
|
264
|
-
# if verbose
|
272
|
+
# if verbose?
|
265
273
|
# puts "Completion times for each batch:"
|
266
274
|
# self.completion_times.each do |x|
|
267
275
|
# puts "Batch #{x[0]}: Time Elapsed: #{x[1][:elapsed]}s, Begin: #{x[1][:begin_time].strftime("%m.%d.%Y %I:%M:%S %p")}, End: #{x[1][:end_time].strftime("%m.%d.%Y %I:%M:%S %p")}"
|
metadata
CHANGED
@@ -1,43 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: each_in_batches
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Boling
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-06-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '3.2'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '5.0'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '3.2'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '5.0'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: bundler
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
|
-
- - "
|
37
|
+
- - ">="
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
39
|
+
version: '0'
|
34
40
|
type: :development
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
|
-
- - "
|
44
|
+
- - ">="
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
46
|
+
version: '0'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: rake
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -105,9 +111,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
111
|
version: '0'
|
106
112
|
requirements: []
|
107
113
|
rubyforge_project:
|
108
|
-
rubygems_version: 2.4.
|
114
|
+
rubygems_version: 2.4.6
|
109
115
|
signing_key:
|
110
116
|
specification_version: 4
|
111
117
|
summary: Batch Processing of Records with Blocks in Rails
|
112
118
|
test_files: []
|
113
|
-
has_rdoc:
|