in_batches 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +10 -10
- data/.rubocop.yml +1228 -1228
- data/.travis.yml +26 -26
- data/CHANGELOG.md +4 -0
- data/CODE_OF_CONDUCT.md +49 -49
- data/LICENSE +21 -21
- data/README.md +58 -41
- data/Rakefile +10 -10
- data/bin/console +14 -14
- data/bin/setup +8 -8
- data/gemfiles/3.2.gemfile +10 -10
- data/gemfiles/4.2.gemfile +10 -10
- data/gemfiles/5.0.gemfile +10 -10
- data/gemfiles/5.1.gemfile +10 -10
- data/gemfiles/5.2.gemfile +10 -10
- data/{atomically.gemspec → in_batches.gemspec} +43 -43
- data/lib/in_batches.rb +10 -5
- data/lib/in_batches/active_record/batches.rb +54 -54
- data/lib/in_batches/active_record/relation/batch_enumerator.rb +67 -67
- data/lib/in_batches/active_record_extensions.rb +10 -10
- data/lib/in_batches/version.rb +5 -5
- metadata +5 -5
data/gemfiles/5.2.gemfile
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
source 'https://rubygems.org'
|
2
|
-
|
3
|
-
gem 'activerecord', '~> 5.2.0'
|
4
|
-
|
5
|
-
group :test do
|
6
|
-
gem 'simplecov'
|
7
|
-
gem 'sqlite3', '~> 1.3.6'
|
8
|
-
end
|
9
|
-
|
10
|
-
gemspec path: '../'
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gem 'activerecord', '~> 5.2.0'
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem 'simplecov'
|
7
|
+
gem 'sqlite3', '~> 1.3.6'
|
8
|
+
end
|
9
|
+
|
10
|
+
gemspec path: '../'
|
@@ -1,43 +1,43 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
lib = File.expand_path('../lib', __FILE__)
|
3
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require 'in_batches/version'
|
5
|
-
|
6
|
-
Gem::Specification.new do |spec|
|
7
|
-
spec.name = 'in_batches'
|
8
|
-
spec.version = InBatches::VERSION
|
9
|
-
spec.authors = ['khiav reoy']
|
10
|
-
spec.email = ['mrtmrt15xn@yahoo.com.tw']
|
11
|
-
|
12
|
-
spec.summary = 'Backport in_batches from Rails 5 for Rails 3 and 4.'
|
13
|
-
spec.description = 'Backport in_batches from Rails 5 for Rails 3 and 4.'
|
14
|
-
spec.homepage = 'https://github.com/khiav223577/in_batches'
|
15
|
-
spec.license = 'MIT'
|
16
|
-
|
17
|
-
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
18
|
-
# delete this section to allow pushing this gem to any host.
|
19
|
-
# if spec.respond_to?(:metadata)
|
20
|
-
# spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
|
21
|
-
# else
|
22
|
-
# raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
-
# end
|
24
|
-
|
25
|
-
spec.files = `git ls-files -z`.split("\x0").reject{|f| f.match(%r{^(test|spec|features)/}) }
|
26
|
-
spec.bindir = 'exe'
|
27
|
-
spec.executables = spec.files.grep(%r{^exe/}){|f| File.basename(f) }
|
28
|
-
spec.require_paths = ['lib']
|
29
|
-
spec.metadata = {
|
30
|
-
'homepage_uri' => 'https://github.com/khiav223577/in_batches',
|
31
|
-
'changelog_uri' => 'https://github.com/khiav223577/in_batches/blob/master/CHANGELOG.md',
|
32
|
-
'source_code_uri' => 'https://github.com/khiav223577/in_batches',
|
33
|
-
'documentation_uri' => 'http://www.rubydoc.info/gems/in_batches',
|
34
|
-
'bug_tracker_uri' => 'https://github.com/khiav223577/in_batches/issues',
|
35
|
-
}
|
36
|
-
|
37
|
-
spec.add_development_dependency 'bundler', '>= 1.17', '< 3.x'
|
38
|
-
spec.add_development_dependency 'rake', '~> 12.0'
|
39
|
-
spec.add_development_dependency 'sqlite3', '~> 1.3'
|
40
|
-
spec.add_development_dependency 'minitest', '~> 5.0'
|
41
|
-
|
42
|
-
spec.add_dependency 'activerecord', '>= 3'
|
43
|
-
end
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'in_batches/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'in_batches'
|
8
|
+
spec.version = InBatches::VERSION
|
9
|
+
spec.authors = ['khiav reoy']
|
10
|
+
spec.email = ['mrtmrt15xn@yahoo.com.tw']
|
11
|
+
|
12
|
+
spec.summary = 'Backport in_batches from Rails 5 for Rails 3 and 4.'
|
13
|
+
spec.description = 'Backport in_batches from Rails 5 for Rails 3 and 4.'
|
14
|
+
spec.homepage = 'https://github.com/khiav223577/in_batches'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
18
|
+
# delete this section to allow pushing this gem to any host.
|
19
|
+
# if spec.respond_to?(:metadata)
|
20
|
+
# spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
|
21
|
+
# else
|
22
|
+
# raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
+
# end
|
24
|
+
|
25
|
+
spec.files = `git ls-files -z`.split("\x0").reject{|f| f.match(%r{^(test|spec|features)/}) }
|
26
|
+
spec.bindir = 'exe'
|
27
|
+
spec.executables = spec.files.grep(%r{^exe/}){|f| File.basename(f) }
|
28
|
+
spec.require_paths = ['lib']
|
29
|
+
spec.metadata = {
|
30
|
+
'homepage_uri' => 'https://github.com/khiav223577/in_batches',
|
31
|
+
'changelog_uri' => 'https://github.com/khiav223577/in_batches/blob/master/CHANGELOG.md',
|
32
|
+
'source_code_uri' => 'https://github.com/khiav223577/in_batches',
|
33
|
+
'documentation_uri' => 'http://www.rubydoc.info/gems/in_batches',
|
34
|
+
'bug_tracker_uri' => 'https://github.com/khiav223577/in_batches/issues',
|
35
|
+
}
|
36
|
+
|
37
|
+
spec.add_development_dependency 'bundler', '>= 1.17', '< 3.x'
|
38
|
+
spec.add_development_dependency 'rake', '~> 12.0'
|
39
|
+
spec.add_development_dependency 'sqlite3', '~> 1.3'
|
40
|
+
spec.add_development_dependency 'minitest', '~> 5.0'
|
41
|
+
|
42
|
+
spec.add_dependency 'activerecord', '>= 3'
|
43
|
+
end
|
data/lib/in_batches.rb
CHANGED
@@ -1,5 +1,10 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'in_batches/version'
|
4
|
-
require 'active_record'
|
5
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'in_batches/version'
|
4
|
+
require 'active_record'
|
5
|
+
|
6
|
+
if defined?(ActiveRecord::Batches::BatchEnumerator)
|
7
|
+
warn "Congratulations on upgrading Rails to 5 or above. You could remove `in_batches` gem now. :)"
|
8
|
+
else
|
9
|
+
require 'in_batches/active_record_extensions'
|
10
|
+
end
|
@@ -1,54 +1,54 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module ActiveRecord
|
4
|
-
module Batches
|
5
|
-
def in_batches(of: 1000, begin_at: nil, end_at: nil, load: false)
|
6
|
-
relation = self
|
7
|
-
unless block_given?
|
8
|
-
return BatchEnumerator.new(of: of, begin_at: begin_at, end_at: end_at, relation: self)
|
9
|
-
end
|
10
|
-
|
11
|
-
if logger && (arel.orders.present? || arel.taken.present?)
|
12
|
-
logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
|
13
|
-
end
|
14
|
-
|
15
|
-
relation = relation.reorder(batch_order).limit(of)
|
16
|
-
relation = apply_limits(relation, begin_at, end_at)
|
17
|
-
batch_relation = relation
|
18
|
-
|
19
|
-
loop do
|
20
|
-
if load
|
21
|
-
records = batch_relation.to_a
|
22
|
-
ids = batch_relation.pluck(primary_key)
|
23
|
-
relation_yielded = self.where(primary_key => ids).reorder(batch_order)
|
24
|
-
relation_yielded.load_records(records)
|
25
|
-
else
|
26
|
-
ids = batch_relation.pluck(primary_key)
|
27
|
-
relation_yielded = self.where(primary_key => ids).reorder(batch_order)
|
28
|
-
end
|
29
|
-
|
30
|
-
break if ids.empty?
|
31
|
-
|
32
|
-
primary_key_offset = ids.last
|
33
|
-
raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
|
34
|
-
|
35
|
-
yield relation_yielded
|
36
|
-
|
37
|
-
break if ids.length < of
|
38
|
-
batch_relation = relation.where(table[primary_key].gt(primary_key_offset))
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def apply_limits(relation, begin_at, end_at)
|
45
|
-
relation = relation.where(table[primary_key].gteq(begin_at)) if begin_at
|
46
|
-
relation = relation.where(table[primary_key].lteq(end_at)) if end_at
|
47
|
-
relation
|
48
|
-
end
|
49
|
-
|
50
|
-
def batch_order
|
51
|
-
"#{quoted_table_name}.#{quoted_primary_key} ASC"
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module Batches
|
5
|
+
def in_batches(of: 1000, begin_at: nil, end_at: nil, load: false)
|
6
|
+
relation = self
|
7
|
+
unless block_given?
|
8
|
+
return BatchEnumerator.new(of: of, begin_at: begin_at, end_at: end_at, relation: self)
|
9
|
+
end
|
10
|
+
|
11
|
+
if logger && (arel.orders.present? || arel.taken.present?)
|
12
|
+
logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
|
13
|
+
end
|
14
|
+
|
15
|
+
relation = relation.reorder(batch_order).limit(of)
|
16
|
+
relation = apply_limits(relation, begin_at, end_at)
|
17
|
+
batch_relation = relation
|
18
|
+
|
19
|
+
loop do
|
20
|
+
if load
|
21
|
+
records = batch_relation.to_a
|
22
|
+
ids = batch_relation.pluck(primary_key)
|
23
|
+
relation_yielded = self.where(primary_key => ids).reorder(batch_order)
|
24
|
+
relation_yielded.load_records(records)
|
25
|
+
else
|
26
|
+
ids = batch_relation.pluck(primary_key)
|
27
|
+
relation_yielded = self.where(primary_key => ids).reorder(batch_order)
|
28
|
+
end
|
29
|
+
|
30
|
+
break if ids.empty?
|
31
|
+
|
32
|
+
primary_key_offset = ids.last
|
33
|
+
raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
|
34
|
+
|
35
|
+
yield relation_yielded
|
36
|
+
|
37
|
+
break if ids.length < of
|
38
|
+
batch_relation = relation.where(table[primary_key].gt(primary_key_offset))
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def apply_limits(relation, begin_at, end_at)
|
45
|
+
relation = relation.where(table[primary_key].gteq(begin_at)) if begin_at
|
46
|
+
relation = relation.where(table[primary_key].lteq(end_at)) if end_at
|
47
|
+
relation
|
48
|
+
end
|
49
|
+
|
50
|
+
def batch_order
|
51
|
+
"#{quoted_table_name}.#{quoted_primary_key} ASC"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -1,67 +1,67 @@
|
|
1
|
-
module ActiveRecord
|
2
|
-
module Batches
|
3
|
-
class BatchEnumerator
|
4
|
-
include Enumerable
|
5
|
-
|
6
|
-
def initialize(of: 1000, begin_at: nil, end_at: nil, relation:) #:nodoc:
|
7
|
-
@of = of
|
8
|
-
@relation = relation
|
9
|
-
@begin_at = begin_at
|
10
|
-
@end_at = end_at
|
11
|
-
end
|
12
|
-
|
13
|
-
# Looping through a collection of records from the database (using the
|
14
|
-
# +all+ method, for example) is very inefficient since it will try to
|
15
|
-
# instantiate all the objects at once.
|
16
|
-
#
|
17
|
-
# In that case, batch processing methods allow you to work with the
|
18
|
-
# records in batches, thereby greatly reducing memory consumption.
|
19
|
-
#
|
20
|
-
# Person.in_batches.each_record do |person|
|
21
|
-
# person.do_awesome_stuff
|
22
|
-
# end
|
23
|
-
#
|
24
|
-
# Person.where("age > 21").in_batches(of: 10).each_record do |person|
|
25
|
-
# person.party_all_night!
|
26
|
-
# end
|
27
|
-
#
|
28
|
-
# If you do not provide a block to #each_record, it will return an Enumerator
|
29
|
-
# for chaining with other methods:
|
30
|
-
#
|
31
|
-
# Person.in_batches.each_record.with_index do |person, index|
|
32
|
-
# person.award_trophy(index + 1)
|
33
|
-
# end
|
34
|
-
def each_record
|
35
|
-
return to_enum(:each_record) unless block_given?
|
36
|
-
|
37
|
-
@relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: true).each do |relation|
|
38
|
-
relation.to_a.each { |record| yield record }
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# Delegates #delete_all, #update_all, #destroy_all methods to each batch.
|
43
|
-
#
|
44
|
-
# People.in_batches.delete_all
|
45
|
-
# People.in_batches.destroy_all('age < 10')
|
46
|
-
# People.in_batches.update_all('age = age + 1')
|
47
|
-
[:delete_all, :update_all, :destroy_all].each do |method|
|
48
|
-
define_method(method) do |*args, &block|
|
49
|
-
@relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false).each do |relation|
|
50
|
-
relation.send(method, *args, &block)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
# Yields an ActiveRecord::Relation object for each batch of records.
|
56
|
-
#
|
57
|
-
# Person.in_batches.each do |relation|
|
58
|
-
# relation.update_all(awesome: true)
|
59
|
-
# end
|
60
|
-
def each
|
61
|
-
enum = @relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false)
|
62
|
-
return enum.each { |relation| yield relation } if block_given?
|
63
|
-
enum
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
1
|
+
module ActiveRecord
|
2
|
+
module Batches
|
3
|
+
class BatchEnumerator
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
def initialize(of: 1000, begin_at: nil, end_at: nil, relation:) #:nodoc:
|
7
|
+
@of = of
|
8
|
+
@relation = relation
|
9
|
+
@begin_at = begin_at
|
10
|
+
@end_at = end_at
|
11
|
+
end
|
12
|
+
|
13
|
+
# Looping through a collection of records from the database (using the
|
14
|
+
# +all+ method, for example) is very inefficient since it will try to
|
15
|
+
# instantiate all the objects at once.
|
16
|
+
#
|
17
|
+
# In that case, batch processing methods allow you to work with the
|
18
|
+
# records in batches, thereby greatly reducing memory consumption.
|
19
|
+
#
|
20
|
+
# Person.in_batches.each_record do |person|
|
21
|
+
# person.do_awesome_stuff
|
22
|
+
# end
|
23
|
+
#
|
24
|
+
# Person.where("age > 21").in_batches(of: 10).each_record do |person|
|
25
|
+
# person.party_all_night!
|
26
|
+
# end
|
27
|
+
#
|
28
|
+
# If you do not provide a block to #each_record, it will return an Enumerator
|
29
|
+
# for chaining with other methods:
|
30
|
+
#
|
31
|
+
# Person.in_batches.each_record.with_index do |person, index|
|
32
|
+
# person.award_trophy(index + 1)
|
33
|
+
# end
|
34
|
+
def each_record
|
35
|
+
return to_enum(:each_record) unless block_given?
|
36
|
+
|
37
|
+
@relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: true).each do |relation|
|
38
|
+
relation.to_a.each { |record| yield record }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
# Delegates #delete_all, #update_all, #destroy_all methods to each batch.
|
43
|
+
#
|
44
|
+
# People.in_batches.delete_all
|
45
|
+
# People.in_batches.destroy_all('age < 10')
|
46
|
+
# People.in_batches.update_all('age = age + 1')
|
47
|
+
[:delete_all, :update_all, :destroy_all].each do |method|
|
48
|
+
define_method(method) do |*args, &block|
|
49
|
+
@relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false).each do |relation|
|
50
|
+
relation.send(method, *args, &block)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Yields an ActiveRecord::Relation object for each batch of records.
|
56
|
+
#
|
57
|
+
# Person.in_batches.each do |relation|
|
58
|
+
# relation.update_all(awesome: true)
|
59
|
+
# end
|
60
|
+
def each
|
61
|
+
enum = @relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false)
|
62
|
+
return enum.each { |relation| yield relation } if block_given?
|
63
|
+
enum
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -1,10 +1,10 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'in_batches/active_record/relation/batch_enumerator'
|
4
|
-
require 'in_batches/active_record/batches'
|
5
|
-
|
6
|
-
class ActiveRecord::Base
|
7
|
-
def self.in_batches(*args)
|
8
|
-
where('').in_batches(*args)
|
9
|
-
end
|
10
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'in_batches/active_record/relation/batch_enumerator'
|
4
|
+
require 'in_batches/active_record/batches'
|
5
|
+
|
6
|
+
class ActiveRecord::Base
|
7
|
+
def self.in_batches(*args)
|
8
|
+
where('').in_batches(*args)
|
9
|
+
end
|
10
|
+
end
|
data/lib/in_batches/version.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module InBatches
|
4
|
-
VERSION = '1.0.
|
5
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module InBatches
|
4
|
+
VERSION = '1.0.1'
|
5
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: in_batches
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- khiav reoy
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-08-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -96,11 +96,11 @@ files:
|
|
96
96
|
- ".gitignore"
|
97
97
|
- ".rubocop.yml"
|
98
98
|
- ".travis.yml"
|
99
|
+
- CHANGELOG.md
|
99
100
|
- CODE_OF_CONDUCT.md
|
100
101
|
- LICENSE
|
101
102
|
- README.md
|
102
103
|
- Rakefile
|
103
|
-
- atomically.gemspec
|
104
104
|
- bin/console
|
105
105
|
- bin/setup
|
106
106
|
- gemfiles/3.2.gemfile
|
@@ -108,6 +108,7 @@ files:
|
|
108
108
|
- gemfiles/5.0.gemfile
|
109
109
|
- gemfiles/5.1.gemfile
|
110
110
|
- gemfiles/5.2.gemfile
|
111
|
+
- in_batches.gemspec
|
111
112
|
- lib/in_batches.rb
|
112
113
|
- lib/in_batches/active_record/batches.rb
|
113
114
|
- lib/in_batches/active_record/relation/batch_enumerator.rb
|
@@ -137,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
138
|
- !ruby/object:Gem::Version
|
138
139
|
version: '0'
|
139
140
|
requirements: []
|
140
|
-
|
141
|
-
rubygems_version: 2.6.14
|
141
|
+
rubygems_version: 3.0.3
|
142
142
|
signing_key:
|
143
143
|
specification_version: 4
|
144
144
|
summary: Backport in_batches from Rails 5 for Rails 3 and 4.
|