ar-ondemand 1.1.3 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/Gemfile.lock +1 -1
- data/lib/ar-ondemand/delete_all_by_pk.rb +39 -35
- data/lib/ar-ondemand/for_reading.rb +49 -41
- data/lib/ar-ondemand/for_streaming.rb +26 -0
- data/lib/ar-ondemand/result.rb +6 -1
- data/lib/ar-ondemand/version.rb +1 -1
- data/lib/ar-ondemand.rb +1 -0
- data/spec/lib/delete_all_by_pk_spec.rb +26 -0
- data/spec/lib/for_reading_spec.rb +10 -2
- data/spec/lib/for_streaming_spec.rb +69 -0
- metadata +4 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
OTQ1OGY0YzBjZGU4MjkzZTUxMzEzZTg1YzU2NjBiNzc2YmU1ZGQ5Yg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MjM1YmU4MmQ3YjA5NDhkOWNmYTI5NTkwMTRmMzhlMmYwODRlMTZlZg==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZTRiODA1ZTM1ZDNiMjI2MzY3MjJhYjA3NmFjODVlZDQwZjE5M2NhZjA0NzBh
|
10
|
+
MzhmNTgwMGJkNWMzMTc2YTM1YWZhMjFjYzdmODNiMWMwYzExODE3YjVmODE3
|
11
|
+
NjlkOTQwZGExODZkMDFjODMyMWFkMjE3ODczYTI0MTY5NmJjMmU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NGQ4NmY3MzUyNmI4NTZlY2JmODZmM2VkZmMyOThkMDVmNjJjM2MwZDBlNjY3
|
14
|
+
MDdiOWI2YjFlOGY3ZTg0MDg2ZDk0MzQzNzBlYWQwMWU5NWY0NDM2MTgyZGU1
|
15
|
+
NGQ1MDcyZjY1ZjMyYTE3OTBjZTVmOGU5ZTg1MzIzZTliYjA5MDQ=
|
data/Gemfile.lock
CHANGED
@@ -2,58 +2,62 @@ require 'active_support/concern'
|
|
2
2
|
|
3
3
|
module ActiveRecord
|
4
4
|
module OnDemand
|
5
|
-
module
|
5
|
+
module DeleteAllByPk
|
6
6
|
extend ::ActiveSupport::Concern
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
8
|
+
module ClassMethods
|
9
|
+
# Use this instead of delete_all to perform a delete using the PK of the table, which prevents a complete table scan that locks it
|
10
|
+
# Based on find_in_batches function
|
11
|
+
def delete_all_by_pk(options = {})
|
12
|
+
relation = self
|
13
|
+
relation = self.scoped unless respond_to? :arel
|
12
14
|
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
unless relation.arel.orders.blank? && relation.arel.taken.blank?
|
16
|
+
::ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
|
17
|
+
end
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
19
|
+
if (finder_options = options.except(:start, :batch_size)).present?
|
20
|
+
raise "You can't specify an order, it's forced to be #{batch_order_delete_all_by_pk}" if options[:order].present?
|
21
|
+
raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit].present?
|
20
22
|
|
21
|
-
|
22
|
-
|
23
|
+
relation = apply_finder_options(finder_options)
|
24
|
+
end
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
+
start = options.delete(:start)
|
27
|
+
batch_size = options.delete(:batch_size)
|
26
28
|
|
27
|
-
|
28
|
-
|
29
|
-
|
29
|
+
relation = relation.reorder(batch_order_delete_all_by_pk).limit(batch_size) if batch_size
|
30
|
+
records = query_delete_all_by_pk(start ? relation.where(relation.table[primary_key].gteq(start)) : relation)
|
31
|
+
deleted = 0
|
30
32
|
|
31
|
-
|
32
|
-
|
33
|
-
|
33
|
+
while records.any?
|
34
|
+
records_size = records.size
|
35
|
+
primary_key_offset = records.last
|
34
36
|
|
35
|
-
|
37
|
+
deleted += self.unscoped.where(id: records).delete_all
|
36
38
|
|
37
|
-
|
39
|
+
break if batch_size.nil? || records_size < batch_size
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
+
records = query_delete_all_by_pk relation.where(relation.table[primary_key].gt(primary_key_offset))
|
42
|
+
end
|
41
43
|
|
42
|
-
|
43
|
-
|
44
|
+
deleted
|
45
|
+
end
|
44
46
|
|
45
|
-
|
47
|
+
private
|
46
48
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
49
|
+
def query_delete_all_by_pk(ar)
|
50
|
+
results = ::ActiveRecord::Base.connection.exec_query ar.select([ar.table[primary_key]]).to_sql
|
51
|
+
results.rows.flatten
|
52
|
+
end
|
51
53
|
|
52
|
-
|
53
|
-
|
54
|
+
def batch_order_delete_all_by_pk
|
55
|
+
"#{quoted_table_name}.#{quoted_primary_key} ASC"
|
56
|
+
end
|
54
57
|
end
|
55
58
|
end
|
56
59
|
end
|
57
60
|
end
|
58
61
|
|
59
|
-
::ActiveRecord::
|
62
|
+
::ActiveRecord::Base.send :include, ::ActiveRecord::OnDemand::DeleteAllByPk
|
63
|
+
::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::DeleteAllByPk::ClassMethods
|
@@ -4,67 +4,75 @@ require 'ar-ondemand/record'
|
|
4
4
|
|
5
5
|
module ActiveRecord
|
6
6
|
module OnDemand
|
7
|
-
module
|
7
|
+
module Reading
|
8
8
|
extend ::ActiveSupport::Concern
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
res = query_for_reading(self)
|
14
|
-
if block_given?
|
15
|
-
yield res
|
16
|
-
return
|
17
|
-
end
|
18
|
-
return res
|
10
|
+
module ClassMethods
|
11
|
+
def raw_results
|
12
|
+
query_for_reading self, readonly: true, raw: true
|
19
13
|
end
|
20
14
|
|
21
|
-
|
15
|
+
# Ripped from the find_in_batches function, but customized to return an ::ActiveRecord::OnDemand::ResultSet
|
16
|
+
def for_reading(options = {})
|
17
|
+
if options.empty?
|
18
|
+
res = query_for_reading self, readonly: true
|
19
|
+
if block_given?
|
20
|
+
yield res
|
21
|
+
return
|
22
|
+
end
|
23
|
+
return res
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
|
25
|
-
end
|
26
|
+
relation = self
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
28
|
+
unless arel.orders.blank? && arel.taken.blank?
|
29
|
+
::ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
|
30
|
+
end
|
30
31
|
|
31
|
-
|
32
|
-
|
32
|
+
if (finder_options = options.except(:start, :batch_size)).present?
|
33
|
+
raise "You can't specify an order, it's forced to be #{batch_order_for_reading}" if options[:order].present?
|
34
|
+
raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit].present?
|
33
35
|
|
34
|
-
|
35
|
-
|
36
|
+
relation = apply_finder_options(finder_options)
|
37
|
+
end
|
36
38
|
|
37
|
-
|
38
|
-
|
39
|
+
start = options.delete(:start)
|
40
|
+
batch_size = options.delete(:batch_size) || 1000
|
39
41
|
|
40
|
-
|
41
|
-
|
42
|
-
primary_key_offset = records.last.id
|
42
|
+
relation = relation.reorder(batch_order_for_reading).limit(batch_size)
|
43
|
+
records = query_for_reading(start ? relation.where(table[primary_key].gteq(start)) : relation, readonly: true)
|
43
44
|
|
44
|
-
|
45
|
+
while records.any?
|
46
|
+
records_size = records.size
|
47
|
+
primary_key_offset = records.last.id
|
45
48
|
|
46
|
-
|
49
|
+
yield records
|
47
50
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
51
|
+
break if records_size < batch_size
|
52
|
+
|
53
|
+
if primary_key_offset
|
54
|
+
records = query_for_reading relation.where(table[primary_key].gt(primary_key_offset)), readonly: true
|
55
|
+
else
|
56
|
+
raise 'Primary key not included in the custom select clause'
|
57
|
+
end
|
52
58
|
end
|
53
59
|
end
|
54
|
-
end
|
55
60
|
|
56
|
-
|
61
|
+
private
|
57
62
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
63
|
+
def query_for_reading(ar, options = {})
|
64
|
+
ar = ar.scoped unless ar.respond_to?(:to_sql)
|
65
|
+
results = ::ActiveRecord::Base.connection.exec_query ar.to_sql
|
66
|
+
::ActiveRecord::OnDemand::ResultSet.new ar.arel.engine, results, options
|
67
|
+
end
|
62
68
|
|
63
|
-
|
64
|
-
|
69
|
+
def batch_order_for_reading
|
70
|
+
"#{quoted_table_name}.#{quoted_primary_key} ASC"
|
71
|
+
end
|
65
72
|
end
|
66
73
|
end
|
67
74
|
end
|
68
75
|
end
|
69
76
|
|
70
|
-
::ActiveRecord::
|
77
|
+
::ActiveRecord::Base.send :include, ::ActiveRecord::OnDemand::Reading
|
78
|
+
::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::Reading::ClassMethods
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
require 'ar-ondemand/for_reading'
|
3
|
+
|
4
|
+
module ActiveRecord
|
5
|
+
module OnDemand
|
6
|
+
module Streaming
|
7
|
+
extend ::ActiveSupport::Concern
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def for_streaming(options = {})
|
11
|
+
options[:batch_size] ||= 50_000
|
12
|
+
fr = options.delete(:for_reading)
|
13
|
+
s = self.respond_to?(:scoped) ? self.scoped : self
|
14
|
+
::Enumerator.new do |n|
|
15
|
+
s.send(fr ? :for_reading : :find_in_batches, options) do |b|
|
16
|
+
b.each { |r| n << r }
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
::ActiveRecord::Base.send :include, ::ActiveRecord::OnDemand::Streaming
|
26
|
+
::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::Streaming::ClassMethods
|
data/lib/ar-ondemand/result.rb
CHANGED
@@ -3,11 +3,14 @@ module ActiveRecord
|
|
3
3
|
class ResultSet
|
4
4
|
include ::Enumerable
|
5
5
|
|
6
|
+
CACHED_READONLY_CLASSES ||= {}
|
7
|
+
|
6
8
|
def initialize(model, results, options = {})
|
7
9
|
@model = model
|
8
10
|
@results = results
|
9
11
|
@column_types = Hash[@model.columns.map { |x| [x.name, x] }]
|
10
12
|
@col_indexes = HashWithIndifferentAccess[@results.columns.each_with_index.map { |x, i| [x,i] }]
|
13
|
+
@raw = options.delete :raw
|
11
14
|
@readonly = options.delete :readonly
|
12
15
|
@readonly_klass = @readonly ? create_readonly_class : nil
|
13
16
|
end
|
@@ -39,6 +42,7 @@ module ActiveRecord
|
|
39
42
|
|
40
43
|
def result_to_record(row)
|
41
44
|
return nil if row.nil?
|
45
|
+
return row if @raw
|
42
46
|
if @readonly
|
43
47
|
convert_to_struct @readonly_klass, row
|
44
48
|
else
|
@@ -48,7 +52,8 @@ module ActiveRecord
|
|
48
52
|
|
49
53
|
def create_readonly_class
|
50
54
|
attrs = @col_indexes.keys.map(&:to_sym)
|
51
|
-
|
55
|
+
return CACHED_READONLY_CLASSES[attrs] if CACHED_READONLY_CLASSES[attrs]
|
56
|
+
CACHED_READONLY_CLASSES[attrs] = ::Struct.new *attrs
|
52
57
|
end
|
53
58
|
|
54
59
|
def convert_to_hash(rec)
|
data/lib/ar-ondemand/version.rb
CHANGED
data/lib/ar-ondemand.rb
CHANGED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ar-ondemand'
|
3
|
+
|
4
|
+
describe 'DeleteAllByPk' do
|
5
|
+
context 'Testing' do
|
6
|
+
context 'ClassMethod' do
|
7
|
+
before(:each) do
|
8
|
+
create(:audit_record)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should delete record' do
|
12
|
+
AuditRecord.delete_all_by_pk.should eq(1)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'Relation' do
|
17
|
+
before(:each) do
|
18
|
+
create(:audit_record)
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should delete record' do
|
22
|
+
AuditRecord.where('id > 0').delete_all_by_pk.should eq(1)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -19,7 +19,7 @@ describe 'ForReading' do
|
|
19
19
|
|
20
20
|
context 'Iterating' do
|
21
21
|
before(:each) do
|
22
|
-
(1..
|
22
|
+
(1..25).each do
|
23
23
|
create(:audit_record)
|
24
24
|
end
|
25
25
|
end
|
@@ -29,7 +29,15 @@ describe 'ForReading' do
|
|
29
29
|
AuditRecord.where(customer_id: 1).for_reading.each do |r|
|
30
30
|
total += 1
|
31
31
|
end
|
32
|
-
total.should eq(
|
32
|
+
total.should eq(25)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should support iterating' do
|
36
|
+
total = 0
|
37
|
+
AuditRecord.for_reading.each do |r|
|
38
|
+
total += 1
|
39
|
+
end
|
40
|
+
total.should eq(25)
|
33
41
|
end
|
34
42
|
|
35
43
|
it 'should produce same results as regular iterating' do
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ar-ondemand'
|
3
|
+
|
4
|
+
describe 'ForStreaming' do
|
5
|
+
context 'Testing' do
|
6
|
+
context 'Iterating' do
|
7
|
+
before(:each) do
|
8
|
+
(1..25).each do
|
9
|
+
create(:audit_record)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
context 'AR' do
|
14
|
+
it 'ClassMethod should support streaming' do
|
15
|
+
total = 0
|
16
|
+
AuditRecord.for_streaming.each do |r|
|
17
|
+
#r.is_a?(AuditRecord).should eq(true)
|
18
|
+
expect(r).to be_an_instance_of(AuditRecord)
|
19
|
+
total += 1
|
20
|
+
end
|
21
|
+
total.should eq(25)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'Relation should support streaming' do
|
25
|
+
total = 0
|
26
|
+
AuditRecord.where(customer_id: 1).for_streaming.each do |r|
|
27
|
+
expect(r).to be_an_instance_of(AuditRecord)
|
28
|
+
total += 1
|
29
|
+
end
|
30
|
+
total.should eq(25)
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should support select' do
|
34
|
+
total = 0
|
35
|
+
AuditRecord.select([:customer_id]).where(customer_id: 1).for_streaming.each do |r|
|
36
|
+
expect(r).to be_an_instance_of(AuditRecord)
|
37
|
+
expect(r.customer_id).to eq(1)
|
38
|
+
expect { r.model_type_id }.to raise_error(::ActiveModel::MissingAttributeError)
|
39
|
+
total += 1
|
40
|
+
end
|
41
|
+
total.should eq(25)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'For Reading' do
|
46
|
+
it 'ClassMethod should support streaming' do
|
47
|
+
total = 0
|
48
|
+
AuditRecord.for_streaming(for_reading: true).each do |r|
|
49
|
+
expect(r).not_to be_an_instance_of(AuditRecord)
|
50
|
+
r.customer_id.should eq(1)
|
51
|
+
total += 1
|
52
|
+
end
|
53
|
+
total.should eq(25)
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'Relation should support streaming' do
|
57
|
+
total = 0
|
58
|
+
AuditRecord.where(customer_id: 1).for_streaming(for_reading: true).each do |r|
|
59
|
+
expect(r).not_to be_an_instance_of(AuditRecord)
|
60
|
+
r.customer_id.should eq(1)
|
61
|
+
total += 1
|
62
|
+
end
|
63
|
+
total.should eq(25)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ar-ondemand
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steve Frank
|
@@ -55,6 +55,7 @@ files:
|
|
55
55
|
- lib/ar-ondemand.rb
|
56
56
|
- lib/ar-ondemand/delete_all_by_pk.rb
|
57
57
|
- lib/ar-ondemand/for_reading.rb
|
58
|
+
- lib/ar-ondemand/for_streaming.rb
|
58
59
|
- lib/ar-ondemand/on_demand.rb
|
59
60
|
- lib/ar-ondemand/record.rb
|
60
61
|
- lib/ar-ondemand/result.rb
|
@@ -64,7 +65,9 @@ files:
|
|
64
65
|
- spec/db/schema.rb
|
65
66
|
- spec/db/seeds.rb
|
66
67
|
- spec/factories/audit_record.rb
|
68
|
+
- spec/lib/delete_all_by_pk_spec.rb
|
67
69
|
- spec/lib/for_reading_spec.rb
|
70
|
+
- spec/lib/for_streaming_spec.rb
|
68
71
|
- spec/spec_helper.rb
|
69
72
|
- spec/support/active_record.rb
|
70
73
|
homepage: https://github.com/CloudHealth/ar-ondemand
|