occams-record 1.0.0.rc9 → 1.0.0.rc10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/occams-record/batches.rb +35 -29
- data/lib/occams-record/raw_query.rb +22 -11
- data/lib/occams-record/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c0cdafe04c023de8a937ebef5fb1c274459e11b10c06f7fec38cf5632b93c969
|
4
|
+
data.tar.gz: f98a82b06655ddaf71bcc68f171b3aa077b08f64bdfca9f35725e8fe2ad0c4f3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dfe0c1a6f0c18be9e37c17b5928a4911e62615cc95e5399b3d181b75d769bdf803c13823773d285daab4602f526c397d15f3d0f9dfa62fa9970539e07028a9e7
|
7
|
+
data.tar.gz: e606aee2bae1577a3d77649979efa76c32b30b40614d0bbd4acb5f19ea1f453c16175e466a9afbad6997112c1cb12a64b6b890dd929cafbee039d2e1cbdb665d
|
data/README.md
CHANGED
@@ -55,9 +55,9 @@ orders = OccamsRecord.
|
|
55
55
|
run
|
56
56
|
````
|
57
57
|
|
58
|
-
`each`, `map`, `reduce`, and other Enumerable methods may be used instead of *run*. `find_each` and `find_in_batches` are also supported
|
58
|
+
`each`, `map`, `reduce`, and other Enumerable methods may be used instead of *run*. `find_each` and `find_in_batches` are also supported, and unlike in ActiveRecord, `ORDER BY` works as you'd expect.
|
59
59
|
|
60
|
-
|
60
|
+
Occams Record has great support for raw SQL queries too, but we'll get to those later.
|
61
61
|
|
62
62
|
## Basic eager loading
|
63
63
|
|
@@ -4,22 +4,21 @@ module OccamsRecord
|
|
4
4
|
#
|
5
5
|
module Batches
|
6
6
|
#
|
7
|
-
# Load records in batches of N and yield each record to a block if given.
|
8
|
-
#
|
7
|
+
# Load records in batches of N and yield each record to a block if given. If no block is given,
|
8
|
+
# returns an Enumerator.
|
9
9
|
#
|
10
|
-
# NOTE Unlike ActiveRecord's find_each,
|
11
|
-
# to the ORDER BY clause to help ensure consistent batches.
|
12
|
-
# batches to be "corrupted" (miss records or repeat records) if table data changes out from
|
13
|
-
# enderneath them. To prevent this, it's strongly recomended to always run find_each inside
|
10
|
+
# NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
|
11
|
+
# to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
|
14
12
|
# of a transaction.
|
15
13
|
#
|
16
14
|
# @param batch_size [Integer]
|
15
|
+
# @param use_transaction [Boolean] Ensure it runs inside of a database transaction
|
17
16
|
# @yield [OccamsRecord::Results::Row]
|
18
17
|
# @return [Enumerator] will yield each record
|
19
18
|
#
|
20
|
-
def find_each(batch_size: 1000)
|
19
|
+
def find_each(batch_size: 1000, use_transaction: true)
|
21
20
|
enum = Enumerator.new { |y|
|
22
|
-
batches(of: batch_size).each { |batch|
|
21
|
+
batches(of: batch_size, use_transaction: use_transaction).each { |batch|
|
23
22
|
batch.each { |record| y.yield record }
|
24
23
|
}
|
25
24
|
}
|
@@ -34,18 +33,17 @@ module OccamsRecord
|
|
34
33
|
# Load records in batches of N and yield each batch to a block if given.
|
35
34
|
# If no block is given, returns an Enumerator.
|
36
35
|
#
|
37
|
-
# NOTE Unlike ActiveRecord's
|
38
|
-
# to the ORDER BY clause to help ensure consistent batches.
|
39
|
-
# batches to be "corrupted" (miss records or repeat records) if table data changes out from
|
40
|
-
# enderneath them. To prevent this, it's strongly recomended to always run find_in_batches inside
|
36
|
+
# NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
|
37
|
+
# to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
|
41
38
|
# of a transaction.
|
42
39
|
#
|
43
40
|
# @param batch_size [Integer]
|
41
|
+
# @param use_transaction [Boolean] Ensure it runs inside of a database transaction
|
44
42
|
# @yield [OccamsRecord::Results::Row]
|
45
43
|
# @return [Enumerator] will yield each batch
|
46
44
|
#
|
47
|
-
def find_in_batches(batch_size: 1000)
|
48
|
-
enum = batches(of: batch_size)
|
45
|
+
def find_in_batches(batch_size: 1000, use_transaction: true)
|
46
|
+
enum = batches(of: batch_size, use_transaction: use_transaction)
|
49
47
|
if block_given?
|
50
48
|
enum.each { |batch| yield batch }
|
51
49
|
else
|
@@ -62,29 +60,37 @@ module OccamsRecord
|
|
62
60
|
# whereas this does.
|
63
61
|
#
|
64
62
|
# @param of [Integer] batch size
|
63
|
+
# @param use_transaction [Boolean] Ensure it runs inside of a database transaction
|
65
64
|
# @return [Enumerator] yields batches
|
66
65
|
#
|
67
|
-
def batches(of:)
|
68
|
-
|
69
|
-
|
66
|
+
def batches(of:, use_transaction: true)
|
67
|
+
Enumerator.new do |y|
|
68
|
+
if use_transaction and model.connection.open_transactions == 0
|
69
|
+
model.connection.transaction {
|
70
|
+
run_batches y, of
|
71
|
+
}
|
72
|
+
else
|
73
|
+
run_batches y, of
|
74
|
+
end
|
70
75
|
end
|
76
|
+
end
|
71
77
|
|
78
|
+
def run_batches(y, of)
|
72
79
|
limit = scope.limit_value
|
73
80
|
batch_size = limit && limit < of ? limit : of
|
74
|
-
Enumerator.new do |y|
|
75
|
-
offset = scope.offset_value || 0
|
76
|
-
out_of_records, count = false, 0
|
77
81
|
|
78
|
-
|
79
|
-
|
80
|
-
q = scope.order(model.primary_key.to_sym).offset(offset).limit(l)
|
81
|
-
results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
|
82
|
+
offset = scope.offset_value || 0
|
83
|
+
out_of_records, count = false, 0
|
82
84
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
85
|
+
until out_of_records
|
86
|
+
l = limit && batch_size > limit - count ? limit - count : batch_size
|
87
|
+
q = scope.order(model.primary_key.to_sym).offset(offset).limit(l)
|
88
|
+
results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
|
89
|
+
|
90
|
+
y.yield results if results.any?
|
91
|
+
count += results.size
|
92
|
+
offset += results.size
|
93
|
+
out_of_records = results.size < batch_size || (limit && count >= limit)
|
88
94
|
end
|
89
95
|
end
|
90
96
|
end
|
@@ -139,26 +139,37 @@ module OccamsRecord
|
|
139
139
|
# The bind values will be provided by OccamsRecord.
|
140
140
|
#
|
141
141
|
# @param of [Integer] batch size
|
142
|
+
# @param use_transaction [Boolean] Ensure it runs inside of a database transaction
|
142
143
|
# @return [Enumerator] yields batches
|
143
144
|
#
|
144
|
-
def batches(of:)
|
145
|
+
def batches(of:, use_transaction: true)
|
145
146
|
unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
|
146
147
|
raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
|
147
148
|
end
|
148
149
|
|
149
150
|
Enumerator.new do |y|
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
y.yield results if results.any?
|
158
|
-
break if results.size < of
|
159
|
-
offset += results.size
|
151
|
+
if use_transaction and @conn.open_transactions == 0
|
152
|
+
@conn.transaction {
|
153
|
+
run_batches y, of
|
154
|
+
}
|
155
|
+
else
|
156
|
+
run_batches y, of
|
160
157
|
end
|
161
158
|
end
|
162
159
|
end
|
160
|
+
|
161
|
+
def run_batches(y, of)
|
162
|
+
offset = 0
|
163
|
+
loop do
|
164
|
+
results = RawQuery.new(@sql, @binds.merge({
|
165
|
+
batch_limit: of,
|
166
|
+
batch_offset: offset,
|
167
|
+
}), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
|
168
|
+
|
169
|
+
y.yield results if results.any?
|
170
|
+
break if results.size < of
|
171
|
+
offset += results.size
|
172
|
+
end
|
173
|
+
end
|
163
174
|
end
|
164
175
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: occams-record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.rc10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jordan Hollinger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-01-
|
11
|
+
date: 2019-01-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|