zdm 1.0.5 → 1.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/gemfiles/4.1.gemfile.lock +1 -1
- data/gemfiles/4.2.gemfile.lock +1 -1
- data/gemfiles/5.0.gemfile.lock +1 -1
- data/lib/version.rb +1 -1
- data/lib/zdm.rb +8 -2
- data/spec/zdm_spec.rb +32 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 30ce840e815a4a41f68dbf90830f69f253b5c75f
|
4
|
+
data.tar.gz: 453c0299bbfe3bfe72850c1f644bcd7460ed1d79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a1fb2a19947baf93d0be201af690a51e22bd5ef05d9766b310ffb194e6ba7f4c6bfed360733b7d3851a77c94cf9ac9632a6cf8a5f5846cec52b00b4b6c4ca43f
|
7
|
+
data.tar.gz: 583007414de8ff16f071760707ec701276aa6fab9c8efcd17e2efe7d6de15c7358b90ee6195f2ed7bf113f4eee369c45fd9de96a81e0760703a2e5c5c2a3929e
|
data/gemfiles/4.1.gemfile.lock
CHANGED
data/gemfiles/4.2.gemfile.lock
CHANGED
data/gemfiles/5.0.gemfile.lock
CHANGED
data/lib/version.rb
CHANGED
data/lib/zdm.rb
CHANGED
@@ -34,6 +34,13 @@ module Zdm
|
|
34
34
|
MIN_BATCH_SIZE = 10_000
|
35
35
|
PROGRESS_EVERY = 30 # seconds
|
36
36
|
def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
|
37
|
+
find_in_batches(table_name, start: start, finish: finish, batch_size: batch_size, progress_every: progress_every) do |batch_start, batch_end|
|
38
|
+
sql = yield batch_start, batch_end
|
39
|
+
execute(sql) if sql
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
|
37
44
|
min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
|
38
45
|
return unless min
|
39
46
|
|
@@ -48,8 +55,7 @@ module Zdm
|
|
48
55
|
batch_end = [batch_start + batch_size - 1, max].min
|
49
56
|
start_batch_time = Time.now
|
50
57
|
|
51
|
-
|
52
|
-
execute(sql) if sql
|
58
|
+
yield batch_start, batch_end
|
53
59
|
|
54
60
|
if $exit
|
55
61
|
write(table_name, 'Received SIGTERM, exiting...')
|
data/spec/zdm_spec.rb
CHANGED
@@ -93,6 +93,31 @@ describe Zdm do
|
|
93
93
|
expect(archive_tables.length).to eq(2)
|
94
94
|
end
|
95
95
|
|
96
|
+
context 'find_in_batches' do
|
97
|
+
before(:example) do
|
98
|
+
@conn = ActiveRecord::Base.connection
|
99
|
+
(1..20).each do |idx|
|
100
|
+
@conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'person-#{idx}','P#{idx}','2017-03-01 23:59:59')])
|
101
|
+
end
|
102
|
+
Zdm.io = StringIO.new
|
103
|
+
end
|
104
|
+
|
105
|
+
after(:example) do
|
106
|
+
@conn.execute(%[DELETE FROM people WHERE name LIKE 'person%'])
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'find all in batches' do
|
110
|
+
found = []
|
111
|
+
Zdm.find_in_batches('people', batch_size: 4, progress_every: 1) do |batch_start, batch_end|
|
112
|
+
sleep(0.8)
|
113
|
+
found << [batch_start, batch_end]
|
114
|
+
end
|
115
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
|
116
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
117
|
+
expect(found).to eq([[1, 4], [5, 8], [9, 12], [13, 16], [17, 20], [21, 22]])
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
96
121
|
context 'execute_in_batches' do
|
97
122
|
before(:example) do
|
98
123
|
@conn = ActiveRecord::Base.connection
|
@@ -112,7 +137,8 @@ describe Zdm do
|
|
112
137
|
sleep(0.8)
|
113
138
|
@sql % [batch_start, batch_end]
|
114
139
|
end
|
115
|
-
expect(Zdm.io.string).to
|
140
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
|
141
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
116
142
|
expect(@conn.select_value(%[SELECT COUNT(*) FROM people WHERE code LIKE '%U'])).to eq(22)
|
117
143
|
end
|
118
144
|
|
@@ -120,10 +146,12 @@ describe Zdm do
|
|
120
146
|
batches = []
|
121
147
|
Zdm.execute_in_batches('people', start: 5, finish: 18, batch_size: 4, progress_every: 1) do |batch_start, batch_end|
|
122
148
|
sleep(0.6)
|
123
|
-
|
124
|
-
|
149
|
+
sql = @sql % [batch_start, batch_end]
|
150
|
+
batches << sql
|
151
|
+
sql
|
125
152
|
end
|
126
|
-
expect(Zdm.io.string).to
|
153
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/14\)/)
|
154
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
127
155
|
expect(batches).to eq([
|
128
156
|
%[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 5 AND 8],
|
129
157
|
%[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 9 AND 12],
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zdm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ITRP Institute, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|