zdm 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gemfiles/4.1.gemfile.lock +1 -1
- data/gemfiles/4.2.gemfile.lock +1 -1
- data/gemfiles/5.0.gemfile.lock +1 -1
- data/lib/version.rb +1 -1
- data/lib/zdm.rb +8 -2
- data/spec/zdm_spec.rb +32 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 30ce840e815a4a41f68dbf90830f69f253b5c75f
|
4
|
+
data.tar.gz: 453c0299bbfe3bfe72850c1f644bcd7460ed1d79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a1fb2a19947baf93d0be201af690a51e22bd5ef05d9766b310ffb194e6ba7f4c6bfed360733b7d3851a77c94cf9ac9632a6cf8a5f5846cec52b00b4b6c4ca43f
|
7
|
+
data.tar.gz: 583007414de8ff16f071760707ec701276aa6fab9c8efcd17e2efe7d6de15c7358b90ee6195f2ed7bf113f4eee369c45fd9de96a81e0760703a2e5c5c2a3929e
|
data/gemfiles/4.1.gemfile.lock
CHANGED
data/gemfiles/4.2.gemfile.lock
CHANGED
data/gemfiles/5.0.gemfile.lock
CHANGED
data/lib/version.rb
CHANGED
data/lib/zdm.rb
CHANGED
@@ -34,6 +34,13 @@ module Zdm
|
|
34
34
|
MIN_BATCH_SIZE = 10_000
|
35
35
|
PROGRESS_EVERY = 30 # seconds
|
36
36
|
def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
|
37
|
+
find_in_batches(table_name, start: start, finish: finish, batch_size: batch_size, progress_every: progress_every) do |batch_start, batch_end|
|
38
|
+
sql = yield batch_start, batch_end
|
39
|
+
execute(sql) if sql
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
|
37
44
|
min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
|
38
45
|
return unless min
|
39
46
|
|
@@ -48,8 +55,7 @@ module Zdm
|
|
48
55
|
batch_end = [batch_start + batch_size - 1, max].min
|
49
56
|
start_batch_time = Time.now
|
50
57
|
|
51
|
-
|
52
|
-
execute(sql) if sql
|
58
|
+
yield batch_start, batch_end
|
53
59
|
|
54
60
|
if $exit
|
55
61
|
write(table_name, 'Received SIGTERM, exiting...')
|
data/spec/zdm_spec.rb
CHANGED
@@ -93,6 +93,31 @@ describe Zdm do
|
|
93
93
|
expect(archive_tables.length).to eq(2)
|
94
94
|
end
|
95
95
|
|
96
|
+
context 'find_in_batches' do
|
97
|
+
before(:example) do
|
98
|
+
@conn = ActiveRecord::Base.connection
|
99
|
+
(1..20).each do |idx|
|
100
|
+
@conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'person-#{idx}','P#{idx}','2017-03-01 23:59:59')])
|
101
|
+
end
|
102
|
+
Zdm.io = StringIO.new
|
103
|
+
end
|
104
|
+
|
105
|
+
after(:example) do
|
106
|
+
@conn.execute(%[DELETE FROM people WHERE name LIKE 'person%'])
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'find all in batches' do
|
110
|
+
found = []
|
111
|
+
Zdm.find_in_batches('people', batch_size: 4, progress_every: 1) do |batch_start, batch_end|
|
112
|
+
sleep(0.8)
|
113
|
+
found << [batch_start, batch_end]
|
114
|
+
end
|
115
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
|
116
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
117
|
+
expect(found).to eq([[1, 4], [5, 8], [9, 12], [13, 16], [17, 20], [21, 22]])
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
96
121
|
context 'execute_in_batches' do
|
97
122
|
before(:example) do
|
98
123
|
@conn = ActiveRecord::Base.connection
|
@@ -112,7 +137,8 @@ describe Zdm do
|
|
112
137
|
sleep(0.8)
|
113
138
|
@sql % [batch_start, batch_end]
|
114
139
|
end
|
115
|
-
expect(Zdm.io.string).to
|
140
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
|
141
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
116
142
|
expect(@conn.select_value(%[SELECT COUNT(*) FROM people WHERE code LIKE '%U'])).to eq(22)
|
117
143
|
end
|
118
144
|
|
@@ -120,10 +146,12 @@ describe Zdm do
|
|
120
146
|
batches = []
|
121
147
|
Zdm.execute_in_batches('people', start: 5, finish: 18, batch_size: 4, progress_every: 1) do |batch_start, batch_end|
|
122
148
|
sleep(0.6)
|
123
|
-
|
124
|
-
|
149
|
+
sql = @sql % [batch_start, batch_end]
|
150
|
+
batches << sql
|
151
|
+
sql
|
125
152
|
end
|
126
|
-
expect(Zdm.io.string).to
|
153
|
+
expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/14\)/)
|
154
|
+
expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
|
127
155
|
expect(batches).to eq([
|
128
156
|
%[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 5 AND 8],
|
129
157
|
%[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 9 AND 12],
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zdm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ITRP Institute, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|