zdm 1.0.5 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3bbad8595a090ca55222e019298a51b521272442
4
- data.tar.gz: a17df5df455399660ae8b4f8ab825ca50339e0cd
3
+ metadata.gz: 30ce840e815a4a41f68dbf90830f69f253b5c75f
4
+ data.tar.gz: 453c0299bbfe3bfe72850c1f644bcd7460ed1d79
5
5
  SHA512:
6
- metadata.gz: 6290405bf6180e28c04e3aa2ed217b7a2bb4b3982a64fa82bb1f31a464a4e493e0e286c0c2f91646cb9b88972e9aa34f5098eba2d2b786b2dba40c1c1ae2f191
7
- data.tar.gz: 10cd09f9ad8a6a86266971d30d75baa7d27c6c95d5cdfbc10bb0b89dca4598af25fa48f94e00ccfd0f5ecd44e288854dae479a6104fdb2f4cd82cd0cf4ad1534
6
+ metadata.gz: a1fb2a19947baf93d0be201af690a51e22bd5ef05d9766b310ffb194e6ba7f4c6bfed360733b7d3851a77c94cf9ac9632a6cf8a5f5846cec52b00b4b6c4ca43f
7
+ data.tar.gz: 583007414de8ff16f071760707ec701276aa6fab9c8efcd17e2efe7d6de15c7358b90ee6195f2ed7bf113f4eee369c45fd9de96a81e0760703a2e5c5c2a3929e
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ..
3
3
  specs:
4
- zdm (1.0.5)
4
+ zdm (1.0.6)
5
5
  activerecord (>= 4.0)
6
6
 
7
7
  GEM
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ..
3
3
  specs:
4
- zdm (1.0.5)
4
+ zdm (1.0.6)
5
5
  activerecord (>= 4.0)
6
6
 
7
7
  GEM
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ..
3
3
  specs:
4
- zdm (1.0.5)
4
+ zdm (1.0.6)
5
5
  activerecord (>= 4.0)
6
6
 
7
7
  GEM
@@ -1,3 +1,3 @@
1
1
  module Zdm
2
- VERSION = '1.0.5'
2
+ VERSION = '1.0.6'
3
3
  end
data/lib/zdm.rb CHANGED
@@ -34,6 +34,13 @@ module Zdm
34
34
  MIN_BATCH_SIZE = 10_000
35
35
  PROGRESS_EVERY = 30 # seconds
36
36
  def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
37
+ find_in_batches(table_name, start: start, finish: finish, batch_size: batch_size, progress_every: progress_every) do |batch_start, batch_end|
38
+ sql = yield batch_start, batch_end
39
+ execute(sql) if sql
40
+ end
41
+ end
42
+
43
+ def find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
37
44
  min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
38
45
  return unless min
39
46
 
@@ -48,8 +55,7 @@ module Zdm
48
55
  batch_end = [batch_start + batch_size - 1, max].min
49
56
  start_batch_time = Time.now
50
57
 
51
- sql = yield batch_start, batch_end
52
- execute(sql) if sql
58
+ yield batch_start, batch_end
53
59
 
54
60
  if $exit
55
61
  write(table_name, 'Received SIGTERM, exiting...')
@@ -93,6 +93,31 @@ describe Zdm do
93
93
  expect(archive_tables.length).to eq(2)
94
94
  end
95
95
 
96
+ context 'find_in_batches' do
97
+ before(:example) do
98
+ @conn = ActiveRecord::Base.connection
99
+ (1..20).each do |idx|
100
+ @conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'person-#{idx}','P#{idx}','2017-03-01 23:59:59')])
101
+ end
102
+ Zdm.io = StringIO.new
103
+ end
104
+
105
+ after(:example) do
106
+ @conn.execute(%[DELETE FROM people WHERE name LIKE 'person%'])
107
+ end
108
+
109
+ it 'find all in batches' do
110
+ found = []
111
+ Zdm.find_in_batches('people', batch_size: 4, progress_every: 1) do |batch_start, batch_end|
112
+ sleep(0.8)
113
+ found << [batch_start, batch_end]
114
+ end
115
+ expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
116
+ expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
117
+ expect(found).to eq([[1, 4], [5, 8], [9, 12], [13, 16], [17, 20], [21, 22]])
118
+ end
119
+ end
120
+
96
121
  context 'execute_in_batches' do
97
122
  before(:example) do
98
123
  @conn = ActiveRecord::Base.connection
@@ -112,7 +137,8 @@ describe Zdm do
112
137
  sleep(0.8)
113
138
  @sql % [batch_start, batch_end]
114
139
  end
115
- expect(Zdm.io.string).to eq(%[people: 36.36% (8/22)\npeople: 72.73% (16/22)\npeople: Completed (4 secs)\n])
140
+ expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
141
+ expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
116
142
  expect(@conn.select_value(%[SELECT COUNT(*) FROM people WHERE code LIKE '%U'])).to eq(22)
117
143
  end
118
144
 
@@ -120,10 +146,12 @@ describe Zdm do
120
146
  batches = []
121
147
  Zdm.execute_in_batches('people', start: 5, finish: 18, batch_size: 4, progress_every: 1) do |batch_start, batch_end|
122
148
  sleep(0.6)
123
- batches << @sql % [batch_start, batch_end]
124
- @sql % [batch_start, batch_end]
149
+ sql = @sql % [batch_start, batch_end]
150
+ batches << sql
151
+ sql
125
152
  end
126
- expect(Zdm.io.string).to eq(%[people: 57.14% (8/14)\npeople: Completed (2 secs)\n])
153
+ expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/14\)/)
154
+ expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
127
155
  expect(batches).to eq([
128
156
  %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 5 AND 8],
129
157
  %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 9 AND 12],
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zdm
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - ITRP Institute, Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-05-21 00:00:00.000000000 Z
11
+ date: 2017-07-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord