zdm 1.0.2 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/gemfiles/4.1.gemfile.lock +1 -1
- data/gemfiles/4.2.gemfile.lock +1 -1
- data/gemfiles/5.0.gemfile.lock +1 -1
- data/lib/version.rb +1 -1
- data/lib/zdm.rb +33 -19
- data/spec/spec_helper.rb +3 -0
- data/spec/zdm_spec.rb +4 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5e342d256ff439757a08cda9ad15897c90278527
|
4
|
+
data.tar.gz: 72894ff47ca9aaca42c6cd0a3b07bc3d0f492490
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a57d77a2afd53a03508aac7a253d7521ced90318e52ce0c8583e8a2cc5960804b4631f4e94fb6e6424599bc0b7b2c8c578beea3c2ec6124203f1b0232ae11514
|
7
|
+
data.tar.gz: ff0c63a2eab00afa7fc4ec348fd1088348cdec28ecd23214b42818d2c3c6e9da01b96c452257292248234f30a983ba60e6b8e1cf031a3e778d132ebda75103fc
|
data/README.md
CHANGED
@@ -8,6 +8,8 @@ Instead of using outfiles we follow [lhm](https://github.com/soundcloud/lhm)'s a
|
|
8
8
|
|
9
9
|
The code is the readme. If you donot grok the code then you really should not use this.
|
10
10
|
|
11
|
+
See also documentation from mysql on [Online Status for DDL Operations](https://dev.mysql.com/doc/refman/5.7/en/innodb-create-index-overview.html#innodb-online-ddl-summary-grid).
|
12
|
+
|
11
13
|
Install
|
12
14
|
=======
|
13
15
|
|
data/gemfiles/4.1.gemfile.lock
CHANGED
data/gemfiles/4.2.gemfile.lock
CHANGED
data/gemfiles/5.0.gemfile.lock
CHANGED
data/lib/version.rb
CHANGED
data/lib/zdm.rb
CHANGED
@@ -76,12 +76,12 @@ module Zdm
|
|
76
76
|
validate
|
77
77
|
set_session_lock_wait_timeouts
|
78
78
|
cleanup
|
79
|
-
|
80
|
-
|
79
|
+
create_copy_table
|
80
|
+
drop_copy_indexes
|
81
81
|
apply_ddl_statements
|
82
82
|
create_triggers
|
83
83
|
batched_copy
|
84
|
-
|
84
|
+
create_copy_indexes
|
85
85
|
atomic_switcharoo!
|
86
86
|
ensure
|
87
87
|
cleanup
|
@@ -124,7 +124,7 @@ module Zdm
|
|
124
124
|
end
|
125
125
|
end
|
126
126
|
|
127
|
-
def
|
127
|
+
def create_copy_table
|
128
128
|
execute('CREATE TABLE `%s` LIKE `%s`' % [table.copy, table.origin])
|
129
129
|
end
|
130
130
|
|
@@ -180,26 +180,39 @@ module Zdm
|
|
180
180
|
"zdmt_#{trigger_type}_#{table.origin}"[0...64]
|
181
181
|
end
|
182
182
|
|
183
|
-
# Drop indexes to speed up batched_copy
|
184
|
-
|
183
|
+
# Drop indexes to speed up batched_copy.
|
184
|
+
#
|
185
|
+
# "Online DDL support for adding secondary indexes means that you can
|
186
|
+
# generally speed the overall process of creating and loading a table
|
187
|
+
# and associated indexes by creating the table without any secondary
|
188
|
+
# indexes, then adding the secondary indexes after the data is loaded."
|
189
|
+
# https://dev.mysql.com/doc/refman/5.7/en/innodb-create-index-overview.html#idm140602200949744
|
190
|
+
def drop_copy_indexes
|
185
191
|
@indexes = connection.indexes(table.copy).reject(&:unique)
|
186
192
|
@indexes.each do |index_def|
|
187
193
|
execute('ALTER TABLE `%s` DROP INDEX `%s`' % [table.copy, index_def.name])
|
188
194
|
end
|
189
195
|
end
|
190
196
|
|
191
|
-
# Recreate the indexes previously dropped
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
+
# Recreate the indexes previously dropped, using 1 statement so the table
|
198
|
+
# is read through once.
|
199
|
+
def create_copy_indexes
|
200
|
+
return if @indexes.empty?
|
201
|
+
indexes = @indexes.map do |index_def|
|
202
|
+
lengths = if index_def.lengths.is_a?(Hash)
|
197
203
|
index_def.lengths
|
198
204
|
elsif index_def.lengths.compact.any?
|
199
205
|
Hash[index_def.columns.map.with_index { |col, idx| [col, index_def.lengths[idx]] }]
|
200
206
|
end
|
201
|
-
|
207
|
+
index_columns = index_def.columns.map.with_index do |col, idx|
|
208
|
+
index_col_name = '`%s`' % col
|
209
|
+
column_length = lengths && lengths[col]
|
210
|
+
index_col_name << '(%s)' % column_length if column_length
|
211
|
+
index_col_name
|
212
|
+
end
|
213
|
+
"ADD INDEX `#{index_def.name}` (#{index_columns.join(',')}) USING #{index_def.using}"
|
202
214
|
end
|
215
|
+
execute('ALTER TABLE `%s` %s' % [table.copy, indexes.join(', ')])
|
203
216
|
end
|
204
217
|
|
205
218
|
BATCH_SIZE = 40_000
|
@@ -216,6 +229,12 @@ module Zdm
|
|
216
229
|
|
217
230
|
insert_columns = common_columns.map {|c| "`#{c}`"}.join(', ')
|
218
231
|
select_columns = common_columns.map {|c| "`#{table.origin}`.`#{c}`"}.join(', ')
|
232
|
+
sql = <<-SQL.squish
|
233
|
+
INSERT IGNORE INTO `#{table.copy}` (#{insert_columns})
|
234
|
+
SELECT #{select_columns}
|
235
|
+
FROM `#{table.origin}`
|
236
|
+
WHERE `#{table.origin}`.`id` BETWEEN %s AND %s
|
237
|
+
SQL
|
219
238
|
|
220
239
|
batch_size = BATCH_SIZE
|
221
240
|
batch_end = min - 1
|
@@ -225,12 +244,7 @@ module Zdm
|
|
225
244
|
batch_end = [batch_start + batch_size - 1, max].min
|
226
245
|
start_batch_time = Time.now
|
227
246
|
|
228
|
-
execute(
|
229
|
-
INSERT IGNORE INTO `#{table.copy}` (#{insert_columns})
|
230
|
-
SELECT #{select_columns}
|
231
|
-
FROM `#{table.origin}`
|
232
|
-
WHERE `#{table.origin}`.`id` BETWEEN #{batch_start} AND #{batch_end}
|
233
|
-
SQL
|
247
|
+
execute(sql % [batch_start, batch_end])
|
234
248
|
|
235
249
|
if $exit
|
236
250
|
write('Received SIGTERM, exiting...')
|
data/spec/spec_helper.rb
CHANGED
@@ -26,6 +26,7 @@ ActiveRecord::Schema.define version: 0 do
|
|
26
26
|
end
|
27
27
|
add_index(:people, :name, unique: true)
|
28
28
|
add_index(:people, [:account_id, :code], length: {account_id: nil, code: 191})
|
29
|
+
add_index(:people, :created_at)
|
29
30
|
|
30
31
|
create_table :people_teams, id: false, force: true do |t|
|
31
32
|
t.integer :team_id, null: false
|
@@ -36,3 +37,5 @@ end
|
|
36
37
|
ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'foo','bar','2017-03-01 23:59:59')])
|
37
38
|
ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (20,'foo2','bar2','2017-03-02 23:59:59')])
|
38
39
|
|
40
|
+
# ActiveRecord::Base.logger = Logger.new($stdout)
|
41
|
+
|
data/spec/zdm_spec.rb
CHANGED
@@ -44,7 +44,8 @@ describe Zdm do
|
|
44
44
|
`created_at` datetime DEFAULT NULL,
|
45
45
|
`test` varchar(32) COLLATE utf8_unicode_ci DEFAULT 'foo',
|
46
46
|
PRIMARY KEY (`id`), UNIQUE KEY `index_people_on_name` (`name`),
|
47
|
-
KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
|
47
|
+
KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE,
|
48
|
+
KEY `index_people_on_created_at` (`created_at`) USING BTREE
|
48
49
|
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
|
49
50
|
EOS
|
50
51
|
|
@@ -77,7 +78,8 @@ describe Zdm do
|
|
77
78
|
`code` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
|
78
79
|
`created_at` datetime DEFAULT NULL, PRIMARY KEY (`id`),
|
79
80
|
UNIQUE KEY `index_people_on_name` (`name`),
|
80
|
-
KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
|
81
|
+
KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE,
|
82
|
+
KEY `index_people_on_created_at` (`created_at`) USING BTREE
|
81
83
|
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
82
84
|
EOS
|
83
85
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zdm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ITRP Institute, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|