tableflip 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'mysql2'
4
+ gem 'eventmachine'
5
+ gem 'em-synchrony'
6
+
7
+ group :development do
8
+ gem 'bundler', '>= 1.0.0'
9
+ gem 'jeweler', '>= 1.8.4'
10
+ end
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2013 Scott Tadman, The Working Group Inc.
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,26 @@
1
+ # tableflip
2
+
3
+ MySQL Table Migration Tool
4
+
5
+ ## Background
6
+
7
+ Maybe you have a number of large tables that need to be packed up and moved
8
+ somewhere else. Maybe you've tried other methods that almost work but don't
9
+ quite. This might be your only hope.
10
+
11
+ ## Inspiration
12
+
13
+ (╯°□°)╯︵ ┻━┻
14
+
15
+ ## Caveats
16
+
17
+ This tool makes an extraordinary number of assumptions about how your data
18
+ is structured and what gems you have available. By some small miracle it might
19
+ work on your project, but it has an equally large chance of not working at all.
20
+
21
+ Needless to say, this is not a fully functional, battle-tested tool. Yet.
22
+
23
+ ## Copyright
24
+
25
+ Copyright (c) 2013 Scott Tadman, The Working Group Inc.
26
+ See LICENSE.txt for further details.
@@ -0,0 +1,38 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+
6
+ begin
7
+ Bundler.setup(:default, :development)
8
+ rescue Bundler::BundlerError => e
9
+ $stderr.puts e.message
10
+ $stderr.puts "Run `bundle install` to install missing gems"
11
+ exit e.status_code
12
+ end
13
+
14
+ require 'rake'
15
+ require 'jeweler'
16
+
17
+ Jeweler::Tasks.new do |gem|
18
+ # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
19
+ gem.name = "tableflip"
20
+ gem.homepage = "http://github.com/twg/tableflip"
21
+ gem.license = "MIT"
22
+ gem.summary = %Q{MySQL Table Flipping System}
23
+ gem.description = %Q{Flips tables from one database to another}
24
+ gem.email = "scott@twg.ca"
25
+ gem.authors = [ "Scott Tadman" ]
26
+ # dependencies defined in Gemfile
27
+ end
28
+
29
+ Jeweler::RubygemsDotOrgTasks.new
30
+
31
+ require 'rake/testtask'
32
+
33
+ Rake::TestTask.new(:test) do |test|
34
+ test.libs << 'lib' << 'test'
35
+ test.pattern = 'test/**/test_*.rb'
36
+ test.verbose = true
37
+ end
38
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.1
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $: << File.expand_path('../lib', File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__))
4
+
5
+ ENV['TZ'] = 'UTC'
6
+
7
+ require 'tableflip'
8
+
9
+ strategy = Tableflip::ArgumentParser.new.parse(ARGV)
10
+
11
+ Tableflip::Executor.new(strategy).execute!
@@ -0,0 +1,6 @@
1
+ module Tableflip
2
+ autoload(:ArgumentParser, 'tableflip/argument_parser')
3
+ autoload(:DatabaseHandle, 'tableflip/database_handle')
4
+ autoload(:Executor, 'tableflip/executor')
5
+ autoload(:Strategy, 'tableflip/strategy')
6
+ end
@@ -0,0 +1,110 @@
1
+ require 'optparse'
2
+
3
+ class Tableflip::ArgumentParser
4
+ # == Constants ============================================================
5
+
6
+ # == Class Methods ========================================================
7
+
8
+ def self.default_env(env = nil)
9
+ (env || ENV)['RAILS_ENV'] || 'development'
10
+ end
11
+
12
+ # == Instance Methods =====================================================
13
+
14
+ def initialize
15
+ end
16
+
17
+ def parse(args, env = nil)
18
+ strategy = Tableflip::Strategy.new
19
+
20
+ strategy.source_env = self.class.default_env(env)
21
+
22
+ _parser = parser(strategy)
23
+
24
+ tables = _parser.parse!(args)
25
+
26
+ tables.each do |table|
27
+ strategy.tables << table
28
+ end
29
+
30
+ if (strategy.tables.empty? or strategy.actions.empty?)
31
+ strategy.message = _parser.to_s
32
+ end
33
+
34
+ strategy
35
+ end
36
+
37
+ def parser(strategy)
38
+ OptionParser.new do |parser|
39
+ parser.banner = "Usage: tableflip [options] [table_name [table_name [...]]]"
40
+
41
+ parser.separator("")
42
+ parser.separator("Options:")
43
+
44
+ parser.on("-a", "--all", "Track all tables") do |s|
45
+ strategy.tables << :__all__
46
+ end
47
+
48
+ parser.on("-b", "--block=s", "Transfer data in blocks of N rows") do |s|
49
+ strategy.block_size = s.to_i
50
+ end
51
+ parser.on("-f", "--config=s") do |path|
52
+ strategy.config_path = path
53
+ end
54
+ parser.on("-t", "--track", "Add tracking triggers on tables") do
55
+ strategy.actions << :tracking_add
56
+ end
57
+ parser.on("-d", "--seed", "Seed the tracking table with entries from the source table") do
58
+ strategy.actions << :tracking_seed
59
+ end
60
+ parser.on("-r", "--remove", "Remove tracking triggers from tables") do
61
+ strategy.actions << :tracking_remove
62
+ end
63
+ parser.on("-o", "--target=s", "Set target environment") do |s|
64
+ strategy.target_env = s
65
+ end
66
+ parser.on("-m", "--migrate", "Migrate tables to environment") do
67
+ strategy.actions << :table_migrate
68
+ end
69
+ parser.on("-c", "--count", "Count number of records in source table") do
70
+ strategy.actions << :table_count
71
+ end
72
+ parser.on("-s", "--status", "Show current status") do
73
+ strategy.actions << :table_report_status
74
+ end
75
+ parser.on("-e", "--env=s", "Establish primary environment") do |s|
76
+ strategy.source_env = s
77
+ end
78
+ parser.on("-x", "--exclude=s", "Exclude column(s) from migration") do |s|
79
+ s.split(/,/).each do |column|
80
+ strategy.exclude_columns << column
81
+ end
82
+ end
83
+ parser.on("-n", "--encoding=s", "Set connection encoding") do |s|
84
+ strategy.encoding = s
85
+ end
86
+ parser.on("-k", "--create-test", "Creates a test table") do
87
+ strategy.actions << :table_create_test
88
+ end
89
+ parser.on("-z", "--fuzz[=d]", "Inserts and alters records on test table") do |d|
90
+ strategy.actions << :table_fuzz
91
+
92
+ if (d)
93
+ strategy.fuzz_intensity = d.to_i
94
+ end
95
+ end
96
+ parser.on("-p", "--persist", "Keep running perpetually") do
97
+ strategy.persist = true
98
+ end
99
+ parser.on("-w","--where=s", "Add conditions to selecting") do |s|
100
+ strategy.where = s
101
+ end
102
+ parser.on("-q", "--debug", "Show the queries as they're executed") do
103
+ strategy.debug_queries = true
104
+ end
105
+ parser.on("-h", "--help", "Display this help") do
106
+ strategy.message = parser.to_s
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,85 @@
1
+ require 'mysql2'
2
+ require 'mysql2/em'
3
+ require 'yaml'
4
+
5
+ class Tableflip::DatabaseHandle
6
+ # == Constants ============================================================
7
+
8
+ DATABASE_CONFIG_FILE = 'database.yml'
9
+
10
+ DEFAULT_OPTIONS = {
11
+ :symbolize_keys => true,
12
+ :encoding => 'utf-8'
13
+ }.freeze
14
+
15
+ PARAM_MAP = Hash.new do |h, k|
16
+ k.to_sym
17
+ end
18
+
19
+ # == Class Methods ========================================================
20
+
21
+ def self.config_path
22
+ path = Dir.pwd
23
+ last_path = nil
24
+
25
+ while (path != last_path)
26
+ config_path = File.expand_path("config/#{DATABASE_CONFIG_FILE}", path)
27
+
28
+ if (File.exist?(config_path))
29
+ return config_path
30
+ end
31
+
32
+ last_path = path
33
+ path = File.expand_path('..', path)
34
+ end
35
+
36
+ nil
37
+ end
38
+
39
+ def self.config
40
+ @config ||= begin
41
+ _config_path = self.config_path
42
+
43
+ if (!_config_path)
44
+ STDERR.puts("Could not find #{DATABASE_CONFIG_FILE}")
45
+ exit(-1)
46
+ elsif (File.exists?(_config_path))
47
+ File.open(_config_path) do |f|
48
+ YAML.load(f)
49
+ end
50
+ else
51
+ STDERR.puts "Could not open #{_config_path}"
52
+ exit(-1)
53
+ end
54
+ end
55
+ end
56
+
57
+ def self.runtime_environment
58
+ DAEMON_ENV or 'development'
59
+ end
60
+
61
+ def self.environment_config(env)
62
+ _config = self.config[env]
63
+
64
+ unless (_config)
65
+ raise "No environment #{env} defined in #{self.config_path}"
66
+ end
67
+
68
+ options = DEFAULT_OPTIONS.dup
69
+
70
+ _config.each do |k, v|
71
+ options[PARAM_MAP[k]] = v
72
+ end
73
+
74
+ options[:loggers] = [ ]
75
+
76
+ options
77
+ end
78
+
79
+ def self.connect(env, options)
80
+ Mysql2::EM::Client.new(self.environment_config(env).merge(options))
81
+ end
82
+
83
+ # == Instance Methods =====================================================
84
+
85
+ end
@@ -0,0 +1,427 @@
1
+ class Tableflip::Executor
2
+ class BinaryString < String
3
+ end
4
+
5
+ # == Instance Methods =====================================================
6
+
7
+ def initialize(strategy)
8
+ @strategy = strategy
9
+
10
+ @time_format = '%Y-%m-%d %H:%M:%S'
11
+ end
12
+
13
+ def log(message)
14
+ puts "[%s] %s" % [ Time.now.strftime(@time_format), message ]
15
+ end
16
+
17
+ def await
18
+ @await ||= Hash.new { |h, k| h[k] = [ ] }
19
+
20
+ fibers = @await[Fiber.current]
21
+
22
+ fibers << Fiber.current
23
+
24
+ yield if (block_given?)
25
+
26
+ fibers.delete(Fiber.current)
27
+
28
+ while (fibers.any?)
29
+ Fiber.yield
30
+ end
31
+ end
32
+
33
+ def defer
34
+ parent_fiber = Fiber.current
35
+
36
+ fibers = @await[parent_fiber]
37
+
38
+ fiber = Fiber.new do
39
+ yield if (block_given?)
40
+
41
+ fibers.delete(Fiber.current)
42
+
43
+ parent_fiber.resume
44
+ end
45
+
46
+ fibers << fiber
47
+
48
+ EventMachine.next_tick do
49
+ fiber.resume
50
+ end
51
+ end
52
+
53
+ def execute!
54
+ require 'eventmachine'
55
+ require 'em-synchrony'
56
+
57
+ if (@strategy.message)
58
+ puts @strategy.message
59
+ exit(0)
60
+ end
61
+
62
+ tables = { }
63
+
64
+ EventMachine.synchrony do
65
+ if (@strategy.tables.include?(:__all__))
66
+ source_db = Tableflip::DatabaseHandle.connect(
67
+ @strategy.source_env,
68
+ :encoding => @strategy.encoding
69
+ )
70
+
71
+ @strategy.tables.delete(:__all__)
72
+
73
+ result = do_query(source_db, "SHOW TABLES")
74
+
75
+ result.each do |row|
76
+ table_name = row.first[1]
77
+
78
+ case (table_name)
79
+ when 'schema_migrations', /__changes/
80
+ next
81
+ end
82
+
83
+ @strategy.tables << table_name
84
+ end
85
+ end
86
+
87
+ await do
88
+ @strategy.tables.each do |table|
89
+ defer do
90
+ queue = @strategy.actions.dup
91
+
92
+ table_config = tables[table] = {
93
+ :table => table,
94
+ :queue => queue
95
+ }
96
+
97
+ while (action = queue.shift)
98
+ log("#{table} [#{action}]")
99
+
100
+ source_db = Tableflip::DatabaseHandle.connect(
101
+ @strategy.source_env,
102
+ :encoding => @strategy.encoding
103
+ )
104
+
105
+ case (action)
106
+ when :tracking_add
107
+ tracking_add(source_db, table_config)
108
+ when :tracking_remove
109
+ tracking_remove(source_db, table_config)
110
+ when :tracking_seed
111
+ tracking_seed(source_db, table_config)
112
+ when :table_migrate
113
+ @strategy.complete = false
114
+
115
+ target_db = Tableflip::DatabaseHandle.connect(
116
+ @strategy.target_env,
117
+ :encoding => @strategy.encoding
118
+ )
119
+ table_migrate(source_db, target_db, table_config)
120
+ when :table_report_status
121
+ target_db = Tableflip::DatabaseHandle.connect(
122
+ @strategy.target_env,
123
+ :encoding => @strategy.encoding
124
+ )
125
+ table_report_status(source_db, target_db, table_config)
126
+ when :table_count
127
+ table_count(source_db, target_db, table_config)
128
+ when :table_create_test
129
+ table_create_test(source_db, table_config)
130
+ when :table_fuzz
131
+ table_fuzz(source_db, table_config, @strategy.fuzz_intensity)
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
137
+
138
+ EventMachine.stop_event_loop
139
+ end
140
+ end
141
+
142
+ def escaper(db, value)
143
+ case (value)
144
+ when nil
145
+ 'NULL'
146
+ when BinaryString
147
+ "0x%s" % value.unpack("H*")
148
+ when Fixnum
149
+ value
150
+ when Date
151
+ '"' + db.escape(value.strftime('%Y-%m-%d')) + '"'
152
+ when DateTime, Time
153
+ '"' + db.escape(value.utc.strftime('%Y-%m-%d %H:%M:%S')) + '"'
154
+ when Array
155
+ value.collect { |v| escaper(db, v) }.join(',')
156
+ else
157
+ '"' + db.escape(value.to_s) + '"'
158
+ end
159
+ end
160
+
161
+ def do_query(db, query, *values)
162
+ fiber = Fiber.current
163
+ query = query.gsub('?') do |s|
164
+ escaper(db, values.shift)
165
+ end
166
+
167
+ if (@strategy.debug_queries?)
168
+ puts "SQL> #{query}"
169
+ end
170
+
171
+ completed = false
172
+
173
+ while (!completed)
174
+ begin
175
+ deferred = db.query(query)
176
+
177
+ deferred.callback do |result|
178
+ EventMachine.next_tick do
179
+ completed = true
180
+
181
+ fiber.resume(result)
182
+ end
183
+ end
184
+
185
+ deferred.errback do |err|
186
+ EventMachine.next_tick do
187
+ completed = true
188
+
189
+ fiber.resume(err)
190
+ end
191
+ end
192
+
193
+ case (response = Fiber.yield)
194
+ when Exception
195
+ raise response
196
+ else
197
+ return response
198
+ end
199
+
200
+ rescue Mysql2::Error => e
201
+ if (e.to_s.match(/MySQL server has gone away/))
202
+ # Ignore
203
+ else
204
+ raise e
205
+ end
206
+ end
207
+ end
208
+ end
209
+
210
+ def table_exists?(db, table)
211
+ do_query(db, "SHOW FIELDS FROM `#{table}`")
212
+
213
+ true
214
+
215
+ rescue Mysql2::Error
216
+ false
217
+ end
218
+
219
+ def tracking_add(db, table_config)
220
+ table = table_config[:table]
221
+ changes_table = "#{table}__changes"
222
+
223
+ if (table_exists?(db, changes_table))
224
+ STDERR.puts("Table #{changes_table} already exists. Not recreated.")
225
+ else
226
+ do_query(db, "CREATE TABLE `#{changes_table}` (id INT PRIMARY KEY, claim INT, INDEX index_claim (claim))")
227
+ do_query(db, "CREATE TRIGGER `#{table}__tai` AFTER INSERT ON `#{table}` FOR EACH ROW INSERT IGNORE INTO `#{changes_table}` (id) VALUES (NEW.id) ON DUPLICATE KEY UPDATE claim=NULL")
228
+ do_query(db, "CREATE TRIGGER `#{table}__tau` AFTER UPDATE ON `#{table}` FOR EACH ROW INSERT IGNORE INTO `#{changes_table}` (id) VALUES (NEW.id) ON DUPLICATE KEY UPDATE claim=NULL")
229
+ end
230
+ end
231
+
232
+ def tracking_remove(db, table_config)
233
+ table = table_config[:table]
234
+ changes_table = "#{table}__changes"
235
+
236
+ if (table_exists?(db, changes_table))
237
+ do_query(db, "DROP TABLE IF EXISTS `#{table}__changes`")
238
+ do_query(db, "DROP TRIGGER IF EXISTS `#{table}__tai`")
239
+ do_query(db, "DROP TRIGGER IF EXISTS `#{table}__tau`")
240
+ else
241
+ STDERR.puts("Table #{changes_table} does not exist. Not removed.")
242
+ end
243
+ end
244
+
245
+ def tracking_seed(db, table_config)
246
+ table = table_config[:table]
247
+ changes_table = "#{table}__changes"
248
+
249
+ result = do_query(db, "SELECT id FROM `#{table}` #{@strategy.where}")
250
+
251
+ ids = result.collect { |r| r[:id] }
252
+ GC.start
253
+
254
+ if (ids.any?)
255
+ log("Populating #{ids.length} entries into #{changes_table} from #{table}")
256
+
257
+ ((ids.length / @strategy.block_size) + 1).times do |n|
258
+ start_offset = @strategy.block_size * n
259
+ id_block = ids[start_offset, @strategy.block_size]
260
+
261
+ if (id_block and id_block.any?)
262
+ query = "INSERT IGNORE INTO `#{changes_table}` (id) VALUES %s" % [
263
+ id_block.collect { |id| "(%d)" % id }.join(',')
264
+ ]
265
+
266
+ do_query(db, query)
267
+
268
+ log("%d/%d entries added to #{changes_table}" % [ start_offset + id_block.length, ids.length ])
269
+ end
270
+ end
271
+ else
272
+ log("No records to migrate from #{table}")
273
+ end
274
+ end
275
+
276
+ def table_report_status(source_db, target_db, table_config)
277
+ table = table_config[:table]
278
+ changes_table = "#{table}__changes"
279
+
280
+ source_table_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{table}`").first[:count]
281
+ target_table_count = do_query(target_db, "SELECT COUNT(*) AS count FROM `#{table}`").first[:count]
282
+ migrated_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{changes_table}` WHERE claim IS NOT NULL").first[:count]
283
+ tracked_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{changes_table}`").first[:count]
284
+
285
+ percentage = tracked_count > 0 ? (migrated_count.to_f * 100 / tracked_count) : 0.0
286
+
287
+ log(
288
+ "%s: %d/%d [%d/%d] (%.1f%%)" % [
289
+ table,
290
+ source_table_count,
291
+ target_table_count,
292
+ migrated_count,
293
+ tracked_count,
294
+ percentage
295
+ ]
296
+ )
297
+ end
298
+
299
+ def table_migrate(source_db, target_db, table_config)
300
+ table = table_config[:table]
301
+ changes_table = "#{table}__changes"
302
+
303
+ result = do_query(source_db, "SELECT COUNT(*) AS rows FROM `#{changes_table}` WHERE claim IS NULL")
304
+ count = table_config[:count] = result.first[:rows]
305
+
306
+ log("#{table} has #{table_config[:count]} records to migrate.")
307
+
308
+ next_claim = do_query(source_db, "SELECT MAX(claim) AS claim FROM `#{changes_table}`").first[:claim] || 0
309
+
310
+ result = do_query(source_db, "SHOW FIELDS FROM `#{table}`")
311
+
312
+ exclusions = Hash[
313
+ @strategy.exclude_columns.collect do |column|
314
+ [ column.to_sym, true ]
315
+ end
316
+ ]
317
+
318
+ columns = [ ]
319
+ binary_columns = { }
320
+
321
+ result.each do |r|
322
+ column = r[:Field].to_sym
323
+
324
+ next if (exclusions[column])
325
+
326
+ columns << column
327
+
328
+ case (r[:Type].downcase)
329
+ when 'tinyblob','blob','mediumblob','longblob','binary','varbinary'
330
+ binary_columns[column] = true
331
+ end
332
+ end
333
+
334
+ if (binary_columns.any?)
335
+ log("#{table} has binary columns: #{binary_columns.keys.join(',')}")
336
+ end
337
+
338
+ @migrating ||= { }
339
+
340
+ fiber = Fiber.current
341
+ migrated = 0
342
+ selected = 1
343
+
344
+ loop do
345
+ next_claim += 1
346
+ do_query(source_db, "UPDATE `#{changes_table}` SET claim=? WHERE claim IS NULL LIMIT ?", next_claim, @strategy.block_size)
347
+
348
+ result = do_query(source_db, "SELECT id FROM `#{changes_table}` WHERE claim=?", next_claim)
349
+
350
+ id_block = result.to_a.collect { |r| r[:id] }
351
+
352
+ if (id_block.length == 0)
353
+ if (@strategy.persist?)
354
+ EventMachine::Timer.new(1) do
355
+ fiber.resume
356
+ end
357
+
358
+ Fiber.yield
359
+
360
+ next
361
+ else
362
+ break
363
+ end
364
+ end
365
+
366
+ log("Claim \##{next_claim} yields #{id_block.length} records.")
367
+
368
+ selected = do_query(source_db, "SELECT * FROM `#{table}` WHERE id IN (?)", id_block)
369
+
370
+ values = selected.collect do |row|
371
+ "(%s)" % [
372
+ escaper(
373
+ source_db,
374
+ columns.collect do |column|
375
+ (binary_columns[column] and row[column]) ? BinaryString.new(row[column]) : row[column]
376
+ end
377
+ )
378
+ ]
379
+ end
380
+
381
+ do_query(target_db, "REPLACE INTO `#{table}` (#{columns.collect { |c| "`#{c}`" }.join(',')}) VALUES #{values.join(',')}")
382
+
383
+ selected = values.length
384
+ migrated += values.length
385
+
386
+ log("Migrated %d/%d records for #{table}" % [ migrated, count ])
387
+ end
388
+ end
389
+
390
+ def table_create_test(db, table_config)
391
+ table = table_config[:table]
392
+
393
+ do_query(db, "CREATE TABLE `#{table}` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(255), created_at DATETIME, updated_at DATETIME)")
394
+ rescue Mysql2::Error => e
395
+ puts e.to_s
396
+ end
397
+
398
+ def table_fuzz(db, table_config, count)
399
+ require 'securerandom'
400
+
401
+ table = table_config[:table]
402
+
403
+ EventMachine::PeriodicTimer.new(1) do
404
+ unless (@inserting)
405
+ @inserting = true
406
+
407
+ Fiber.new do
408
+ now = Time.now.utc.strftime('%Y-%m-%d %H:%M:%S')
409
+
410
+ log("Adding #{count} rows to #{table}")
411
+
412
+ count.times do
413
+ do_query(db,
414
+ "INSERT IGNORE INTO `#{table}` (id, name, created_at, updated_at) VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE name=VALUES(name), updated_at=VALUES(updated_at)",
415
+ SecureRandom.random_number(1<<20),
416
+ SecureRandom.hex,
417
+ now,
418
+ now
419
+ )
420
+ end
421
+
422
+ @inserting = false
423
+ end.resume
424
+ end
425
+ end
426
+ end
427
+ end
@@ -0,0 +1,46 @@
1
+ require 'ostruct'
2
+
3
+ class Tableflip::Strategy
4
+ # == Properties ===========================================================
5
+
6
+ attr_accessor :actions
7
+ attr_accessor :block_size
8
+ attr_accessor :complete
9
+ attr_accessor :config_path
10
+ attr_accessor :debug_queries
11
+ attr_accessor :encoding
12
+ attr_accessor :exclude_columns
13
+ attr_accessor :fuzz_intensity
14
+ attr_accessor :message
15
+ attr_accessor :persist
16
+ attr_accessor :source_env
17
+ attr_accessor :tables
18
+ attr_accessor :target_env
19
+ attr_accessor :where
20
+
21
+ # == Class Methods ========================================================
22
+
23
+ # == Instance Methods =====================================================
24
+
25
+ def initialize
26
+ @actions = [ ]
27
+ @tables = [ ]
28
+ @exclude_columns = [ ]
29
+ @fuzz_intensity = 1
30
+ @block_size = 10000
31
+
32
+ yield(self) if (block_given?)
33
+ end
34
+
35
+ def persist?
36
+ !!@persist
37
+ end
38
+
39
+ def complete?
40
+ !!@complete
41
+ end
42
+
43
+ def debug_queries?
44
+ !!@debug_queries
45
+ end
46
+ end
@@ -0,0 +1,70 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = "tableflip"
8
+ s.version = "0.1.1"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Scott Tadman"]
12
+ s.date = "2013-07-29"
13
+ s.description = "Flips tables from one database to another"
14
+ s.email = "scott@twg.ca"
15
+ s.executables = ["tableflip"]
16
+ s.extra_rdoc_files = [
17
+ "LICENSE.txt",
18
+ "README.md"
19
+ ]
20
+ s.files = [
21
+ ".document",
22
+ "Gemfile",
23
+ "LICENSE.txt",
24
+ "README.md",
25
+ "Rakefile",
26
+ "VERSION",
27
+ "bin/tableflip",
28
+ "lib/tableflip.rb",
29
+ "lib/tableflip/argument_parser.rb",
30
+ "lib/tableflip/database_handle.rb",
31
+ "lib/tableflip/executor.rb",
32
+ "lib/tableflip/strategy.rb",
33
+ "tableflip.gemspec",
34
+ "test/config/.gitignore",
35
+ "test/helper.rb",
36
+ "test/unit/test_tableflip.rb",
37
+ "test/unit/test_tableflip_argument_parser.rb",
38
+ "test/unit/test_tableflip_strategy.rb"
39
+ ]
40
+ s.homepage = "http://github.com/twg/tableflip"
41
+ s.licenses = ["MIT"]
42
+ s.require_paths = ["lib"]
43
+ s.rubygems_version = "1.8.23"
44
+ s.summary = "MySQL Table Flipping System"
45
+
46
+ if s.respond_to? :specification_version then
47
+ s.specification_version = 3
48
+
49
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
50
+ s.add_runtime_dependency(%q<mysql2>, [">= 0"])
51
+ s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
52
+ s.add_runtime_dependency(%q<em-synchrony>, [">= 0"])
53
+ s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
54
+ s.add_development_dependency(%q<jeweler>, [">= 1.8.4"])
55
+ else
56
+ s.add_dependency(%q<mysql2>, [">= 0"])
57
+ s.add_dependency(%q<eventmachine>, [">= 0"])
58
+ s.add_dependency(%q<em-synchrony>, [">= 0"])
59
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
60
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
61
+ end
62
+ else
63
+ s.add_dependency(%q<mysql2>, [">= 0"])
64
+ s.add_dependency(%q<eventmachine>, [">= 0"])
65
+ s.add_dependency(%q<em-synchrony>, [">= 0"])
66
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
67
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
68
+ end
69
+ end
70
+
@@ -0,0 +1,2 @@
1
+ database.yml
2
+
@@ -0,0 +1,20 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+
4
+ begin
5
+ Bundler.setup(:default, :development)
6
+ rescue Bundler::BundlerError => e
7
+ $stderr.puts e.message
8
+ $stderr.puts "Run `bundle install` to install missing gems"
9
+ exit e.status_code
10
+ end
11
+
12
+ require 'test/unit'
13
+
14
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
15
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
16
+
17
+ require 'tableflip'
18
+
19
+ class Test::Unit::TestCase
20
+ end
@@ -0,0 +1,7 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflip < Test::Unit::TestCase
4
+ def test_module
5
+ assert Tableflip
6
+ end
7
+ end
@@ -0,0 +1,40 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflipArgumentParser < Test::Unit::TestCase
4
+ def test_default_env
5
+ assert_equal 'test', Tableflip::ArgumentParser.default_env('RAILS_ENV' => 'test')
6
+ end
7
+
8
+ def test_defaults
9
+ strategy = Tableflip::ArgumentParser.new.parse([ ])
10
+
11
+ assert_equal Tableflip::ArgumentParser.default_env, strategy.source_env
12
+ end
13
+
14
+ def test_defaults_with_env
15
+ strategy = Tableflip::ArgumentParser.new.parse([ ], 'RAILS_ENV' => 'test')
16
+
17
+ assert_equal 'test', strategy.source_env
18
+ end
19
+
20
+ def test_help
21
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --help ])
22
+
23
+ assert strategy.message
24
+ end
25
+
26
+ def test_one_table_one_action
27
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --track example_table ])
28
+
29
+ assert_equal [ :tracking_add ], strategy.actions
30
+ assert_equal [ 'example_table' ], strategy.tables
31
+ end
32
+
33
+ def test_one_table_many_actions
34
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --track --migrate --target=target_env example_table ])
35
+
36
+ assert_equal [ :tracking_add, :table_migrate ], strategy.actions
37
+ assert_equal [ 'example_table' ], strategy.tables
38
+ assert_equal 'target_env', strategy.target_env
39
+ end
40
+ end
@@ -0,0 +1,32 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflipStrategy < Test::Unit::TestCase
4
+ def test_defaults
5
+ strategy = Tableflip::Strategy.new
6
+
7
+ assert_equal [ ], strategy.actions
8
+ assert_equal nil, strategy.config_path
9
+ assert_equal nil, strategy.source_env
10
+ assert_equal [ ], strategy.tables
11
+ assert_equal nil, strategy.target_env
12
+ end
13
+
14
+ def test_example
15
+ strategy = Tableflip::Strategy.new do |strategy|
16
+ strategy.actions << :test
17
+
18
+ strategy.tables << :table_a
19
+ strategy.tables << :table_b
20
+
21
+ strategy.source_env = 'staging'
22
+ strategy.target_env = 'test'
23
+ end
24
+
25
+ assert_equal [ :test ], strategy.actions
26
+ assert_equal [ :table_a, :table_b ], strategy.tables
27
+ assert_equal 'staging', strategy.source_env
28
+ assert_equal 'test', strategy.target_env
29
+
30
+ assert_equal nil, strategy.message
31
+ end
32
+ end
metadata ADDED
@@ -0,0 +1,146 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: tableflip
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Scott Tadman
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-07-29 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: mysql2
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: eventmachine
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: em-synchrony
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: bundler
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 1.0.0
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: 1.0.0
78
+ - !ruby/object:Gem::Dependency
79
+ name: jeweler
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: 1.8.4
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: 1.8.4
94
+ description: Flips tables from one database to another
95
+ email: scott@twg.ca
96
+ executables:
97
+ - tableflip
98
+ extensions: []
99
+ extra_rdoc_files:
100
+ - LICENSE.txt
101
+ - README.md
102
+ files:
103
+ - .document
104
+ - Gemfile
105
+ - LICENSE.txt
106
+ - README.md
107
+ - Rakefile
108
+ - VERSION
109
+ - bin/tableflip
110
+ - lib/tableflip.rb
111
+ - lib/tableflip/argument_parser.rb
112
+ - lib/tableflip/database_handle.rb
113
+ - lib/tableflip/executor.rb
114
+ - lib/tableflip/strategy.rb
115
+ - tableflip.gemspec
116
+ - test/config/.gitignore
117
+ - test/helper.rb
118
+ - test/unit/test_tableflip.rb
119
+ - test/unit/test_tableflip_argument_parser.rb
120
+ - test/unit/test_tableflip_strategy.rb
121
+ homepage: http://github.com/twg/tableflip
122
+ licenses:
123
+ - MIT
124
+ post_install_message:
125
+ rdoc_options: []
126
+ require_paths:
127
+ - lib
128
+ required_ruby_version: !ruby/object:Gem::Requirement
129
+ none: false
130
+ requirements:
131
+ - - ! '>='
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ required_rubygems_version: !ruby/object:Gem::Requirement
135
+ none: false
136
+ requirements:
137
+ - - ! '>='
138
+ - !ruby/object:Gem::Version
139
+ version: '0'
140
+ requirements: []
141
+ rubyforge_project:
142
+ rubygems_version: 1.8.23
143
+ signing_key:
144
+ specification_version: 3
145
+ summary: MySQL Table Flipping System
146
+ test_files: []