tableflip 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,5 @@
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'mysql2'
4
+ gem 'eventmachine'
5
+ gem 'em-synchrony'
6
+
7
+ group :development do
8
+ gem 'bundler', '>= 1.0.0'
9
+ gem 'jeweler', '>= 1.8.4'
10
+ end
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2013 Scott Tadman, The Working Group Inc.
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,26 @@
1
+ # tableflip
2
+
3
+ MySQL Table Migration Tool
4
+
5
+ ## Background
6
+
7
+ Maybe you have a number of large tables that need to be packed up and moved
8
+ somewhere else. Maybe you've tried other methods that almost work but don't
9
+ quite. This might be your only hope.
10
+
11
+ ## Inspiration
12
+
13
+ (╯°□°)╯︵ ┻━┻
14
+
15
+ ## Caveats
16
+
17
+ This tool makes an extraordinary number of assumptions about how your data
18
+ is structured and what gems you have available. By some small miracle it might
19
+ work on your project, but it has an equally large chance of not working at all.
20
+
21
+ Needless to say, this is not a fully functional, battle-tested tool. Yet.
22
+
23
+ ## Copyright
24
+
25
+ Copyright (c) 2013 Scott Tadman, The Working Group Inc.
26
+ See LICENSE.txt for further details.
@@ -0,0 +1,38 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+
6
+ begin
7
+ Bundler.setup(:default, :development)
8
+ rescue Bundler::BundlerError => e
9
+ $stderr.puts e.message
10
+ $stderr.puts "Run `bundle install` to install missing gems"
11
+ exit e.status_code
12
+ end
13
+
14
+ require 'rake'
15
+ require 'jeweler'
16
+
17
+ Jeweler::Tasks.new do |gem|
18
+ # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
19
+ gem.name = "tableflip"
20
+ gem.homepage = "http://github.com/twg/tableflip"
21
+ gem.license = "MIT"
22
+ gem.summary = %Q{MySQL Table Flipping System}
23
+ gem.description = %Q{Flips tables from one database to another}
24
+ gem.email = "scott@twg.ca"
25
+ gem.authors = [ "Scott Tadman" ]
26
+ # dependencies defined in Gemfile
27
+ end
28
+
29
+ Jeweler::RubygemsDotOrgTasks.new
30
+
31
+ require 'rake/testtask'
32
+
33
+ Rake::TestTask.new(:test) do |test|
34
+ test.libs << 'lib' << 'test'
35
+ test.pattern = 'test/**/test_*.rb'
36
+ test.verbose = true
37
+ end
38
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.1
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $: << File.expand_path('../lib', File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__))
4
+
5
+ ENV['TZ'] = 'UTC'
6
+
7
+ require 'tableflip'
8
+
9
+ strategy = Tableflip::ArgumentParser.new.parse(ARGV)
10
+
11
+ Tableflip::Executor.new(strategy).execute!
@@ -0,0 +1,6 @@
1
+ module Tableflip
2
+ autoload(:ArgumentParser, 'tableflip/argument_parser')
3
+ autoload(:DatabaseHandle, 'tableflip/database_handle')
4
+ autoload(:Executor, 'tableflip/executor')
5
+ autoload(:Strategy, 'tableflip/strategy')
6
+ end
@@ -0,0 +1,110 @@
1
+ require 'optparse'
2
+
3
+ class Tableflip::ArgumentParser
4
+ # == Constants ============================================================
5
+
6
+ # == Class Methods ========================================================
7
+
8
+ def self.default_env(env = nil)
9
+ (env || ENV)['RAILS_ENV'] || 'development'
10
+ end
11
+
12
+ # == Instance Methods =====================================================
13
+
14
+ def initialize
15
+ end
16
+
17
+ def parse(args, env = nil)
18
+ strategy = Tableflip::Strategy.new
19
+
20
+ strategy.source_env = self.class.default_env(env)
21
+
22
+ _parser = parser(strategy)
23
+
24
+ tables = _parser.parse!(args)
25
+
26
+ tables.each do |table|
27
+ strategy.tables << table
28
+ end
29
+
30
+ if (strategy.tables.empty? or strategy.actions.empty?)
31
+ strategy.message = _parser.to_s
32
+ end
33
+
34
+ strategy
35
+ end
36
+
37
+ def parser(strategy)
38
+ OptionParser.new do |parser|
39
+ parser.banner = "Usage: tableflip [options] [table_name [table_name [...]]]"
40
+
41
+ parser.separator("")
42
+ parser.separator("Options:")
43
+
44
+ parser.on("-a", "--all", "Track all tables") do |s|
45
+ strategy.tables << :__all__
46
+ end
47
+
48
+ parser.on("-b", "--block=s", "Transfer data in blocks of N rows") do |s|
49
+ strategy.block_size = s.to_i
50
+ end
51
+ parser.on("-f", "--config=s") do |path|
52
+ strategy.config_path = path
53
+ end
54
+ parser.on("-t", "--track", "Add tracking triggers on tables") do
55
+ strategy.actions << :tracking_add
56
+ end
57
+ parser.on("-d", "--seed", "Seed the tracking table with entries from the source table") do
58
+ strategy.actions << :tracking_seed
59
+ end
60
+ parser.on("-r", "--remove", "Remove tracking triggers from tables") do
61
+ strategy.actions << :tracking_remove
62
+ end
63
+ parser.on("-o", "--target=s", "Set target environment") do |s|
64
+ strategy.target_env = s
65
+ end
66
+ parser.on("-m", "--migrate", "Migrate tables to environment") do
67
+ strategy.actions << :table_migrate
68
+ end
69
+ parser.on("-c", "--count", "Count number of records in source table") do
70
+ strategy.actions << :table_count
71
+ end
72
+ parser.on("-s", "--status", "Show current status") do
73
+ strategy.actions << :table_report_status
74
+ end
75
+ parser.on("-e", "--env=s", "Establish primary environment") do |s|
76
+ strategy.source_env = s
77
+ end
78
+ parser.on("-x", "--exclude=s", "Exclude column(s) from migration") do |s|
79
+ s.split(/,/).each do |column|
80
+ strategy.exclude_columns << column
81
+ end
82
+ end
83
+ parser.on("-n", "--encoding=s", "Set connection encoding") do |s|
84
+ strategy.encoding = s
85
+ end
86
+ parser.on("-k", "--create-test", "Creates a test table") do
87
+ strategy.actions << :table_create_test
88
+ end
89
+ parser.on("-z", "--fuzz[=d]", "Inserts and alters records on test table") do |d|
90
+ strategy.actions << :table_fuzz
91
+
92
+ if (d)
93
+ strategy.fuzz_intensity = d.to_i
94
+ end
95
+ end
96
+ parser.on("-p", "--persist", "Keep running perpetually") do
97
+ strategy.persist = true
98
+ end
99
+ parser.on("-w","--where=s", "Add conditions to selecting") do |s|
100
+ strategy.where = s
101
+ end
102
+ parser.on("-q", "--debug", "Show the queries as they're executed") do
103
+ strategy.debug_queries = true
104
+ end
105
+ parser.on("-h", "--help", "Display this help") do
106
+ strategy.message = parser.to_s
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,85 @@
1
+ require 'mysql2'
2
+ require 'mysql2/em'
3
+ require 'yaml'
4
+
5
+ class Tableflip::DatabaseHandle
6
+ # == Constants ============================================================
7
+
8
+ DATABASE_CONFIG_FILE = 'database.yml'
9
+
10
+ DEFAULT_OPTIONS = {
11
+ :symbolize_keys => true,
12
+ :encoding => 'utf-8'
13
+ }.freeze
14
+
15
+ PARAM_MAP = Hash.new do |h, k|
16
+ k.to_sym
17
+ end
18
+
19
+ # == Class Methods ========================================================
20
+
21
+ def self.config_path
22
+ path = Dir.pwd
23
+ last_path = nil
24
+
25
+ while (path != last_path)
26
+ config_path = File.expand_path("config/#{DATABASE_CONFIG_FILE}", path)
27
+
28
+ if (File.exist?(config_path))
29
+ return config_path
30
+ end
31
+
32
+ last_path = path
33
+ path = File.expand_path('..', path)
34
+ end
35
+
36
+ nil
37
+ end
38
+
39
+ def self.config
40
+ @config ||= begin
41
+ _config_path = self.config_path
42
+
43
+ if (!_config_path)
44
+ STDERR.puts("Could not find #{DATABASE_CONFIG_FILE}")
45
+ exit(-1)
46
+ elsif (File.exists?(_config_path))
47
+ File.open(_config_path) do |f|
48
+ YAML.load(f)
49
+ end
50
+ else
51
+ STDERR.puts "Could not open #{_config_path}"
52
+ exit(-1)
53
+ end
54
+ end
55
+ end
56
+
57
+ def self.runtime_environment
58
+ DAEMON_ENV or 'development'
59
+ end
60
+
61
+ def self.environment_config(env)
62
+ _config = self.config[env]
63
+
64
+ unless (_config)
65
+ raise "No environment #{env} defined in #{self.config_path}"
66
+ end
67
+
68
+ options = DEFAULT_OPTIONS.dup
69
+
70
+ _config.each do |k, v|
71
+ options[PARAM_MAP[k]] = v
72
+ end
73
+
74
+ options[:loggers] = [ ]
75
+
76
+ options
77
+ end
78
+
79
+ def self.connect(env, options)
80
+ Mysql2::EM::Client.new(self.environment_config(env).merge(options))
81
+ end
82
+
83
+ # == Instance Methods =====================================================
84
+
85
+ end
@@ -0,0 +1,427 @@
1
+ class Tableflip::Executor
2
+ class BinaryString < String
3
+ end
4
+
5
+ # == Instance Methods =====================================================
6
+
7
+ def initialize(strategy)
8
+ @strategy = strategy
9
+
10
+ @time_format = '%Y-%m-%d %H:%M:%S'
11
+ end
12
+
13
+ def log(message)
14
+ puts "[%s] %s" % [ Time.now.strftime(@time_format), message ]
15
+ end
16
+
17
+ def await
18
+ @await ||= Hash.new { |h, k| h[k] = [ ] }
19
+
20
+ fibers = @await[Fiber.current]
21
+
22
+ fibers << Fiber.current
23
+
24
+ yield if (block_given?)
25
+
26
+ fibers.delete(Fiber.current)
27
+
28
+ while (fibers.any?)
29
+ Fiber.yield
30
+ end
31
+ end
32
+
33
+ def defer
34
+ parent_fiber = Fiber.current
35
+
36
+ fibers = @await[parent_fiber]
37
+
38
+ fiber = Fiber.new do
39
+ yield if (block_given?)
40
+
41
+ fibers.delete(Fiber.current)
42
+
43
+ parent_fiber.resume
44
+ end
45
+
46
+ fibers << fiber
47
+
48
+ EventMachine.next_tick do
49
+ fiber.resume
50
+ end
51
+ end
52
+
53
+ def execute!
54
+ require 'eventmachine'
55
+ require 'em-synchrony'
56
+
57
+ if (@strategy.message)
58
+ puts @strategy.message
59
+ exit(0)
60
+ end
61
+
62
+ tables = { }
63
+
64
+ EventMachine.synchrony do
65
+ if (@strategy.tables.include?(:__all__))
66
+ source_db = Tableflip::DatabaseHandle.connect(
67
+ @strategy.source_env,
68
+ :encoding => @strategy.encoding
69
+ )
70
+
71
+ @strategy.tables.delete(:__all__)
72
+
73
+ result = do_query(source_db, "SHOW TABLES")
74
+
75
+ result.each do |row|
76
+ table_name = row.first[1]
77
+
78
+ case (table_name)
79
+ when 'schema_migrations', /__changes/
80
+ next
81
+ end
82
+
83
+ @strategy.tables << table_name
84
+ end
85
+ end
86
+
87
+ await do
88
+ @strategy.tables.each do |table|
89
+ defer do
90
+ queue = @strategy.actions.dup
91
+
92
+ table_config = tables[table] = {
93
+ :table => table,
94
+ :queue => queue
95
+ }
96
+
97
+ while (action = queue.shift)
98
+ log("#{table} [#{action}]")
99
+
100
+ source_db = Tableflip::DatabaseHandle.connect(
101
+ @strategy.source_env,
102
+ :encoding => @strategy.encoding
103
+ )
104
+
105
+ case (action)
106
+ when :tracking_add
107
+ tracking_add(source_db, table_config)
108
+ when :tracking_remove
109
+ tracking_remove(source_db, table_config)
110
+ when :tracking_seed
111
+ tracking_seed(source_db, table_config)
112
+ when :table_migrate
113
+ @strategy.complete = false
114
+
115
+ target_db = Tableflip::DatabaseHandle.connect(
116
+ @strategy.target_env,
117
+ :encoding => @strategy.encoding
118
+ )
119
+ table_migrate(source_db, target_db, table_config)
120
+ when :table_report_status
121
+ target_db = Tableflip::DatabaseHandle.connect(
122
+ @strategy.target_env,
123
+ :encoding => @strategy.encoding
124
+ )
125
+ table_report_status(source_db, target_db, table_config)
126
+ when :table_count
127
+ table_count(source_db, target_db, table_config)
128
+ when :table_create_test
129
+ table_create_test(source_db, table_config)
130
+ when :table_fuzz
131
+ table_fuzz(source_db, table_config, @strategy.fuzz_intensity)
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
137
+
138
+ EventMachine.stop_event_loop
139
+ end
140
+ end
141
+
142
+ def escaper(db, value)
143
+ case (value)
144
+ when nil
145
+ 'NULL'
146
+ when BinaryString
147
+ "0x%s" % value.unpack("H*")
148
+ when Fixnum
149
+ value
150
+ when Date
151
+ '"' + db.escape(value.strftime('%Y-%m-%d')) + '"'
152
+ when DateTime, Time
153
+ '"' + db.escape(value.utc.strftime('%Y-%m-%d %H:%M:%S')) + '"'
154
+ when Array
155
+ value.collect { |v| escaper(db, v) }.join(',')
156
+ else
157
+ '"' + db.escape(value.to_s) + '"'
158
+ end
159
+ end
160
+
161
+ def do_query(db, query, *values)
162
+ fiber = Fiber.current
163
+ query = query.gsub('?') do |s|
164
+ escaper(db, values.shift)
165
+ end
166
+
167
+ if (@strategy.debug_queries?)
168
+ puts "SQL> #{query}"
169
+ end
170
+
171
+ completed = false
172
+
173
+ while (!completed)
174
+ begin
175
+ deferred = db.query(query)
176
+
177
+ deferred.callback do |result|
178
+ EventMachine.next_tick do
179
+ completed = true
180
+
181
+ fiber.resume(result)
182
+ end
183
+ end
184
+
185
+ deferred.errback do |err|
186
+ EventMachine.next_tick do
187
+ completed = true
188
+
189
+ fiber.resume(err)
190
+ end
191
+ end
192
+
193
+ case (response = Fiber.yield)
194
+ when Exception
195
+ raise response
196
+ else
197
+ return response
198
+ end
199
+
200
+ rescue Mysql2::Error => e
201
+ if (e.to_s.match(/MySQL server has gone away/))
202
+ # Ignore
203
+ else
204
+ raise e
205
+ end
206
+ end
207
+ end
208
+ end
209
+
210
+ def table_exists?(db, table)
211
+ do_query(db, "SHOW FIELDS FROM `#{table}`")
212
+
213
+ true
214
+
215
+ rescue Mysql2::Error
216
+ false
217
+ end
218
+
219
+ def tracking_add(db, table_config)
220
+ table = table_config[:table]
221
+ changes_table = "#{table}__changes"
222
+
223
+ if (table_exists?(db, changes_table))
224
+ STDERR.puts("Table #{changes_table} already exists. Not recreated.")
225
+ else
226
+ do_query(db, "CREATE TABLE `#{changes_table}` (id INT PRIMARY KEY, claim INT, INDEX index_claim (claim))")
227
+ do_query(db, "CREATE TRIGGER `#{table}__tai` AFTER INSERT ON `#{table}` FOR EACH ROW INSERT IGNORE INTO `#{changes_table}` (id) VALUES (NEW.id) ON DUPLICATE KEY UPDATE claim=NULL")
228
+ do_query(db, "CREATE TRIGGER `#{table}__tau` AFTER UPDATE ON `#{table}` FOR EACH ROW INSERT IGNORE INTO `#{changes_table}` (id) VALUES (NEW.id) ON DUPLICATE KEY UPDATE claim=NULL")
229
+ end
230
+ end
231
+
232
+ def tracking_remove(db, table_config)
233
+ table = table_config[:table]
234
+ changes_table = "#{table}__changes"
235
+
236
+ if (table_exists?(db, changes_table))
237
+ do_query(db, "DROP TABLE IF EXISTS `#{table}__changes`")
238
+ do_query(db, "DROP TRIGGER IF EXISTS `#{table}__tai`")
239
+ do_query(db, "DROP TRIGGER IF EXISTS `#{table}__tau`")
240
+ else
241
+ STDERR.puts("Table #{changes_table} does not exist. Not removed.")
242
+ end
243
+ end
244
+
245
+ def tracking_seed(db, table_config)
246
+ table = table_config[:table]
247
+ changes_table = "#{table}__changes"
248
+
249
+ result = do_query(db, "SELECT id FROM `#{table}` #{@strategy.where}")
250
+
251
+ ids = result.collect { |r| r[:id] }
252
+ GC.start
253
+
254
+ if (ids.any?)
255
+ log("Populating #{ids.length} entries into #{changes_table} from #{table}")
256
+
257
+ ((ids.length / @strategy.block_size) + 1).times do |n|
258
+ start_offset = @strategy.block_size * n
259
+ id_block = ids[start_offset, @strategy.block_size]
260
+
261
+ if (id_block and id_block.any?)
262
+ query = "INSERT IGNORE INTO `#{changes_table}` (id) VALUES %s" % [
263
+ id_block.collect { |id| "(%d)" % id }.join(',')
264
+ ]
265
+
266
+ do_query(db, query)
267
+
268
+ log("%d/%d entries added to #{changes_table}" % [ start_offset + id_block.length, ids.length ])
269
+ end
270
+ end
271
+ else
272
+ log("No records to migrate from #{table}")
273
+ end
274
+ end
275
+
276
+ def table_report_status(source_db, target_db, table_config)
277
+ table = table_config[:table]
278
+ changes_table = "#{table}__changes"
279
+
280
+ source_table_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{table}`").first[:count]
281
+ target_table_count = do_query(target_db, "SELECT COUNT(*) AS count FROM `#{table}`").first[:count]
282
+ migrated_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{changes_table}` WHERE claim IS NOT NULL").first[:count]
283
+ tracked_count = do_query(source_db, "SELECT COUNT(*) AS count FROM `#{changes_table}`").first[:count]
284
+
285
+ percentage = tracked_count > 0 ? (migrated_count.to_f * 100 / tracked_count) : 0.0
286
+
287
+ log(
288
+ "%s: %d/%d [%d/%d] (%.1f%%)" % [
289
+ table,
290
+ source_table_count,
291
+ target_table_count,
292
+ migrated_count,
293
+ tracked_count,
294
+ percentage
295
+ ]
296
+ )
297
+ end
298
+
299
+ def table_migrate(source_db, target_db, table_config)
300
+ table = table_config[:table]
301
+ changes_table = "#{table}__changes"
302
+
303
+ result = do_query(source_db, "SELECT COUNT(*) AS rows FROM `#{changes_table}` WHERE claim IS NULL")
304
+ count = table_config[:count] = result.first[:rows]
305
+
306
+ log("#{table} has #{table_config[:count]} records to migrate.")
307
+
308
+ next_claim = do_query(source_db, "SELECT MAX(claim) AS claim FROM `#{changes_table}`").first[:claim] || 0
309
+
310
+ result = do_query(source_db, "SHOW FIELDS FROM `#{table}`")
311
+
312
+ exclusions = Hash[
313
+ @strategy.exclude_columns.collect do |column|
314
+ [ column.to_sym, true ]
315
+ end
316
+ ]
317
+
318
+ columns = [ ]
319
+ binary_columns = { }
320
+
321
+ result.each do |r|
322
+ column = r[:Field].to_sym
323
+
324
+ next if (exclusions[column])
325
+
326
+ columns << column
327
+
328
+ case (r[:Type].downcase)
329
+ when 'tinyblob','blob','mediumblob','longblob','binary','varbinary'
330
+ binary_columns[column] = true
331
+ end
332
+ end
333
+
334
+ if (binary_columns.any?)
335
+ log("#{table} has binary columns: #{binary_columns.keys.join(',')}")
336
+ end
337
+
338
+ @migrating ||= { }
339
+
340
+ fiber = Fiber.current
341
+ migrated = 0
342
+ selected = 1
343
+
344
+ loop do
345
+ next_claim += 1
346
+ do_query(source_db, "UPDATE `#{changes_table}` SET claim=? WHERE claim IS NULL LIMIT ?", next_claim, @strategy.block_size)
347
+
348
+ result = do_query(source_db, "SELECT id FROM `#{changes_table}` WHERE claim=?", next_claim)
349
+
350
+ id_block = result.to_a.collect { |r| r[:id] }
351
+
352
+ if (id_block.length == 0)
353
+ if (@strategy.persist?)
354
+ EventMachine::Timer.new(1) do
355
+ fiber.resume
356
+ end
357
+
358
+ Fiber.yield
359
+
360
+ next
361
+ else
362
+ break
363
+ end
364
+ end
365
+
366
+ log("Claim \##{next_claim} yields #{id_block.length} records.")
367
+
368
+ selected = do_query(source_db, "SELECT * FROM `#{table}` WHERE id IN (?)", id_block)
369
+
370
+ values = selected.collect do |row|
371
+ "(%s)" % [
372
+ escaper(
373
+ source_db,
374
+ columns.collect do |column|
375
+ (binary_columns[column] and row[column]) ? BinaryString.new(row[column]) : row[column]
376
+ end
377
+ )
378
+ ]
379
+ end
380
+
381
+ do_query(target_db, "REPLACE INTO `#{table}` (#{columns.collect { |c| "`#{c}`" }.join(',')}) VALUES #{values.join(',')}")
382
+
383
+ selected = values.length
384
+ migrated += values.length
385
+
386
+ log("Migrated %d/%d records for #{table}" % [ migrated, count ])
387
+ end
388
+ end
389
+
390
+ def table_create_test(db, table_config)
391
+ table = table_config[:table]
392
+
393
+ do_query(db, "CREATE TABLE `#{table}` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(255), created_at DATETIME, updated_at DATETIME)")
394
+ rescue Mysql2::Error => e
395
+ puts e.to_s
396
+ end
397
+
398
+ def table_fuzz(db, table_config, count)
399
+ require 'securerandom'
400
+
401
+ table = table_config[:table]
402
+
403
+ EventMachine::PeriodicTimer.new(1) do
404
+ unless (@inserting)
405
+ @inserting = true
406
+
407
+ Fiber.new do
408
+ now = Time.now.utc.strftime('%Y-%m-%d %H:%M:%S')
409
+
410
+ log("Adding #{count} rows to #{table}")
411
+
412
+ count.times do
413
+ do_query(db,
414
+ "INSERT IGNORE INTO `#{table}` (id, name, created_at, updated_at) VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE name=VALUES(name), updated_at=VALUES(updated_at)",
415
+ SecureRandom.random_number(1<<20),
416
+ SecureRandom.hex,
417
+ now,
418
+ now
419
+ )
420
+ end
421
+
422
+ @inserting = false
423
+ end.resume
424
+ end
425
+ end
426
+ end
427
+ end
@@ -0,0 +1,46 @@
1
+ require 'ostruct'
2
+
3
+ class Tableflip::Strategy
4
+ # == Properties ===========================================================
5
+
6
+ attr_accessor :actions
7
+ attr_accessor :block_size
8
+ attr_accessor :complete
9
+ attr_accessor :config_path
10
+ attr_accessor :debug_queries
11
+ attr_accessor :encoding
12
+ attr_accessor :exclude_columns
13
+ attr_accessor :fuzz_intensity
14
+ attr_accessor :message
15
+ attr_accessor :persist
16
+ attr_accessor :source_env
17
+ attr_accessor :tables
18
+ attr_accessor :target_env
19
+ attr_accessor :where
20
+
21
+ # == Class Methods ========================================================
22
+
23
+ # == Instance Methods =====================================================
24
+
25
+ def initialize
26
+ @actions = [ ]
27
+ @tables = [ ]
28
+ @exclude_columns = [ ]
29
+ @fuzz_intensity = 1
30
+ @block_size = 10000
31
+
32
+ yield(self) if (block_given?)
33
+ end
34
+
35
+ def persist?
36
+ !!@persist
37
+ end
38
+
39
+ def complete?
40
+ !!@complete
41
+ end
42
+
43
+ def debug_queries?
44
+ !!@debug_queries
45
+ end
46
+ end
@@ -0,0 +1,70 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = "tableflip"
8
+ s.version = "0.1.1"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Scott Tadman"]
12
+ s.date = "2013-07-29"
13
+ s.description = "Flips tables from one database to another"
14
+ s.email = "scott@twg.ca"
15
+ s.executables = ["tableflip"]
16
+ s.extra_rdoc_files = [
17
+ "LICENSE.txt",
18
+ "README.md"
19
+ ]
20
+ s.files = [
21
+ ".document",
22
+ "Gemfile",
23
+ "LICENSE.txt",
24
+ "README.md",
25
+ "Rakefile",
26
+ "VERSION",
27
+ "bin/tableflip",
28
+ "lib/tableflip.rb",
29
+ "lib/tableflip/argument_parser.rb",
30
+ "lib/tableflip/database_handle.rb",
31
+ "lib/tableflip/executor.rb",
32
+ "lib/tableflip/strategy.rb",
33
+ "tableflip.gemspec",
34
+ "test/config/.gitignore",
35
+ "test/helper.rb",
36
+ "test/unit/test_tableflip.rb",
37
+ "test/unit/test_tableflip_argument_parser.rb",
38
+ "test/unit/test_tableflip_strategy.rb"
39
+ ]
40
+ s.homepage = "http://github.com/twg/tableflip"
41
+ s.licenses = ["MIT"]
42
+ s.require_paths = ["lib"]
43
+ s.rubygems_version = "1.8.23"
44
+ s.summary = "MySQL Table Flipping System"
45
+
46
+ if s.respond_to? :specification_version then
47
+ s.specification_version = 3
48
+
49
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
50
+ s.add_runtime_dependency(%q<mysql2>, [">= 0"])
51
+ s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
52
+ s.add_runtime_dependency(%q<em-synchrony>, [">= 0"])
53
+ s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
54
+ s.add_development_dependency(%q<jeweler>, [">= 1.8.4"])
55
+ else
56
+ s.add_dependency(%q<mysql2>, [">= 0"])
57
+ s.add_dependency(%q<eventmachine>, [">= 0"])
58
+ s.add_dependency(%q<em-synchrony>, [">= 0"])
59
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
60
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
61
+ end
62
+ else
63
+ s.add_dependency(%q<mysql2>, [">= 0"])
64
+ s.add_dependency(%q<eventmachine>, [">= 0"])
65
+ s.add_dependency(%q<em-synchrony>, [">= 0"])
66
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
67
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
68
+ end
69
+ end
70
+
@@ -0,0 +1,2 @@
1
+ database.yml
2
+
@@ -0,0 +1,20 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+
4
+ begin
5
+ Bundler.setup(:default, :development)
6
+ rescue Bundler::BundlerError => e
7
+ $stderr.puts e.message
8
+ $stderr.puts "Run `bundle install` to install missing gems"
9
+ exit e.status_code
10
+ end
11
+
12
+ require 'test/unit'
13
+
14
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
15
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
16
+
17
+ require 'tableflip'
18
+
19
+ class Test::Unit::TestCase
20
+ end
@@ -0,0 +1,7 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflip < Test::Unit::TestCase
4
+ def test_module
5
+ assert Tableflip
6
+ end
7
+ end
@@ -0,0 +1,40 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflipArgumentParser < Test::Unit::TestCase
4
+ def test_default_env
5
+ assert_equal 'test', Tableflip::ArgumentParser.default_env('RAILS_ENV' => 'test')
6
+ end
7
+
8
+ def test_defaults
9
+ strategy = Tableflip::ArgumentParser.new.parse([ ])
10
+
11
+ assert_equal Tableflip::ArgumentParser.default_env, strategy.source_env
12
+ end
13
+
14
+ def test_defaults_with_env
15
+ strategy = Tableflip::ArgumentParser.new.parse([ ], 'RAILS_ENV' => 'test')
16
+
17
+ assert_equal 'test', strategy.source_env
18
+ end
19
+
20
+ def test_help
21
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --help ])
22
+
23
+ assert strategy.message
24
+ end
25
+
26
+ def test_one_table_one_action
27
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --track example_table ])
28
+
29
+ assert_equal [ :tracking_add ], strategy.actions
30
+ assert_equal [ 'example_table' ], strategy.tables
31
+ end
32
+
33
+ def test_one_table_many_actions
34
+ strategy = Tableflip::ArgumentParser.new.parse(%w[ --track --migrate --target=target_env example_table ])
35
+
36
+ assert_equal [ :tracking_add, :table_migrate ], strategy.actions
37
+ assert_equal [ 'example_table' ], strategy.tables
38
+ assert_equal 'target_env', strategy.target_env
39
+ end
40
+ end
@@ -0,0 +1,32 @@
1
+ require_relative '../helper'
2
+
3
+ class TestTableflipStrategy < Test::Unit::TestCase
4
+ def test_defaults
5
+ strategy = Tableflip::Strategy.new
6
+
7
+ assert_equal [ ], strategy.actions
8
+ assert_equal nil, strategy.config_path
9
+ assert_equal nil, strategy.source_env
10
+ assert_equal [ ], strategy.tables
11
+ assert_equal nil, strategy.target_env
12
+ end
13
+
14
+ def test_example
15
+ strategy = Tableflip::Strategy.new do |strategy|
16
+ strategy.actions << :test
17
+
18
+ strategy.tables << :table_a
19
+ strategy.tables << :table_b
20
+
21
+ strategy.source_env = 'staging'
22
+ strategy.target_env = 'test'
23
+ end
24
+
25
+ assert_equal [ :test ], strategy.actions
26
+ assert_equal [ :table_a, :table_b ], strategy.tables
27
+ assert_equal 'staging', strategy.source_env
28
+ assert_equal 'test', strategy.target_env
29
+
30
+ assert_equal nil, strategy.message
31
+ end
32
+ end
metadata ADDED
@@ -0,0 +1,146 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: tableflip
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Scott Tadman
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-07-29 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: mysql2
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: eventmachine
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: em-synchrony
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: bundler
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: 1.0.0
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: 1.0.0
78
+ - !ruby/object:Gem::Dependency
79
+ name: jeweler
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: 1.8.4
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: 1.8.4
94
+ description: Flips tables from one database to another
95
+ email: scott@twg.ca
96
+ executables:
97
+ - tableflip
98
+ extensions: []
99
+ extra_rdoc_files:
100
+ - LICENSE.txt
101
+ - README.md
102
+ files:
103
+ - .document
104
+ - Gemfile
105
+ - LICENSE.txt
106
+ - README.md
107
+ - Rakefile
108
+ - VERSION
109
+ - bin/tableflip
110
+ - lib/tableflip.rb
111
+ - lib/tableflip/argument_parser.rb
112
+ - lib/tableflip/database_handle.rb
113
+ - lib/tableflip/executor.rb
114
+ - lib/tableflip/strategy.rb
115
+ - tableflip.gemspec
116
+ - test/config/.gitignore
117
+ - test/helper.rb
118
+ - test/unit/test_tableflip.rb
119
+ - test/unit/test_tableflip_argument_parser.rb
120
+ - test/unit/test_tableflip_strategy.rb
121
+ homepage: http://github.com/twg/tableflip
122
+ licenses:
123
+ - MIT
124
+ post_install_message:
125
+ rdoc_options: []
126
+ require_paths:
127
+ - lib
128
+ required_ruby_version: !ruby/object:Gem::Requirement
129
+ none: false
130
+ requirements:
131
+ - - ! '>='
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ required_rubygems_version: !ruby/object:Gem::Requirement
135
+ none: false
136
+ requirements:
137
+ - - ! '>='
138
+ - !ruby/object:Gem::Version
139
+ version: '0'
140
+ requirements: []
141
+ rubyforge_project:
142
+ rubygems_version: 1.8.23
143
+ signing_key:
144
+ specification_version: 3
145
+ summary: MySQL Table Flipping System
146
+ test_files: []