flydata 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +9 -5
- data/lib/flydata/command/sync.rb +89 -988
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +6 -1
- data/lib/flydata/helpers.rb +11 -0
- data/lib/flydata/output/forwarder.rb +166 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +729 -0
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +214 -2
- data/lib/flydata/sync_file_manager.rb +1 -1
- data/lib/flydata/table_def/mysql_table_def.rb +61 -47
- data/lib/flydata/table_def/redshift_table_def.rb +30 -26
- data/spec/flydata/command/sync_spec.rb +0 -1160
- data/spec/flydata/output/forwarder_spec.rb +105 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +224 -23
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +900 -0
- data/spec/flydata/sync_file_manager_spec.rb +159 -0
- data/spec/flydata/table_def/mysql_table_def_spec.rb +2 -2
- data/spec/flydata/table_def/redshift_table_def_spec.rb +199 -44
- metadata +8 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c83d98244448112a8c91cc27a003b144e9924c4d
|
4
|
+
data.tar.gz: dd68c29eb63ef0c262d8a325dee230d145361e97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21adba2fa6c2410d2fdf64dd109e420ea4cbbfabc81175095929ba1561806178c062db181baa19fdea69473bc6613d9df31836c3572b0703316ac679b5d2acf4
|
7
|
+
data.tar.gz: ce8f1af52818ba2aae912b3d4f3ff7ef6173c4c594bc0650c7069bd4aa5da3833b178e4d0d903cc74d1c94c1059f0994e32dcadea2702b42f894e67539ed6904
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.9
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,14 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.2.8 ruby lib
|
6
5
|
|
7
6
|
Gem::Specification.new do |s|
|
8
7
|
s.name = "flydata"
|
9
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.9"
|
10
9
|
|
11
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
|
-
s.require_paths = ["lib"]
|
13
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2014-10-
|
12
|
+
s.date = "2014-10-25"
|
15
13
|
s.description = "FlyData Agent"
|
16
14
|
s.email = "sysadmin@flydata.com"
|
17
15
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -72,6 +70,8 @@ Gem::Specification.new do |s|
|
|
72
70
|
"lib/flydata/heroku/configuration_methods.rb",
|
73
71
|
"lib/flydata/heroku/instance_methods.rb",
|
74
72
|
"lib/flydata/log_monitor.rb",
|
73
|
+
"lib/flydata/output/forwarder.rb",
|
74
|
+
"lib/flydata/parser/mysql/dump_parser.rb",
|
75
75
|
"lib/flydata/parser/mysql/mysql_alter_table.treetop",
|
76
76
|
"lib/flydata/parser_provider.rb",
|
77
77
|
"lib/flydata/preference/data_entry_preference.rb",
|
@@ -91,7 +91,10 @@ Gem::Specification.new do |s|
|
|
91
91
|
"spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
|
92
92
|
"spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
|
93
93
|
"spec/flydata/heroku_spec.rb",
|
94
|
+
"spec/flydata/output/forwarder_spec.rb",
|
94
95
|
"spec/flydata/parser/mysql/alter_table_parser_spec.rb",
|
96
|
+
"spec/flydata/parser/mysql/dump_parser_spec.rb",
|
97
|
+
"spec/flydata/sync_file_manager_spec.rb",
|
95
98
|
"spec/flydata/table_def/mysql_table_def_spec.rb",
|
96
99
|
"spec/flydata/table_def/mysqldump_test_foreign_key.dump",
|
97
100
|
"spec/flydata/table_def/mysqldump_test_table_all.dump",
|
@@ -110,7 +113,8 @@ Gem::Specification.new do |s|
|
|
110
113
|
]
|
111
114
|
s.homepage = "http://flydata.com/"
|
112
115
|
s.licenses = ["All right reserved."]
|
113
|
-
s.
|
116
|
+
s.require_paths = ["lib"]
|
117
|
+
s.rubygems_version = "2.0.14"
|
114
118
|
s.summary = "FlyData Agent"
|
115
119
|
|
116
120
|
if s.respond_to? :specification_version then
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -5,23 +5,26 @@ require 'mysql2'
|
|
5
5
|
require 'flydata/sync_file_manager'
|
6
6
|
require 'flydata/errors'
|
7
7
|
require 'flydata/table_def'
|
8
|
+
require 'flydata/output/forwarder'
|
9
|
+
require 'flydata/parser/mysql/dump_parser'
|
8
10
|
#require 'ruby-prof'
|
9
11
|
|
10
12
|
module Flydata
|
11
13
|
module Command
|
12
14
|
class Sync < Base
|
13
15
|
include Helpers
|
14
|
-
CREATE_TABLE_OPTION = !!(ENV['FLYDATA_CREATE_TABLE_OPTION']) || false
|
15
16
|
INSERT_PROGRESS_INTERVAL = 1000
|
16
17
|
|
17
18
|
# for dump.pos file
|
18
19
|
STATUS_PARSING = 'PARSING'
|
20
|
+
STATUS_WAITING = 'WAITING'
|
19
21
|
STATUS_COMPLETE = 'COMPLETE'
|
20
22
|
|
21
23
|
def self.slop
|
22
24
|
Slop.new do
|
23
25
|
on 'c', 'skip-cleanup', 'Skip server cleanup'
|
24
26
|
on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
|
27
|
+
on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.'
|
25
28
|
end
|
26
29
|
end
|
27
30
|
|
@@ -37,8 +40,10 @@ module Flydata
|
|
37
40
|
end
|
38
41
|
exit 1
|
39
42
|
end
|
43
|
+
|
40
44
|
de = retrieve_data_entry
|
41
45
|
de = load_sync_info(override_tables(de, tables))
|
46
|
+
validate_initial_sync_status(de, tables)
|
42
47
|
flush_buffer_and_stop unless de['mysql_data_entry_preference']['initial_sync']
|
43
48
|
sync_mysql_to_redshift(de)
|
44
49
|
end
|
@@ -148,12 +153,23 @@ module Flydata
|
|
148
153
|
|
149
154
|
def generate_table_ddl(*tables)
|
150
155
|
de = retrieve_data_entry
|
151
|
-
Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
|
156
|
+
Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
|
152
157
|
do_generate_table_ddl(override_tables(de, tables))
|
153
158
|
end
|
154
159
|
|
155
160
|
private
|
156
161
|
|
162
|
+
def validate_initial_sync_status(de, tables)
|
163
|
+
sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
|
164
|
+
dump_pos_info = sync_fm.load_dump_pos
|
165
|
+
fp = sync_fm.dump_file_path
|
166
|
+
|
167
|
+
# status is parsing but dumpfile doesn't exist due to streaming -> raise error
|
168
|
+
if dump_pos_info[:status] == STATUS_PARSING && !File.exists?(fp)
|
169
|
+
raise "FlyData Sync was interrupted with invalid state. Run 'flydata sync:reset#{tables.empty? ? '' : ' ' + tables.join(' ')}' first."
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
157
173
|
def retrieve_data_entry
|
158
174
|
de = retrieve_data_entries.first
|
159
175
|
raise "There are no data entry." unless de
|
@@ -257,15 +273,15 @@ module Flydata
|
|
257
273
|
Flydata::Command::Conf.new.copy_templates
|
258
274
|
end
|
259
275
|
|
260
|
-
|
276
|
+
generate_mysqldump(de, sync_fm, opts.dump_file?) do |mysqldump_io, db_bytesize|
|
261
277
|
sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
|
262
|
-
parse_mysqldump_and_send(dp, de, sync_fm)
|
263
|
-
complete
|
278
|
+
parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
|
264
279
|
end
|
280
|
+
wait_for_mysqldump_processed(dp, de, sync_fm)
|
281
|
+
complete
|
265
282
|
end
|
266
283
|
|
267
|
-
def generate_mysqldump(de, sync_fm,
|
268
|
-
|
284
|
+
def generate_mysqldump(de, sync_fm, file_dump = true, &block)
|
269
285
|
# validate parameter
|
270
286
|
%w(host username database).each do |k|
|
271
287
|
if de['mysql_data_entry_preference'][k].to_s.empty?
|
@@ -274,10 +290,15 @@ module Flydata
|
|
274
290
|
end
|
275
291
|
end
|
276
292
|
|
293
|
+
# Status is waiting or complete -> skip dump and parse
|
294
|
+
dump_pos_info = sync_fm.load_dump_pos
|
295
|
+
return if dump_pos_info[:status] == STATUS_WAITING || dump_pos_info[:status] == STATUS_COMPLETE
|
296
|
+
|
297
|
+
# mysqldump file exists -> skip dump
|
277
298
|
fp = sync_fm.dump_file_path
|
278
|
-
if File.exists?(fp)
|
299
|
+
if file_dump && File.exists?(fp) && File.size(fp) > 0
|
279
300
|
puts " -> Skip"
|
280
|
-
return fp
|
301
|
+
return call_block_or_return_io(fp, &block)
|
281
302
|
end
|
282
303
|
|
283
304
|
tables = de['mysql_data_entry_preference']['tables']
|
@@ -292,17 +313,29 @@ FlyData Sync will start synchonizing the following database tables
|
|
292
313
|
username: #{de['mysql_data_entry_preference']['username']}
|
293
314
|
database: #{de['mysql_data_entry_preference']['database']}
|
294
315
|
tables: #{tables}#{data_servers}
|
316
|
+
EOM
|
317
|
+
confirmation_text << <<-EOM if file_dump
|
295
318
|
dump file: #{fp}
|
296
319
|
|
297
320
|
Dump file saves contents of your tables temporarily. Make sure you have enough disk space.
|
298
|
-
|
299
|
-
|
321
|
+
EOM
|
300
322
|
print confirmation_text
|
301
323
|
|
302
324
|
if ask_yes_no('Start Sync?')
|
303
|
-
|
325
|
+
puts "Checking database size(not same as mysqldump data size)..."
|
326
|
+
db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
|
327
|
+
puts " -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
|
304
328
|
puts "Exporting data from database..."
|
305
|
-
|
329
|
+
if file_dump
|
330
|
+
Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
|
331
|
+
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
332
|
+
new(de['mysql_data_entry_preference']).dump(fp)
|
333
|
+
puts " -> Done"
|
334
|
+
call_block_or_return_io(fp, &block)
|
335
|
+
else
|
336
|
+
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
337
|
+
new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
|
338
|
+
end
|
306
339
|
else
|
307
340
|
newline
|
308
341
|
puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
|
@@ -311,8 +344,23 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
311
344
|
puts
|
312
345
|
return nil
|
313
346
|
end
|
314
|
-
|
315
|
-
|
347
|
+
end
|
348
|
+
|
349
|
+
def call_block_or_return_io(fp, &block)
|
350
|
+
if block
|
351
|
+
f_io = open_file_io(fp)
|
352
|
+
begin
|
353
|
+
block.call(f_io)
|
354
|
+
return nil
|
355
|
+
ensure
|
356
|
+
f_io.close rescue nil
|
357
|
+
end
|
358
|
+
end
|
359
|
+
return open_file_io(fp)
|
360
|
+
end
|
361
|
+
|
362
|
+
def open_file_io(file_path)
|
363
|
+
File.open(file_path, 'r', encoding: "utf-8")
|
316
364
|
end
|
317
365
|
|
318
366
|
# Checkpoint
|
@@ -335,7 +383,7 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
335
383
|
# <- checkpoint
|
336
384
|
#...
|
337
385
|
#CREATE TABLE ...
|
338
|
-
def parse_mysqldump_and_send(dp, de, sync_fm)
|
386
|
+
def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize = nil)
|
339
387
|
# Prepare forwarder
|
340
388
|
de_tag_name = de["tag_name#{env_suffix}"]
|
341
389
|
server_port = dp['server_port']
|
@@ -351,7 +399,7 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
351
399
|
# Load dump.pos file for resume
|
352
400
|
dump_pos_info = sync_fm.load_dump_pos
|
353
401
|
option = dump_pos_info || {}
|
354
|
-
if option[:table_name]
|
402
|
+
if option[:table_name] && option[:last_pos].to_i != -1
|
355
403
|
puts "Resuming... Last processed table: #{option[:table_name]}"
|
356
404
|
else
|
357
405
|
#If its a new sync, ensure server side resources are clean
|
@@ -364,31 +412,14 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
364
412
|
# Start parsing dump file
|
365
413
|
tmp_num_inserted_record = 0
|
366
414
|
dump_fp = sync_fm.dump_file_path
|
367
|
-
dump_file_size = File.size(dump_fp)
|
368
|
-
binlog_pos = Flydata::Mysql::MysqlDumpParser.new(
|
415
|
+
dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
|
416
|
+
binlog_pos = Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
|
417
|
+
mysqldump_io,
|
369
418
|
# create table
|
370
419
|
Proc.new { |mysql_table|
|
371
|
-
redshift_table = Flydata::Mysql::RedshiftTableAdapter.new(mysql_table)
|
372
|
-
mysql_table.set_adapter(:redshift, redshift_table)
|
373
|
-
|
374
420
|
tmp_num_inserted_record = 0
|
375
|
-
|
376
|
-
if CREATE_TABLE_OPTION
|
377
|
-
print "- Creating table: #{redshift_table.table_name}"
|
378
|
-
sql = redshift_table.create_table_sql
|
379
|
-
ret = flydata.redshift_cluster.run_query(sql)
|
380
|
-
if ret['message'].index('ERROR:')
|
381
|
-
if ret['message'].index('already exists')
|
382
|
-
puts " -> Skip"
|
383
|
-
else
|
384
|
-
raise "Failed to create table. error=#{ret['message']}"
|
385
|
-
end
|
386
|
-
else
|
387
|
-
puts " -> OK"
|
388
|
-
end
|
389
|
-
end
|
390
|
-
|
391
421
|
# dump mysql_table for resume
|
422
|
+
#TODO: make it option
|
392
423
|
sync_fm.save_mysql_table_marshal_dump(mysql_table)
|
393
424
|
},
|
394
425
|
# insert record
|
@@ -403,14 +434,18 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
403
434
|
ret
|
404
435
|
},
|
405
436
|
# checkpoint
|
406
|
-
Proc.new { |mysql_table, last_pos, binlog_pos, state, substate|
|
437
|
+
Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
407
438
|
# flush if buffer records exist
|
408
439
|
if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
|
409
440
|
forwarder.flush # send buffer data to the server before checkpoint
|
410
441
|
end
|
411
442
|
|
412
443
|
# show the current progress
|
413
|
-
|
444
|
+
if last_pos.to_i == -1 # stream dump
|
445
|
+
puts " -> #{as_size(bytesize)} (#{bytesize} byte) completed..."
|
446
|
+
else
|
447
|
+
puts " -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
|
448
|
+
end
|
414
449
|
|
415
450
|
# save check point
|
416
451
|
table_name = mysql_table.nil? ? '' : mysql_table.table_name
|
@@ -418,21 +453,28 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
418
453
|
}
|
419
454
|
)
|
420
455
|
forwarder.close
|
421
|
-
|
422
456
|
puts " -> Done"
|
457
|
+
sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
|
423
458
|
|
424
459
|
if ENV['FLYDATA_BENCHMARK']
|
425
460
|
bench_end_time = Time.now
|
426
461
|
elapsed_time = bench_end_time.to_i - bench_start_time.to_i
|
427
462
|
puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
|
428
|
-
return true
|
429
463
|
end
|
430
|
-
|
431
|
-
|
464
|
+
end
|
465
|
+
|
466
|
+
def wait_for_mysqldump_processed(dp, de, sync_fm)
|
467
|
+
return if ENV['FLYDATA_BENCHMARK']
|
432
468
|
|
433
|
-
|
469
|
+
# Status is not waiting -> skip waiting
|
470
|
+
dump_pos_info = sync_fm.load_dump_pos
|
471
|
+
return unless dump_pos_info[:status] == STATUS_WAITING
|
472
|
+
binlog_pos = dump_pos_info[:binlog_pos]
|
473
|
+
|
474
|
+
wait_for_server_data_processing
|
434
475
|
tables = de['mysql_data_entry_preference']['tables'].split(',').join(' ')
|
435
476
|
sync_fm.save_table_binlog_pos(tables, binlog_pos)
|
477
|
+
sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
|
436
478
|
end
|
437
479
|
|
438
480
|
ALL_DONE_MESSAGE_TEMPLATE = <<-EOM
|
@@ -450,6 +492,7 @@ What's next?
|
|
450
492
|
|
451
493
|
Thank you for using FlyData!
|
452
494
|
EOM
|
495
|
+
|
453
496
|
def complete
|
454
497
|
de = load_sync_info(retrieve_data_entry)
|
455
498
|
sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
|
@@ -465,8 +508,7 @@ Thank you for using FlyData!
|
|
465
508
|
Flydata::Command::Sender.new.start(quiet: true)
|
466
509
|
puts " -> Done"
|
467
510
|
|
468
|
-
|
469
|
-
dashboard_url = "#{flydata.flydata_api_host}/data_ports/#{data_port['id']}"
|
511
|
+
dashboard_url = "#{flydata.flydata_api_host}/dashboard"
|
470
512
|
redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
|
471
513
|
last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
|
472
514
|
puts last_message
|
@@ -509,945 +551,4 @@ Thank you for using FlyData!
|
|
509
551
|
end
|
510
552
|
end
|
511
553
|
end
|
512
|
-
|
513
|
-
module Output
|
514
|
-
class ForwarderFactory
|
515
|
-
|
516
|
-
def self.create(forwarder_key, tag, servers, options = {})
|
517
|
-
case forwarder_key
|
518
|
-
when nil, "tcpforwarder"
|
519
|
-
puts "Creating TCP connection" if FLYDATA_DEBUG
|
520
|
-
forward = TcpForwarder.new(tag, servers, options)
|
521
|
-
when "sslforwarder"
|
522
|
-
puts "Creating SSL connection" if FLYDATA_DEBUG
|
523
|
-
forward = SslForwarder.new(tag, servers, options)
|
524
|
-
else
|
525
|
-
raise "Unsupported Forwarding type #{forwarder_key}"
|
526
|
-
end
|
527
|
-
forward
|
528
|
-
end
|
529
|
-
|
530
|
-
end
|
531
|
-
class TcpForwarder
|
532
|
-
FORWARD_HEADER = [0x92].pack('C')
|
533
|
-
BUFFER_SIZE = 1024 * 1024 * 32 # 32M
|
534
|
-
DEFUALT_SEND_TIMEOUT = 60 # 1 minute
|
535
|
-
RETRY_INTERVAL = 2
|
536
|
-
RETRY_LIMIT = 10
|
537
|
-
|
538
|
-
def initialize(tag, servers, options = {})
|
539
|
-
@tag = tag
|
540
|
-
unless servers and servers.kind_of?(Array) and not servers.empty?
|
541
|
-
raise "Servers must not be empty."
|
542
|
-
end
|
543
|
-
@servers = servers
|
544
|
-
@server_index = 0
|
545
|
-
set_options(options)
|
546
|
-
reset
|
547
|
-
end
|
548
|
-
|
549
|
-
def set_options(options)
|
550
|
-
if options[:buffer_size_limit]
|
551
|
-
@buffer_size_limit = options[:buffer_size_limit]
|
552
|
-
else
|
553
|
-
@buffer_size_limit = BUFFER_SIZE
|
554
|
-
end
|
555
|
-
end
|
556
|
-
|
557
|
-
attr_reader :buffer_record_count, :buffer_size
|
558
|
-
|
559
|
-
def emit(records, time = Time.now.to_i)
|
560
|
-
records = [records] unless records.kind_of?(Array)
|
561
|
-
records.each do |record|
|
562
|
-
event_data = [time,record].to_msgpack
|
563
|
-
@buffer_records << event_data
|
564
|
-
@buffer_record_count += 1
|
565
|
-
@buffer_size += event_data.bytesize
|
566
|
-
end
|
567
|
-
if @buffer_size > @buffer_size_limit
|
568
|
-
send
|
569
|
-
else
|
570
|
-
false
|
571
|
-
end
|
572
|
-
end
|
573
|
-
|
574
|
-
#TODO retry logic
|
575
|
-
def send
|
576
|
-
if @buffer_size > 0
|
577
|
-
else
|
578
|
-
return false
|
579
|
-
end
|
580
|
-
if ENV['FLYDATA_BENCHMARK']
|
581
|
-
reset
|
582
|
-
return true
|
583
|
-
end
|
584
|
-
sock = nil
|
585
|
-
retry_count = 0
|
586
|
-
begin
|
587
|
-
sock = connect(pickup_server)
|
588
|
-
|
589
|
-
# Write header
|
590
|
-
sock.write FORWARD_HEADER
|
591
|
-
# Write tag
|
592
|
-
sock.write @tag.to_msgpack
|
593
|
-
# Write records
|
594
|
-
sock.write [0xdb, @buffer_records.bytesize].pack('CN')
|
595
|
-
StringIO.open(@buffer_records) do |i|
|
596
|
-
FileUtils.copy_stream(i, sock)
|
597
|
-
end
|
598
|
-
rescue => e
|
599
|
-
retry_count += 1
|
600
|
-
if retry_count > RETRY_LIMIT
|
601
|
-
puts "! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}"
|
602
|
-
raise e
|
603
|
-
end
|
604
|
-
puts "! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}"
|
605
|
-
wait_time = RETRY_INTERVAL ** retry_count
|
606
|
-
puts " Now waiting for next retry. time=#{wait_time}sec"
|
607
|
-
sleep wait_time
|
608
|
-
retry
|
609
|
-
ensure
|
610
|
-
if sock
|
611
|
-
sock.close rescue nil
|
612
|
-
end
|
613
|
-
end
|
614
|
-
reset
|
615
|
-
true
|
616
|
-
end
|
617
|
-
|
618
|
-
#TODO: Check server status
|
619
|
-
def pickup_server
|
620
|
-
ret_server = @servers[@server_index]
|
621
|
-
@server_index += 1
|
622
|
-
if @server_index >= (@servers.count)
|
623
|
-
@server_index = 0
|
624
|
-
end
|
625
|
-
ret_server
|
626
|
-
end
|
627
|
-
|
628
|
-
def connect(server)
|
629
|
-
host, port = server.split(':')
|
630
|
-
sock = TCPSocket.new(host, port.to_i)
|
631
|
-
|
632
|
-
# Set options
|
633
|
-
opt = [1, DEFUALT_SEND_TIMEOUT].pack('I!I!')
|
634
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
635
|
-
opt = [DEFUALT_SEND_TIMEOUT, 0].pack('L!L!')
|
636
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
637
|
-
|
638
|
-
sock
|
639
|
-
end
|
640
|
-
|
641
|
-
def reset
|
642
|
-
@buffer_records = ''
|
643
|
-
@buffer_record_count = 0
|
644
|
-
@buffer_size = 0
|
645
|
-
end
|
646
|
-
|
647
|
-
def flush
|
648
|
-
send
|
649
|
-
end
|
650
|
-
|
651
|
-
def close
|
652
|
-
flush
|
653
|
-
end
|
654
|
-
end
|
655
|
-
class SslForwarder < TcpForwarder
|
656
|
-
def connect(server)
|
657
|
-
tcp_sock = super
|
658
|
-
ssl_ctx = ssl_ctx_with_verification
|
659
|
-
ssl_sock = OpenSSL::SSL::SSLSocket.new(tcp_sock, ssl_ctx)
|
660
|
-
ssl_sock.sync_close = true
|
661
|
-
ssl_sock.connect
|
662
|
-
ssl_sock
|
663
|
-
end
|
664
|
-
|
665
|
-
private
|
666
|
-
def ssl_ctx_with_verification
|
667
|
-
cert_store = OpenSSL::X509::Store.new
|
668
|
-
cert_store.set_default_paths
|
669
|
-
ssl_ctx = OpenSSL::SSL::SSLContext.new
|
670
|
-
ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
671
|
-
ssl_ctx.cert_store = cert_store
|
672
|
-
ssl_ctx
|
673
|
-
end
|
674
|
-
end
|
675
|
-
end
|
676
|
-
|
677
|
-
module Redshift
|
678
|
-
module Util
|
679
|
-
MAX_TABLENAME_LENGTH = 127
|
680
|
-
REDSHIFT_RESERVED_WORDS = %w[
|
681
|
-
aes128 aes256 all allowoverwrite analyse analyze and any array
|
682
|
-
as asc authorization backup between binary blanksasnull both
|
683
|
-
bytedict case cast check collate column constraint create
|
684
|
-
credentials cross current_date current_time current_timestamp
|
685
|
-
current_user current_user_id default deferrable deflate defrag
|
686
|
-
delta delta32k desc disable distinct do else emptyasnull enable
|
687
|
-
encode encrypt encryption end except explicit false for foreign
|
688
|
-
freeze from full globaldict256 globaldict64k grant group gzip having
|
689
|
-
identity ignore ilike in initially inner intersect into is isnull
|
690
|
-
join leading left like limit localtime localtimestamp lun luns
|
691
|
-
minus mostly13 mostly32 mostly8 natural new not notnull null nulls
|
692
|
-
off offline offset old on only open or order outer overlaps parallel
|
693
|
-
partition percent placing primary raw readratio recover references
|
694
|
-
rejectlog resort restore right select session_user similar some
|
695
|
-
sysdate system table tag tdes text255 text32k then to top trailing
|
696
|
-
true truncatecolumns union unique user using verbose wallet when
|
697
|
-
where with without]
|
698
|
-
# Create a symbol-keyed hash for performance
|
699
|
-
REDSHIFT_RESERVED_WORDS_HASH = REDSHIFT_RESERVED_WORDS.inject({}) {|h, word| h[word.to_sym] = true; h}
|
700
|
-
|
701
|
-
REDSHIFT_SYSTEM_COLUMNS = %w[oid tableoid xmin cmin xmax cmax ctid]
|
702
|
-
REDSHIFT_SYSTEM_COLUMNS_HASH = REDSHIFT_SYSTEM_COLUMNS.inject({}) {|h, word| h[word.to_sym] = true; h}
|
703
|
-
|
704
|
-
def convert_to_valid_name(key, type = :table)
|
705
|
-
@memo ||= { table:{}, column:{} }
|
706
|
-
key_sym = key.to_sym
|
707
|
-
return @memo[type][key_sym] if @memo[type][key_sym]
|
708
|
-
|
709
|
-
name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
|
710
|
-
name = "_#{name}" if is_redshift_reserved_word?(name, type) or name =~ /^[0-9$]/
|
711
|
-
if name.length > MAX_TABLENAME_LENGTH
|
712
|
-
name = nil
|
713
|
-
end
|
714
|
-
@memo[key_sym] = name
|
715
|
-
name
|
716
|
-
end
|
717
|
-
|
718
|
-
def is_redshift_reserved_word?(name, type = :table)
|
719
|
-
return false unless name
|
720
|
-
return true if REDSHIFT_RESERVED_WORDS_HASH[name.to_sym] == true
|
721
|
-
|
722
|
-
case type
|
723
|
-
when :table
|
724
|
-
false
|
725
|
-
when :column
|
726
|
-
REDSHIFT_SYSTEM_COLUMNS_HASH[name.to_sym] == true
|
727
|
-
else
|
728
|
-
false
|
729
|
-
end
|
730
|
-
end
|
731
|
-
end
|
732
|
-
end
|
733
|
-
|
734
|
-
module Mysql
|
735
|
-
class MysqlTable
|
736
|
-
def initialize(table_name, columns = {}, primary_keys = [])
|
737
|
-
@table_name = table_name
|
738
|
-
@columns = columns
|
739
|
-
@primary_keys = primary_keys
|
740
|
-
@adapters = {}
|
741
|
-
end
|
742
|
-
|
743
|
-
attr_accessor :table_name, :columns, :primary_keys
|
744
|
-
|
745
|
-
def add_column(column)
|
746
|
-
@columns[column[:column_name]] = column
|
747
|
-
end
|
748
|
-
|
749
|
-
def set_adapter(key, adapter)
|
750
|
-
@adapters[key] = adapter
|
751
|
-
end
|
752
|
-
|
753
|
-
def adapter(key)
|
754
|
-
@adapters[key]
|
755
|
-
end
|
756
|
-
end
|
757
|
-
|
758
|
-
class RedshiftTableAdapter
|
759
|
-
include Flydata::Redshift::Util
|
760
|
-
def initialize(mysql_table)
|
761
|
-
@table_name = convert_to_valid_name(mysql_table.table_name)
|
762
|
-
set_columns(mysql_table.columns)
|
763
|
-
@primary_keys = mysql_table.primary_keys
|
764
|
-
end
|
765
|
-
|
766
|
-
attr_reader :table_name, :columns, :primary_keys
|
767
|
-
|
768
|
-
def create_table_sql
|
769
|
-
col_def = @columns.inject([]) { |list, (cn, column)|
|
770
|
-
list << build_column_def(column)
|
771
|
-
list
|
772
|
-
}
|
773
|
-
if @primary_keys.count > 0
|
774
|
-
col_def << "primary key (#{@primary_keys.join(',')})"
|
775
|
-
end
|
776
|
-
<<EOT
|
777
|
-
CREATE TABLE #{@table_name} (#{col_def.join(',')});
|
778
|
-
EOT
|
779
|
-
end
|
780
|
-
|
781
|
-
private
|
782
|
-
|
783
|
-
def set_columns(columns)
|
784
|
-
@columns = {}
|
785
|
-
columns.each do |k, column|
|
786
|
-
new_k = convert_to_valid_name(k, :column)
|
787
|
-
new_column = column.dup
|
788
|
-
new_column[:column_name] = new_k
|
789
|
-
@columns[new_k] = convert_column_format_type(new_column)
|
790
|
-
end
|
791
|
-
end
|
792
|
-
|
793
|
-
# Mysql Field Types
|
794
|
-
# http://help.scibit.com/mascon/masconMySQL_Field_Types.html
|
795
|
-
def convert_column_format_type(column)
|
796
|
-
ret_c = {}.merge(column)
|
797
|
-
ret_c.delete(:format_type_str)
|
798
|
-
ret_c[:format_type] = case column[:format_type]
|
799
|
-
when 'tinyint'
|
800
|
-
'smallint'
|
801
|
-
when 'smallint'
|
802
|
-
column[:unsigned] ? 'integer' : 'smallint'
|
803
|
-
when 'mediumint'
|
804
|
-
'integer'
|
805
|
-
when 'int', 'integer'
|
806
|
-
column[:unsigned] ? 'bigint' : 'integer'
|
807
|
-
when 'bigint'
|
808
|
-
# max unsigned bigint is 18446744073709551615
|
809
|
-
column[:unsigned] ? 'decimal(20,0)' : 'bigint'
|
810
|
-
when 'float'
|
811
|
-
'real'
|
812
|
-
when 'double', 'double precision', 'real'
|
813
|
-
'double precision'
|
814
|
-
when 'decimal', 'numeric'
|
815
|
-
ret_c[:format_type_str] = "decimal(#{column[:decimal_precision]},#{column[:decimal_scale]})"
|
816
|
-
'decimal'
|
817
|
-
when 'date'
|
818
|
-
'date'
|
819
|
-
when 'datetime'
|
820
|
-
'timestamp'
|
821
|
-
when 'time'
|
822
|
-
'timestamp' #TODO: redshift does not support time only column type
|
823
|
-
when 'year'
|
824
|
-
'smallint'
|
825
|
-
when 'char'
|
826
|
-
ret_c[:format_type_str] = "char(#{column[:format_size]})"
|
827
|
-
'char'
|
828
|
-
when 'varchar'
|
829
|
-
ret_c[:format_type_str] = "varchar(#{column[:format_size]})"
|
830
|
-
'varchar'
|
831
|
-
when 'tinyblob','tinytext'
|
832
|
-
ret_c[:format_size] = 255
|
833
|
-
ret_c[:format_type_str] = "varchar(#{ret_c[:format_size]})"
|
834
|
-
'varchar'
|
835
|
-
when 'blob','text', 'mediumblob', 'mediumtext', 'longblob', 'longtext'
|
836
|
-
ret_c[:format_size] = 65535 #TODO: review
|
837
|
-
ret_c[:format_type_str] = "varchar(#{ret_c[:format_size]})"
|
838
|
-
'varchar'
|
839
|
-
else
|
840
|
-
#TODO: discuss
|
841
|
-
'varchar'
|
842
|
-
end
|
843
|
-
ret_c
|
844
|
-
end
|
845
|
-
|
846
|
-
def build_column_def(column)
|
847
|
-
format_type = column[:format_type]
|
848
|
-
format_type = column[:format_type_str] if column[:format_type_str]
|
849
|
-
def_str = "#{column[:column_name]} #{format_type}"
|
850
|
-
if column[:not_null]
|
851
|
-
def_str << " not null"
|
852
|
-
elsif column.has_key?(:default)
|
853
|
-
val = column[:default]
|
854
|
-
val = val.nil? ? 'null' : "'#{val}'"
|
855
|
-
def_str << " default #{val}"
|
856
|
-
end
|
857
|
-
def_str
|
858
|
-
end
|
859
|
-
|
860
|
-
end
|
861
|
-
|
862
|
-
class MysqlDumpGenerator
|
863
|
-
# host, port, username, password, database, tables
|
864
|
-
MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
|
865
|
-
EXTRA_MYSQLDUMP_PARAMS = ""
|
866
|
-
def initialize(conf)
|
867
|
-
password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
|
868
|
-
tables = if conf['tables']
|
869
|
-
conf['tables'].split(',').join(' ')
|
870
|
-
else
|
871
|
-
''
|
872
|
-
end
|
873
|
-
@dump_cmd = MYSQL_DUMP_CMD_TEMPLATE %
|
874
|
-
[conf['host'], conf['port'], conf['username'], password, self.class::EXTRA_MYSQLDUMP_PARAMS, conf['database'], tables]
|
875
|
-
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
|
876
|
-
end
|
877
|
-
def dump(file_path)
|
878
|
-
raise "subclass must implement the method"
|
879
|
-
end
|
880
|
-
end
|
881
|
-
|
882
|
-
class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
|
883
|
-
EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
|
884
|
-
def dump(file_path)
|
885
|
-
cmd = "#{@dump_cmd} -r #{file_path}"
|
886
|
-
o, e, s = Open3.capture3(cmd)
|
887
|
-
e.to_s.each_line {|l| puts l unless /^Warning:/ =~ l } unless e.to_s.empty?
|
888
|
-
unless s.exitstatus == 0
|
889
|
-
if File.exists?(file_path)
|
890
|
-
File.open(file_path, 'r') {|f| f.each_line{|l| puts l}}
|
891
|
-
FileUtils.rm(file_path)
|
892
|
-
end
|
893
|
-
raise "Failed to run mysqldump command."
|
894
|
-
end
|
895
|
-
unless File.exists?(file_path)
|
896
|
-
raise "mysqldump file does not exist. Something wrong..."
|
897
|
-
end
|
898
|
-
if File.size(file_path) == 0
|
899
|
-
raise "mysqldump file is empty. Something wrong..."
|
900
|
-
end
|
901
|
-
true
|
902
|
-
end
|
903
|
-
end
|
904
|
-
|
905
|
-
class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
|
906
|
-
EXTRA_MYSQLDUMP_PARAMS = ""
|
907
|
-
CHANGE_MASTER_TEMPLATE = <<EOS
|
908
|
-
--
|
909
|
-
-- Position to start replication or point-in-time recovery from
|
910
|
-
--
|
911
|
-
|
912
|
-
-- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
|
913
|
-
|
914
|
-
EOS
|
915
|
-
|
916
|
-
def dump(file_path)
|
917
|
-
# RDS doesn't allow obtaining binlog position using mysqldump. Get it separately and insert it into the dump file.
|
918
|
-
table_locker = Fiber.new do
|
919
|
-
client = Mysql2::Client.new(@db_opts)
|
920
|
-
# Lock tables
|
921
|
-
client.query "FLUSH LOCAL TABLES;"
|
922
|
-
q = flush_tables_with_read_lock_query(client)
|
923
|
-
puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
|
924
|
-
client.query q
|
925
|
-
begin
|
926
|
-
Fiber.yield # Lock is done. Let dump to start
|
927
|
-
# obtain binlog pos
|
928
|
-
result = client.query "SHOW MASTER STATUS;"
|
929
|
-
row = result.first
|
930
|
-
if row.nil?
|
931
|
-
raise "MySQL DB has no replication master status. Check if the DB is set up as a replication master. In case of RDS, make sure that Backup Retention Period is set to more than 0."
|
932
|
-
end
|
933
|
-
ensure
|
934
|
-
# unlock tables
|
935
|
-
client.query "UNLOCK TABLES;"
|
936
|
-
client.close
|
937
|
-
end
|
938
|
-
|
939
|
-
[row["File"], row['Position']]
|
940
|
-
end
|
941
|
-
|
942
|
-
table_locker.resume # Lock tables
|
943
|
-
begin
|
944
|
-
# start dump
|
945
|
-
Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
|
946
|
-
cmd_in.close_write
|
947
|
-
cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
|
948
|
-
File.open(file_path, "w", encoding: "utf-8") do |f|
|
949
|
-
find_insert_pos = :not_started
|
950
|
-
cmd_out.each_line do |line|
|
951
|
-
if find_insert_pos == :not_started && /^-- Server version/ === line
|
952
|
-
find_insert_pos = :finding
|
953
|
-
elsif find_insert_pos == :finding && /^--/ === line
|
954
|
-
# wait before writing the first database queries
|
955
|
-
file, pos = table_locker.resume # Get binlog pos
|
956
|
-
# insert binlog pos
|
957
|
-
change_master = CHANGE_MASTER_TEMPLATE % [file, pos]
|
958
|
-
f.print change_master
|
959
|
-
|
960
|
-
find_insert_pos = :found
|
961
|
-
# resume dump
|
962
|
-
end
|
963
|
-
f.print line
|
964
|
-
end
|
965
|
-
end
|
966
|
-
cmd_err.each_line do |line|
|
967
|
-
$stderr.print line unless /^Warning:/ === line
|
968
|
-
end
|
969
|
-
end
|
970
|
-
rescue
|
971
|
-
# Cleanup
|
972
|
-
FileUtils.rm(file_path) if File.exists?(file_path)
|
973
|
-
raise
|
974
|
-
ensure
|
975
|
-
# Let table_locker finish its task even if an exception happened
|
976
|
-
table_locker.resume if table_locker.alive?
|
977
|
-
end
|
978
|
-
end
|
979
|
-
|
980
|
-
private
|
981
|
-
# This query generates a query which flushes user tables with read lock
|
982
|
-
FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
|
983
|
-
USER_TABLES_QUERY = <<EOS
|
984
|
-
SELECT CONCAT('`',
|
985
|
-
REPLACE(TABLE_SCHEMA, '`', '``'), '`.`',
|
986
|
-
REPLACE(TABLE_NAME, '`', '``'), '` ')
|
987
|
-
AS tables
|
988
|
-
FROM INFORMATION_SCHEMA.TABLES
|
989
|
-
WHERE TABLE_TYPE = 'BASE TABLE'
|
990
|
-
AND ENGINE NOT IN ('MEMORY', 'CSV', 'PERFORMANCE_SCHEMA');
|
991
|
-
EOS
|
992
|
-
def flush_tables_with_read_lock_query(client)
|
993
|
-
tables = ""
|
994
|
-
if mysql_server_version(client) >= "5.5"
|
995
|
-
# FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
|
996
|
-
result = client.query(USER_TABLES_QUERY)
|
997
|
-
tables = result.collect{|r| r['tables']}.join(", ")
|
998
|
-
end
|
999
|
-
FLUSH_TABLES_QUERY_TEMPLATE % [tables]
|
1000
|
-
end
|
1001
|
-
|
1002
|
-
VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
|
1003
|
-
def mysql_server_version(client)
|
1004
|
-
result = client.query(VERSION_QUERY)
|
1005
|
-
result.first['Value']
|
1006
|
-
end
|
1007
|
-
end
|
1008
|
-
|
1009
|
-
class MysqlDumpParser
|
1010
|
-
|
1011
|
-
module State
|
1012
|
-
START = 'START'
|
1013
|
-
CREATE_TABLE = 'CREATE_TABLE'
|
1014
|
-
CREATE_TABLE_COLUMNS = 'CREATE_TABLE_COLUMNS'
|
1015
|
-
CREATE_TABLE_CONSTRAINTS = 'CREATE_TABLE_CONSTRAINTS'
|
1016
|
-
INSERT_RECORD = 'INSERT_RECORD'
|
1017
|
-
PARSING_INSERT_RECORD = 'PARSING_INSERT_RECORD'
|
1018
|
-
end
|
1019
|
-
|
1020
|
-
attr_accessor :binlog_pos
|
1021
|
-
|
1022
|
-
def initialize(file_path, option = {})
|
1023
|
-
@file_path = file_path
|
1024
|
-
raise "Dump file does not exist. file_path:#{file_path}" unless File.exist?(file_path)
|
1025
|
-
@binlog_pos = option[:binlog_pos]
|
1026
|
-
@option = option
|
1027
|
-
end
|
1028
|
-
|
1029
|
-
def parse(create_table_block, insert_record_block, check_point_block)
|
1030
|
-
invalid_file = false
|
1031
|
-
current_state = State::START
|
1032
|
-
substate = nil
|
1033
|
-
|
1034
|
-
state_start = Proc.new do |f|
|
1035
|
-
line = f.readline.strip
|
1036
|
-
# -- CHANGE MASTER TO MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=120;
|
1037
|
-
m = /^\-\- CHANGE MASTER TO MASTER_LOG_FILE='(?<binfile>[^']+)', MASTER_LOG_POS=(?<pos>\d+)/.match(line)
|
1038
|
-
if m
|
1039
|
-
@binlog_pos = {binfile: m[:binfile], pos: m[:pos].to_i}
|
1040
|
-
current_state = State::CREATE_TABLE
|
1041
|
-
check_point_block.call(nil, f.pos, @binlog_pos, current_state)
|
1042
|
-
end
|
1043
|
-
end
|
1044
|
-
|
1045
|
-
current_table = nil
|
1046
|
-
state_create_table = Proc.new do |f|
|
1047
|
-
line = f.readline.strip
|
1048
|
-
# CREATE TABLE `active_admin_comments` (
|
1049
|
-
m = /^CREATE TABLE `(?<table_name>[^`]+)`/.match(line)
|
1050
|
-
if m
|
1051
|
-
current_table = MysqlTable.new(m[:table_name])
|
1052
|
-
current_state = State::CREATE_TABLE_COLUMNS
|
1053
|
-
end
|
1054
|
-
end
|
1055
|
-
|
1056
|
-
state_create_table_constraints = Proc.new do |f|
|
1057
|
-
line = f.readline.strip
|
1058
|
-
# PRIMARY KEY (`id`),
|
1059
|
-
if line.start_with?(')')
|
1060
|
-
create_table_block.call(current_table)
|
1061
|
-
current_state = State::INSERT_RECORD
|
1062
|
-
check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
|
1063
|
-
elsif m = /^PRIMARY KEY \((?<primary_keys>[^\)]+)\)/.match(line)
|
1064
|
-
current_table.primary_keys = m[:primary_keys].split(',').collect do |pk_str|
|
1065
|
-
pk_str[1..-2]
|
1066
|
-
end
|
1067
|
-
end
|
1068
|
-
end
|
1069
|
-
|
1070
|
-
state_create_table_columns = Proc.new do |f|
|
1071
|
-
start_pos = f.pos
|
1072
|
-
line = f.readline.strip
|
1073
|
-
# `author_type` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
|
1074
|
-
if line.start_with?("\`")
|
1075
|
-
column = {}
|
1076
|
-
|
1077
|
-
# parse column line
|
1078
|
-
line = line[0..-2] if line.end_with?(',')
|
1079
|
-
items = line.split
|
1080
|
-
column[:column_name] = items.shift[1..-2]
|
1081
|
-
column[:format_type_str] = format_type_str = items.shift
|
1082
|
-
pos = format_type_str.index('(')
|
1083
|
-
if pos
|
1084
|
-
ft = column[:format_type] = format_type_str[0..pos-1]
|
1085
|
-
if ft == 'decimal'
|
1086
|
-
precision, scale = format_type_str[pos+1..-2].split(',').collect{|v| v.to_i}
|
1087
|
-
column[:decimal_precision] = precision
|
1088
|
-
column[:decimal_scale] = scale
|
1089
|
-
else
|
1090
|
-
column[:format_size] = format_type_str[pos+1..-2].to_i
|
1091
|
-
end
|
1092
|
-
else
|
1093
|
-
column[:format_type] = format_type_str
|
1094
|
-
end
|
1095
|
-
while (item = items.shift) do
|
1096
|
-
case item
|
1097
|
-
when 'DEFAULT'
|
1098
|
-
value = items.shift
|
1099
|
-
value = value.start_with?('\'') ? value[1..-2] : value
|
1100
|
-
value = nil if value == 'NULL'
|
1101
|
-
column[:default] = value
|
1102
|
-
when 'NOT'
|
1103
|
-
if items[1] == 'NULL'
|
1104
|
-
items.shift
|
1105
|
-
column[:not_null] = true
|
1106
|
-
end
|
1107
|
-
when 'unsigned'
|
1108
|
-
column[:unsigned] = true
|
1109
|
-
else
|
1110
|
-
#ignore other options
|
1111
|
-
end
|
1112
|
-
end
|
1113
|
-
|
1114
|
-
current_table.add_column(column)
|
1115
|
-
else
|
1116
|
-
current_state = State::CREATE_TABLE_CONSTRAINTS
|
1117
|
-
f.pos = start_pos
|
1118
|
-
state_create_table_constraints.call(f)
|
1119
|
-
end
|
1120
|
-
end
|
1121
|
-
|
1122
|
-
state_insert_record = Proc.new do |f|
|
1123
|
-
original_pos = f.pos
|
1124
|
-
command = f.read(6)
|
1125
|
-
if command == 'INSERT'
|
1126
|
-
current_state = State::PARSING_INSERT_RECORD
|
1127
|
-
else
|
1128
|
-
f.pos = original_pos
|
1129
|
-
f.readline
|
1130
|
-
if command == 'UNLOCK'
|
1131
|
-
current_state = State::CREATE_TABLE
|
1132
|
-
check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
|
1133
|
-
end
|
1134
|
-
end
|
1135
|
-
end
|
1136
|
-
|
1137
|
-
state_parsing_insert_record = Proc.new do |f|
|
1138
|
-
values_set = InsertParser.new(f).parse
|
1139
|
-
current_state = State::INSERT_RECORD
|
1140
|
-
if insert_record_block.call(current_table, values_set)
|
1141
|
-
check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
|
1142
|
-
end
|
1143
|
-
end
|
1144
|
-
|
1145
|
-
# Start reading file from top
|
1146
|
-
File.open(@file_path, 'r', encoding: "utf-8") do |f|
|
1147
|
-
last_saved_pos = 0
|
1148
|
-
|
1149
|
-
# resume
|
1150
|
-
if @option[:last_pos]
|
1151
|
-
f.pos = @option[:last_pos].to_i
|
1152
|
-
current_state = @option[:state]
|
1153
|
-
substate = @option[:substate]
|
1154
|
-
current_table = @option[:mysql_table]
|
1155
|
-
end
|
1156
|
-
|
1157
|
-
until f.eof? do
|
1158
|
-
case current_state
|
1159
|
-
when State::START
|
1160
|
-
state_start.call(f)
|
1161
|
-
when State::CREATE_TABLE
|
1162
|
-
state_create_table.call(f)
|
1163
|
-
when State::CREATE_TABLE_COLUMNS
|
1164
|
-
state_create_table_columns.call(f)
|
1165
|
-
when State::CREATE_TABLE_CONSTRAINTS
|
1166
|
-
state_create_table_constraints.call(f)
|
1167
|
-
when State::INSERT_RECORD
|
1168
|
-
state_insert_record.call(f)
|
1169
|
-
when State::PARSING_INSERT_RECORD
|
1170
|
-
state_parsing_insert_record.call(f)
|
1171
|
-
end
|
1172
|
-
end
|
1173
|
-
end
|
1174
|
-
@binlog_pos
|
1175
|
-
end
|
1176
|
-
|
1177
|
-
# Parse the insert line containing multiple values. (max line size is 1kb)
|
1178
|
-
# ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
|
1179
|
-
class InsertParser
|
1180
|
-
#INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
|
1181
|
-
module State
|
1182
|
-
IN_VALUE = 'IN_VALUE'
|
1183
|
-
NEXT_VALUES = 'NEXT_VALUES'
|
1184
|
-
end
|
1185
|
-
|
1186
|
-
def initialize(file)
|
1187
|
-
@file = file
|
1188
|
-
@values = []
|
1189
|
-
@values_set = []
|
1190
|
-
end
|
1191
|
-
|
1192
|
-
def start_ruby_prof
|
1193
|
-
RubyProf.start if defined?(RubyProf) and not RubyProf.running?
|
1194
|
-
end
|
1195
|
-
|
1196
|
-
def stop_ruby_prof
|
1197
|
-
if defined?(RubyProf) and RubyProf.running?
|
1198
|
-
result = RubyProf.stop
|
1199
|
-
#printer = RubyProf::GraphPrinter.new(result)
|
1200
|
-
printer = RubyProf::GraphHtmlPrinter.new(result)
|
1201
|
-
#printer.print(STDOUT)
|
1202
|
-
printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
|
1203
|
-
end
|
1204
|
-
end
|
1205
|
-
|
1206
|
-
def parse
|
1207
|
-
start_ruby_prof
|
1208
|
-
bench_start_time = Time.now
|
1209
|
-
target_line = @file.readline
|
1210
|
-
_parse(target_line)
|
1211
|
-
ensure
|
1212
|
-
stop_ruby_prof
|
1213
|
-
if ENV['FLYDATA_BENCHMARK']
|
1214
|
-
puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
|
1215
|
-
end
|
1216
|
-
end
|
1217
|
-
|
1218
|
-
private
|
1219
|
-
|
1220
|
-
def _parse(target_line)
|
1221
|
-
target_line = target_line.strip
|
1222
|
-
start_index = target_line.index('(')
|
1223
|
-
target_line = target_line[start_index..-2]
|
1224
|
-
|
1225
|
-
# Split insert line text with ',' and take care of ',' inside of the values later.
|
1226
|
-
#
|
1227
|
-
# We are using the C native method that is like 'split', 'start_with?', 'regexp'
|
1228
|
-
# instead of 'String#each_char' and string comparision for the performance.
|
1229
|
-
# 'String#each_char' is twice as slow as the current storategy.
|
1230
|
-
items = target_line.split(',')
|
1231
|
-
index = 0
|
1232
|
-
cur_state = State::NEXT_VALUES
|
1233
|
-
|
1234
|
-
loop do
|
1235
|
-
case cur_state
|
1236
|
-
when State::NEXT_VALUES
|
1237
|
-
chars = items[index]
|
1238
|
-
break unless chars
|
1239
|
-
items[index] = chars[1..-1]
|
1240
|
-
cur_state = State::IN_VALUE
|
1241
|
-
when State::IN_VALUE
|
1242
|
-
chars = items[index]
|
1243
|
-
index += 1
|
1244
|
-
if chars.start_with?("'")
|
1245
|
-
# single item (not last item)
|
1246
|
-
# size check added below otherwise end_with? matches the single quote which was also used by start_with?
|
1247
|
-
if chars.size > 1 and chars.end_with?("'") and !last_char_escaped?(chars)
|
1248
|
-
@values << replace_escape_char(chars[1..-2])
|
1249
|
-
# single item (last item)
|
1250
|
-
# size check added below otherwise end_with? matches the single quote which was also used by start_with?
|
1251
|
-
elsif chars.size > 2 and chars.end_with?("')") and !last_char_escaped?(chars[0..-2])
|
1252
|
-
@values << replace_escape_char(chars[1..-3])
|
1253
|
-
@values_set << @values
|
1254
|
-
@values = []
|
1255
|
-
cur_state = State::NEXT_VALUES
|
1256
|
-
# multi items
|
1257
|
-
else
|
1258
|
-
cur_value = chars[1..-1]
|
1259
|
-
loop do
|
1260
|
-
next_chars = items[index]
|
1261
|
-
index += 1
|
1262
|
-
if next_chars.end_with?('\'') and !last_char_escaped?(next_chars)
|
1263
|
-
cur_value << ','
|
1264
|
-
cur_value << next_chars[0..-2]
|
1265
|
-
@values << replace_escape_char(cur_value)
|
1266
|
-
break
|
1267
|
-
elsif next_chars.end_with?("')") and !last_char_escaped?(next_chars[0..-2])
|
1268
|
-
cur_value << ','
|
1269
|
-
cur_value << next_chars[0..-3]
|
1270
|
-
@values << replace_escape_char(cur_value)
|
1271
|
-
@values_set << @values
|
1272
|
-
@values = []
|
1273
|
-
cur_state = State::NEXT_VALUES
|
1274
|
-
break
|
1275
|
-
else
|
1276
|
-
cur_value << ','
|
1277
|
-
cur_value << next_chars
|
1278
|
-
end
|
1279
|
-
end
|
1280
|
-
end
|
1281
|
-
else
|
1282
|
-
if chars.end_with?(')')
|
1283
|
-
chars = chars[0..-2]
|
1284
|
-
@values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
|
1285
|
-
@values_set << @values
|
1286
|
-
@values = []
|
1287
|
-
cur_state = State::NEXT_VALUES
|
1288
|
-
else
|
1289
|
-
@values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
|
1290
|
-
end
|
1291
|
-
end
|
1292
|
-
else
|
1293
|
-
raise "Invalid state: #{cur_state}"
|
1294
|
-
end
|
1295
|
-
end
|
1296
|
-
return @values_set
|
1297
|
-
end
|
1298
|
-
|
1299
|
-
ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
|
1300
|
-
|
1301
|
-
def replace_escape_char(original)
|
1302
|
-
original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
|
1303
|
-
end
|
1304
|
-
|
1305
|
-
# This method assume that the last character is '(single quotation)
|
1306
|
-
# abcd\' -> true
|
1307
|
-
# abcd\\' -> false (back slash escape back slash)
|
1308
|
-
# abcd\\\' -> true
|
1309
|
-
def last_char_escaped?(text)
|
1310
|
-
flag = false
|
1311
|
-
(text.length - 2).downto(0) do |i|
|
1312
|
-
if text[i] == '\\'
|
1313
|
-
flag = !flag
|
1314
|
-
else
|
1315
|
-
break
|
1316
|
-
end
|
1317
|
-
end
|
1318
|
-
flag
|
1319
|
-
end
|
1320
|
-
|
1321
|
-
def remove_leading_zeros(number_string)
|
1322
|
-
if number_string.start_with?('0')
|
1323
|
-
number_string.sub(/^0*([1-9][0-9]*(\.\d*)?|0(\.\d*)?)$/,'\1')
|
1324
|
-
else
|
1325
|
-
number_string
|
1326
|
-
end
|
1327
|
-
end
|
1328
|
-
end
|
1329
|
-
end
|
1330
|
-
class CompatibilityCheck
|
1331
|
-
|
1332
|
-
class CompatibilityError < StandardError
|
1333
|
-
end
|
1334
|
-
|
1335
|
-
SELECT_QUERY_TMPLT = "SELECT %s"
|
1336
|
-
|
1337
|
-
def initialize(de_hash, dump_dir=nil)
|
1338
|
-
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
1339
|
-
@dump_dir = dump_dir
|
1340
|
-
@errors=[]
|
1341
|
-
end
|
1342
|
-
|
1343
|
-
def check
|
1344
|
-
self.methods.grep(/^check_/).each do |m|
|
1345
|
-
begin
|
1346
|
-
send(m)
|
1347
|
-
rescue CompatibilityError => e
|
1348
|
-
@errors << e
|
1349
|
-
end
|
1350
|
-
end
|
1351
|
-
print_errors
|
1352
|
-
end
|
1353
|
-
|
1354
|
-
def print_errors
|
1355
|
-
return if @errors.empty?
|
1356
|
-
puts "There may be some compatibility issues with your MySQL credentials: "
|
1357
|
-
@errors.each do |error|
|
1358
|
-
puts " * #{error.message}"
|
1359
|
-
end
|
1360
|
-
raise "Please correct these errors if you wish to run FlyData Sync"
|
1361
|
-
end
|
1362
|
-
|
1363
|
-
def check_mysql_user_compat
|
1364
|
-
client = Mysql2::Client.new(@db_opts)
|
1365
|
-
grants_sql = "SHOW GRANTS"
|
1366
|
-
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
1367
|
-
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
1368
|
-
all_privileges_field= ["ALL PRIVILEGES"]
|
1369
|
-
result = client.query(grants_sql)
|
1370
|
-
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
1371
|
-
client.close
|
1372
|
-
missing_priv = []
|
1373
|
-
result.each do |res|
|
1374
|
-
# SHOW GRANTS should only return one column
|
1375
|
-
res_value = res.values.first
|
1376
|
-
if correct_db.all? {|perm| res_value.match(perm)}
|
1377
|
-
necessary_permission_fields.each do |priv|
|
1378
|
-
missing_priv << priv unless res_value.match(priv)
|
1379
|
-
end
|
1380
|
-
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
1381
|
-
end
|
1382
|
-
end
|
1383
|
-
raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
1384
|
-
end
|
1385
|
-
|
1386
|
-
def check_mysql_protocol_tcp_compat
|
1387
|
-
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
1388
|
-
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
1389
|
-
|
1390
|
-
Open3.popen3(query) do |stdin, stdout, stderr|
|
1391
|
-
stdin.close
|
1392
|
-
while !stderr.eof?
|
1393
|
-
line = stderr.gets
|
1394
|
-
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
1395
|
-
raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
1396
|
-
end
|
1397
|
-
end
|
1398
|
-
end
|
1399
|
-
end
|
1400
|
-
|
1401
|
-
def check_mysql_row_mode_compat
|
1402
|
-
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
1403
|
-
errors={}
|
1404
|
-
|
1405
|
-
client = Mysql2::Client.new(@db_opts)
|
1406
|
-
|
1407
|
-
begin
|
1408
|
-
sys_var_to_check.each_key do |sys_var|
|
1409
|
-
sel_query = SELECT_QUERY_TMPLT % sys_var
|
1410
|
-
begin
|
1411
|
-
result = client.query(sel_query)
|
1412
|
-
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
1413
|
-
errors[sys_var]=result.first[sys_var]
|
1414
|
-
end
|
1415
|
-
rescue Mysql2::Error => e
|
1416
|
-
if e.message =~ /Unknown system variable/
|
1417
|
-
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
1418
|
-
errors[sys_var] = false
|
1419
|
-
end
|
1420
|
-
else
|
1421
|
-
raise e
|
1422
|
-
end
|
1423
|
-
end
|
1424
|
-
end
|
1425
|
-
ensure
|
1426
|
-
client.close
|
1427
|
-
end
|
1428
|
-
unless errors.empty?
|
1429
|
-
error_explanation = ""
|
1430
|
-
errors.each_key do |err_key|
|
1431
|
-
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
1432
|
-
end
|
1433
|
-
raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
1434
|
-
end
|
1435
|
-
end
|
1436
|
-
|
1437
|
-
def check_writing_permissions
|
1438
|
-
write_errors = []
|
1439
|
-
paths_to_check = ["~/.flydata"]
|
1440
|
-
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
1441
|
-
paths_to_check.each do |path|
|
1442
|
-
full_path = File.expand_path(path)
|
1443
|
-
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
1444
|
-
write_errors << full_path unless File.writable?(full_path)
|
1445
|
-
end
|
1446
|
-
unless write_errors.empty?
|
1447
|
-
error_dir = write_errors.join(", ")
|
1448
|
-
raise CompatibilityError, "We cannot access the directories: #{error_dir}"
|
1449
|
-
end
|
1450
|
-
end
|
1451
|
-
end
|
1452
|
-
end
|
1453
554
|
end
|