flydata 0.2.8 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c5fda2fb95c38692e1a0995b3ee7430ea706ec09
4
- data.tar.gz: 5028033ad26d7399e7bcb091257540f1fb4ab46a
3
+ metadata.gz: c83d98244448112a8c91cc27a003b144e9924c4d
4
+ data.tar.gz: dd68c29eb63ef0c262d8a325dee230d145361e97
5
5
  SHA512:
6
- metadata.gz: dbcb9055ead8575095ef96d378d2817743f0c03cb629d6d7ef5d46bc99f04fd22e420f2c8011c28a97a3c5041ec683ed57045d59bd86c31fdb6ec680dbb81769
7
- data.tar.gz: 1e77c0720dd22cad760cd00f11eef55514bb768937acb5b6970027f352d5cf97d9f2177b6d904aeb74c8716ab6f9f374757f3ce09780e3c166f79ad3b6a7360d
6
+ metadata.gz: 21adba2fa6c2410d2fdf64dd109e420ea4cbbfabc81175095929ba1561806178c062db181baa19fdea69473bc6613d9df31836c3572b0703316ac679b5d2acf4
7
+ data.tar.gz: ce8f1af52818ba2aae912b3d4f3ff7ef6173c4c594bc0650c7069bd4aa5da3833b178e4d0d903cc74d1c94c1059f0994e32dcadea2702b42f894e67539ed6904
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.8
1
+ 0.2.9
data/flydata.gemspec CHANGED
@@ -2,16 +2,14 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: flydata 0.2.8 ruby lib
6
5
 
7
6
  Gem::Specification.new do |s|
8
7
  s.name = "flydata"
9
- s.version = "0.2.8"
8
+ s.version = "0.2.9"
10
9
 
11
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
- s.require_paths = ["lib"]
13
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
14
- s.date = "2014-10-15"
12
+ s.date = "2014-10-25"
15
13
  s.description = "FlyData Agent"
16
14
  s.email = "sysadmin@flydata.com"
17
15
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -72,6 +70,8 @@ Gem::Specification.new do |s|
72
70
  "lib/flydata/heroku/configuration_methods.rb",
73
71
  "lib/flydata/heroku/instance_methods.rb",
74
72
  "lib/flydata/log_monitor.rb",
73
+ "lib/flydata/output/forwarder.rb",
74
+ "lib/flydata/parser/mysql/dump_parser.rb",
75
75
  "lib/flydata/parser/mysql/mysql_alter_table.treetop",
76
76
  "lib/flydata/parser_provider.rb",
77
77
  "lib/flydata/preference/data_entry_preference.rb",
@@ -91,7 +91,10 @@ Gem::Specification.new do |s|
91
91
  "spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
92
92
  "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
93
93
  "spec/flydata/heroku_spec.rb",
94
+ "spec/flydata/output/forwarder_spec.rb",
94
95
  "spec/flydata/parser/mysql/alter_table_parser_spec.rb",
96
+ "spec/flydata/parser/mysql/dump_parser_spec.rb",
97
+ "spec/flydata/sync_file_manager_spec.rb",
95
98
  "spec/flydata/table_def/mysql_table_def_spec.rb",
96
99
  "spec/flydata/table_def/mysqldump_test_foreign_key.dump",
97
100
  "spec/flydata/table_def/mysqldump_test_table_all.dump",
@@ -110,7 +113,8 @@ Gem::Specification.new do |s|
110
113
  ]
111
114
  s.homepage = "http://flydata.com/"
112
115
  s.licenses = ["All right reserved."]
113
- s.rubygems_version = "2.2.2"
116
+ s.require_paths = ["lib"]
117
+ s.rubygems_version = "2.0.14"
114
118
  s.summary = "FlyData Agent"
115
119
 
116
120
  if s.respond_to? :specification_version then
@@ -5,23 +5,26 @@ require 'mysql2'
5
5
  require 'flydata/sync_file_manager'
6
6
  require 'flydata/errors'
7
7
  require 'flydata/table_def'
8
+ require 'flydata/output/forwarder'
9
+ require 'flydata/parser/mysql/dump_parser'
8
10
  #require 'ruby-prof'
9
11
 
10
12
  module Flydata
11
13
  module Command
12
14
  class Sync < Base
13
15
  include Helpers
14
- CREATE_TABLE_OPTION = !!(ENV['FLYDATA_CREATE_TABLE_OPTION']) || false
15
16
  INSERT_PROGRESS_INTERVAL = 1000
16
17
 
17
18
  # for dump.pos file
18
19
  STATUS_PARSING = 'PARSING'
20
+ STATUS_WAITING = 'WAITING'
19
21
  STATUS_COMPLETE = 'COMPLETE'
20
22
 
21
23
  def self.slop
22
24
  Slop.new do
23
25
  on 'c', 'skip-cleanup', 'Skip server cleanup'
24
26
  on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
27
+ on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.'
25
28
  end
26
29
  end
27
30
 
@@ -37,8 +40,10 @@ module Flydata
37
40
  end
38
41
  exit 1
39
42
  end
43
+
40
44
  de = retrieve_data_entry
41
45
  de = load_sync_info(override_tables(de, tables))
46
+ validate_initial_sync_status(de, tables)
42
47
  flush_buffer_and_stop unless de['mysql_data_entry_preference']['initial_sync']
43
48
  sync_mysql_to_redshift(de)
44
49
  end
@@ -148,12 +153,23 @@ module Flydata
148
153
 
149
154
  def generate_table_ddl(*tables)
150
155
  de = retrieve_data_entry
151
- Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
156
+ Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
152
157
  do_generate_table_ddl(override_tables(de, tables))
153
158
  end
154
159
 
155
160
  private
156
161
 
162
+ def validate_initial_sync_status(de, tables)
163
+ sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
164
+ dump_pos_info = sync_fm.load_dump_pos
165
+ fp = sync_fm.dump_file_path
166
+
167
+ # status is parsing but dumpfile doesn't exist due to streaming -> raise error
168
+ if dump_pos_info[:status] == STATUS_PARSING && !File.exists?(fp)
169
+ raise "FlyData Sync was interrupted with invalid state. Run 'flydata sync:reset#{tables.empty? ? '' : ' ' + tables.join(' ')}' first."
170
+ end
171
+ end
172
+
157
173
  def retrieve_data_entry
158
174
  de = retrieve_data_entries.first
159
175
  raise "There are no data entry." unless de
@@ -257,15 +273,15 @@ module Flydata
257
273
  Flydata::Command::Conf.new.copy_templates
258
274
  end
259
275
 
260
- if generate_mysqldump(de, sync_fm)
276
+ generate_mysqldump(de, sync_fm, opts.dump_file?) do |mysqldump_io, db_bytesize|
261
277
  sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
262
- parse_mysqldump_and_send(dp, de, sync_fm)
263
- complete
278
+ parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
264
279
  end
280
+ wait_for_mysqldump_processed(dp, de, sync_fm)
281
+ complete
265
282
  end
266
283
 
267
- def generate_mysqldump(de, sync_fm, overwrite = false)
268
-
284
+ def generate_mysqldump(de, sync_fm, file_dump = true, &block)
269
285
  # validate parameter
270
286
  %w(host username database).each do |k|
271
287
  if de['mysql_data_entry_preference'][k].to_s.empty?
@@ -274,10 +290,15 @@ module Flydata
274
290
  end
275
291
  end
276
292
 
293
+ # Status is waiting or complete -> skip dump and parse
294
+ dump_pos_info = sync_fm.load_dump_pos
295
+ return if dump_pos_info[:status] == STATUS_WAITING || dump_pos_info[:status] == STATUS_COMPLETE
296
+
297
+ # mysqldump file exists -> skip dump
277
298
  fp = sync_fm.dump_file_path
278
- if File.exists?(fp) and File.size(fp) > 0 and not overwrite
299
+ if file_dump && File.exists?(fp) && File.size(fp) > 0
279
300
  puts " -> Skip"
280
- return fp
301
+ return call_block_or_return_io(fp, &block)
281
302
  end
282
303
 
283
304
  tables = de['mysql_data_entry_preference']['tables']
@@ -292,17 +313,29 @@ FlyData Sync will start synchonizing the following database tables
292
313
  username: #{de['mysql_data_entry_preference']['username']}
293
314
  database: #{de['mysql_data_entry_preference']['database']}
294
315
  tables: #{tables}#{data_servers}
316
+ EOM
317
+ confirmation_text << <<-EOM if file_dump
295
318
  dump file: #{fp}
296
319
 
297
320
  Dump file saves contents of your tables temporarily. Make sure you have enough disk space.
298
- EOM
299
-
321
+ EOM
300
322
  print confirmation_text
301
323
 
302
324
  if ask_yes_no('Start Sync?')
303
- Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
325
+ puts "Checking database size(not same as mysqldump data size)..."
326
+ db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
327
+ puts " -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
304
328
  puts "Exporting data from database..."
305
- Flydata::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference']).dump(fp)
329
+ if file_dump
330
+ Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
331
+ Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
332
+ new(de['mysql_data_entry_preference']).dump(fp)
333
+ puts " -> Done"
334
+ call_block_or_return_io(fp, &block)
335
+ else
336
+ Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
337
+ new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
338
+ end
306
339
  else
307
340
  newline
308
341
  puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
@@ -311,8 +344,23 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
311
344
  puts
312
345
  return nil
313
346
  end
314
- puts " -> Done"
315
- fp
347
+ end
348
+
349
+ def call_block_or_return_io(fp, &block)
350
+ if block
351
+ f_io = open_file_io(fp)
352
+ begin
353
+ block.call(f_io)
354
+ return nil
355
+ ensure
356
+ f_io.close rescue nil
357
+ end
358
+ end
359
+ return open_file_io(fp)
360
+ end
361
+
362
+ def open_file_io(file_path)
363
+ File.open(file_path, 'r', encoding: "utf-8")
316
364
  end
317
365
 
318
366
  # Checkpoint
@@ -335,7 +383,7 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
335
383
  # <- checkpoint
336
384
  #...
337
385
  #CREATE TABLE ...
338
- def parse_mysqldump_and_send(dp, de, sync_fm)
386
+ def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize = nil)
339
387
  # Prepare forwarder
340
388
  de_tag_name = de["tag_name#{env_suffix}"]
341
389
  server_port = dp['server_port']
@@ -351,7 +399,7 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
351
399
  # Load dump.pos file for resume
352
400
  dump_pos_info = sync_fm.load_dump_pos
353
401
  option = dump_pos_info || {}
354
- if option[:table_name]
402
+ if option[:table_name] && option[:last_pos].to_i != -1
355
403
  puts "Resuming... Last processed table: #{option[:table_name]}"
356
404
  else
357
405
  #If its a new sync, ensure server side resources are clean
@@ -364,31 +412,14 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
364
412
  # Start parsing dump file
365
413
  tmp_num_inserted_record = 0
366
414
  dump_fp = sync_fm.dump_file_path
367
- dump_file_size = File.size(dump_fp)
368
- binlog_pos = Flydata::Mysql::MysqlDumpParser.new(dump_fp, option).parse(
415
+ dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
416
+ binlog_pos = Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
417
+ mysqldump_io,
369
418
  # create table
370
419
  Proc.new { |mysql_table|
371
- redshift_table = Flydata::Mysql::RedshiftTableAdapter.new(mysql_table)
372
- mysql_table.set_adapter(:redshift, redshift_table)
373
-
374
420
  tmp_num_inserted_record = 0
375
-
376
- if CREATE_TABLE_OPTION
377
- print "- Creating table: #{redshift_table.table_name}"
378
- sql = redshift_table.create_table_sql
379
- ret = flydata.redshift_cluster.run_query(sql)
380
- if ret['message'].index('ERROR:')
381
- if ret['message'].index('already exists')
382
- puts " -> Skip"
383
- else
384
- raise "Failed to create table. error=#{ret['message']}"
385
- end
386
- else
387
- puts " -> OK"
388
- end
389
- end
390
-
391
421
  # dump mysql_table for resume
422
+ #TODO: make it option
392
423
  sync_fm.save_mysql_table_marshal_dump(mysql_table)
393
424
  },
394
425
  # insert record
@@ -403,14 +434,18 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
403
434
  ret
404
435
  },
405
436
  # checkpoint
406
- Proc.new { |mysql_table, last_pos, binlog_pos, state, substate|
437
+ Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
407
438
  # flush if buffer records exist
408
439
  if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
409
440
  forwarder.flush # send buffer data to the server before checkpoint
410
441
  end
411
442
 
412
443
  # show the current progress
413
- puts " -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
444
+ if last_pos.to_i == -1 # stream dump
445
+ puts " -> #{as_size(bytesize)} (#{bytesize} byte) completed..."
446
+ else
447
+ puts " -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
448
+ end
414
449
 
415
450
  # save check point
416
451
  table_name = mysql_table.nil? ? '' : mysql_table.table_name
@@ -418,21 +453,28 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
418
453
  }
419
454
  )
420
455
  forwarder.close
421
-
422
456
  puts " -> Done"
457
+ sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
423
458
 
424
459
  if ENV['FLYDATA_BENCHMARK']
425
460
  bench_end_time = Time.now
426
461
  elapsed_time = bench_end_time.to_i - bench_start_time.to_i
427
462
  puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
428
- return true
429
463
  end
430
- # wait until finish
431
- wait_for_server_data_processing
464
+ end
465
+
466
+ def wait_for_mysqldump_processed(dp, de, sync_fm)
467
+ return if ENV['FLYDATA_BENCHMARK']
432
468
 
433
- sync_fm.save_dump_pos(STATUS_COMPLETE, '', dump_file_size, binlog_pos)
469
+ # Status is not waiting -> skip waiting
470
+ dump_pos_info = sync_fm.load_dump_pos
471
+ return unless dump_pos_info[:status] == STATUS_WAITING
472
+ binlog_pos = dump_pos_info[:binlog_pos]
473
+
474
+ wait_for_server_data_processing
434
475
  tables = de['mysql_data_entry_preference']['tables'].split(',').join(' ')
435
476
  sync_fm.save_table_binlog_pos(tables, binlog_pos)
477
+ sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
436
478
  end
437
479
 
438
480
  ALL_DONE_MESSAGE_TEMPLATE = <<-EOM
@@ -450,6 +492,7 @@ What's next?
450
492
 
451
493
  Thank you for using FlyData!
452
494
  EOM
495
+
453
496
  def complete
454
497
  de = load_sync_info(retrieve_data_entry)
455
498
  sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
@@ -465,8 +508,7 @@ Thank you for using FlyData!
465
508
  Flydata::Command::Sender.new.start(quiet: true)
466
509
  puts " -> Done"
467
510
 
468
- data_port = flydata.data_port.get
469
- dashboard_url = "#{flydata.flydata_api_host}/data_ports/#{data_port['id']}"
511
+ dashboard_url = "#{flydata.flydata_api_host}/dashboard"
470
512
  redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
471
513
  last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
472
514
  puts last_message
@@ -509,945 +551,4 @@ Thank you for using FlyData!
509
551
  end
510
552
  end
511
553
  end
512
-
513
- module Output
514
- class ForwarderFactory
515
-
516
- def self.create(forwarder_key, tag, servers, options = {})
517
- case forwarder_key
518
- when nil, "tcpforwarder"
519
- puts "Creating TCP connection" if FLYDATA_DEBUG
520
- forward = TcpForwarder.new(tag, servers, options)
521
- when "sslforwarder"
522
- puts "Creating SSL connection" if FLYDATA_DEBUG
523
- forward = SslForwarder.new(tag, servers, options)
524
- else
525
- raise "Unsupported Forwarding type #{forwarder_key}"
526
- end
527
- forward
528
- end
529
-
530
- end
531
- class TcpForwarder
532
- FORWARD_HEADER = [0x92].pack('C')
533
- BUFFER_SIZE = 1024 * 1024 * 32 # 32M
534
- DEFUALT_SEND_TIMEOUT = 60 # 1 minute
535
- RETRY_INTERVAL = 2
536
- RETRY_LIMIT = 10
537
-
538
- def initialize(tag, servers, options = {})
539
- @tag = tag
540
- unless servers and servers.kind_of?(Array) and not servers.empty?
541
- raise "Servers must not be empty."
542
- end
543
- @servers = servers
544
- @server_index = 0
545
- set_options(options)
546
- reset
547
- end
548
-
549
- def set_options(options)
550
- if options[:buffer_size_limit]
551
- @buffer_size_limit = options[:buffer_size_limit]
552
- else
553
- @buffer_size_limit = BUFFER_SIZE
554
- end
555
- end
556
-
557
- attr_reader :buffer_record_count, :buffer_size
558
-
559
- def emit(records, time = Time.now.to_i)
560
- records = [records] unless records.kind_of?(Array)
561
- records.each do |record|
562
- event_data = [time,record].to_msgpack
563
- @buffer_records << event_data
564
- @buffer_record_count += 1
565
- @buffer_size += event_data.bytesize
566
- end
567
- if @buffer_size > @buffer_size_limit
568
- send
569
- else
570
- false
571
- end
572
- end
573
-
574
- #TODO retry logic
575
- def send
576
- if @buffer_size > 0
577
- else
578
- return false
579
- end
580
- if ENV['FLYDATA_BENCHMARK']
581
- reset
582
- return true
583
- end
584
- sock = nil
585
- retry_count = 0
586
- begin
587
- sock = connect(pickup_server)
588
-
589
- # Write header
590
- sock.write FORWARD_HEADER
591
- # Write tag
592
- sock.write @tag.to_msgpack
593
- # Write records
594
- sock.write [0xdb, @buffer_records.bytesize].pack('CN')
595
- StringIO.open(@buffer_records) do |i|
596
- FileUtils.copy_stream(i, sock)
597
- end
598
- rescue => e
599
- retry_count += 1
600
- if retry_count > RETRY_LIMIT
601
- puts "! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}"
602
- raise e
603
- end
604
- puts "! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}"
605
- wait_time = RETRY_INTERVAL ** retry_count
606
- puts " Now waiting for next retry. time=#{wait_time}sec"
607
- sleep wait_time
608
- retry
609
- ensure
610
- if sock
611
- sock.close rescue nil
612
- end
613
- end
614
- reset
615
- true
616
- end
617
-
618
- #TODO: Check server status
619
- def pickup_server
620
- ret_server = @servers[@server_index]
621
- @server_index += 1
622
- if @server_index >= (@servers.count)
623
- @server_index = 0
624
- end
625
- ret_server
626
- end
627
-
628
- def connect(server)
629
- host, port = server.split(':')
630
- sock = TCPSocket.new(host, port.to_i)
631
-
632
- # Set options
633
- opt = [1, DEFUALT_SEND_TIMEOUT].pack('I!I!')
634
- sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
635
- opt = [DEFUALT_SEND_TIMEOUT, 0].pack('L!L!')
636
- sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
637
-
638
- sock
639
- end
640
-
641
- def reset
642
- @buffer_records = ''
643
- @buffer_record_count = 0
644
- @buffer_size = 0
645
- end
646
-
647
- def flush
648
- send
649
- end
650
-
651
- def close
652
- flush
653
- end
654
- end
655
- class SslForwarder < TcpForwarder
656
- def connect(server)
657
- tcp_sock = super
658
- ssl_ctx = ssl_ctx_with_verification
659
- ssl_sock = OpenSSL::SSL::SSLSocket.new(tcp_sock, ssl_ctx)
660
- ssl_sock.sync_close = true
661
- ssl_sock.connect
662
- ssl_sock
663
- end
664
-
665
- private
666
- def ssl_ctx_with_verification
667
- cert_store = OpenSSL::X509::Store.new
668
- cert_store.set_default_paths
669
- ssl_ctx = OpenSSL::SSL::SSLContext.new
670
- ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
671
- ssl_ctx.cert_store = cert_store
672
- ssl_ctx
673
- end
674
- end
675
- end
676
-
677
- module Redshift
678
- module Util
679
- MAX_TABLENAME_LENGTH = 127
680
- REDSHIFT_RESERVED_WORDS = %w[
681
- aes128 aes256 all allowoverwrite analyse analyze and any array
682
- as asc authorization backup between binary blanksasnull both
683
- bytedict case cast check collate column constraint create
684
- credentials cross current_date current_time current_timestamp
685
- current_user current_user_id default deferrable deflate defrag
686
- delta delta32k desc disable distinct do else emptyasnull enable
687
- encode encrypt encryption end except explicit false for foreign
688
- freeze from full globaldict256 globaldict64k grant group gzip having
689
- identity ignore ilike in initially inner intersect into is isnull
690
- join leading left like limit localtime localtimestamp lun luns
691
- minus mostly13 mostly32 mostly8 natural new not notnull null nulls
692
- off offline offset old on only open or order outer overlaps parallel
693
- partition percent placing primary raw readratio recover references
694
- rejectlog resort restore right select session_user similar some
695
- sysdate system table tag tdes text255 text32k then to top trailing
696
- true truncatecolumns union unique user using verbose wallet when
697
- where with without]
698
- # Create a symbol-keyed hash for performance
699
- REDSHIFT_RESERVED_WORDS_HASH = REDSHIFT_RESERVED_WORDS.inject({}) {|h, word| h[word.to_sym] = true; h}
700
-
701
- REDSHIFT_SYSTEM_COLUMNS = %w[oid tableoid xmin cmin xmax cmax ctid]
702
- REDSHIFT_SYSTEM_COLUMNS_HASH = REDSHIFT_SYSTEM_COLUMNS.inject({}) {|h, word| h[word.to_sym] = true; h}
703
-
704
- def convert_to_valid_name(key, type = :table)
705
- @memo ||= { table:{}, column:{} }
706
- key_sym = key.to_sym
707
- return @memo[type][key_sym] if @memo[type][key_sym]
708
-
709
- name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
710
- name = "_#{name}" if is_redshift_reserved_word?(name, type) or name =~ /^[0-9$]/
711
- if name.length > MAX_TABLENAME_LENGTH
712
- name = nil
713
- end
714
- @memo[key_sym] = name
715
- name
716
- end
717
-
718
- def is_redshift_reserved_word?(name, type = :table)
719
- return false unless name
720
- return true if REDSHIFT_RESERVED_WORDS_HASH[name.to_sym] == true
721
-
722
- case type
723
- when :table
724
- false
725
- when :column
726
- REDSHIFT_SYSTEM_COLUMNS_HASH[name.to_sym] == true
727
- else
728
- false
729
- end
730
- end
731
- end
732
- end
733
-
734
- module Mysql
735
- class MysqlTable
736
- def initialize(table_name, columns = {}, primary_keys = [])
737
- @table_name = table_name
738
- @columns = columns
739
- @primary_keys = primary_keys
740
- @adapters = {}
741
- end
742
-
743
- attr_accessor :table_name, :columns, :primary_keys
744
-
745
- def add_column(column)
746
- @columns[column[:column_name]] = column
747
- end
748
-
749
- def set_adapter(key, adapter)
750
- @adapters[key] = adapter
751
- end
752
-
753
- def adapter(key)
754
- @adapters[key]
755
- end
756
- end
757
-
758
- class RedshiftTableAdapter
759
- include Flydata::Redshift::Util
760
- def initialize(mysql_table)
761
- @table_name = convert_to_valid_name(mysql_table.table_name)
762
- set_columns(mysql_table.columns)
763
- @primary_keys = mysql_table.primary_keys
764
- end
765
-
766
- attr_reader :table_name, :columns, :primary_keys
767
-
768
- def create_table_sql
769
- col_def = @columns.inject([]) { |list, (cn, column)|
770
- list << build_column_def(column)
771
- list
772
- }
773
- if @primary_keys.count > 0
774
- col_def << "primary key (#{@primary_keys.join(',')})"
775
- end
776
- <<EOT
777
- CREATE TABLE #{@table_name} (#{col_def.join(',')});
778
- EOT
779
- end
780
-
781
- private
782
-
783
- def set_columns(columns)
784
- @columns = {}
785
- columns.each do |k, column|
786
- new_k = convert_to_valid_name(k, :column)
787
- new_column = column.dup
788
- new_column[:column_name] = new_k
789
- @columns[new_k] = convert_column_format_type(new_column)
790
- end
791
- end
792
-
793
- # Mysql Field Types
794
- # http://help.scibit.com/mascon/masconMySQL_Field_Types.html
795
- def convert_column_format_type(column)
796
- ret_c = {}.merge(column)
797
- ret_c.delete(:format_type_str)
798
- ret_c[:format_type] = case column[:format_type]
799
- when 'tinyint'
800
- 'smallint'
801
- when 'smallint'
802
- column[:unsigned] ? 'integer' : 'smallint'
803
- when 'mediumint'
804
- 'integer'
805
- when 'int', 'integer'
806
- column[:unsigned] ? 'bigint' : 'integer'
807
- when 'bigint'
808
- # max unsigned bigint is 18446744073709551615
809
- column[:unsigned] ? 'decimal(20,0)' : 'bigint'
810
- when 'float'
811
- 'real'
812
- when 'double', 'double precision', 'real'
813
- 'double precision'
814
- when 'decimal', 'numeric'
815
- ret_c[:format_type_str] = "decimal(#{column[:decimal_precision]},#{column[:decimal_scale]})"
816
- 'decimal'
817
- when 'date'
818
- 'date'
819
- when 'datetime'
820
- 'timestamp'
821
- when 'time'
822
- 'timestamp' #TODO: redshift does not support time only column type
823
- when 'year'
824
- 'smallint'
825
- when 'char'
826
- ret_c[:format_type_str] = "char(#{column[:format_size]})"
827
- 'char'
828
- when 'varchar'
829
- ret_c[:format_type_str] = "varchar(#{column[:format_size]})"
830
- 'varchar'
831
- when 'tinyblob','tinytext'
832
- ret_c[:format_size] = 255
833
- ret_c[:format_type_str] = "varchar(#{ret_c[:format_size]})"
834
- 'varchar'
835
- when 'blob','text', 'mediumblob', 'mediumtext', 'longblob', 'longtext'
836
- ret_c[:format_size] = 65535 #TODO: review
837
- ret_c[:format_type_str] = "varchar(#{ret_c[:format_size]})"
838
- 'varchar'
839
- else
840
- #TODO: discuss
841
- 'varchar'
842
- end
843
- ret_c
844
- end
845
-
846
- def build_column_def(column)
847
- format_type = column[:format_type]
848
- format_type = column[:format_type_str] if column[:format_type_str]
849
- def_str = "#{column[:column_name]} #{format_type}"
850
- if column[:not_null]
851
- def_str << " not null"
852
- elsif column.has_key?(:default)
853
- val = column[:default]
854
- val = val.nil? ? 'null' : "'#{val}'"
855
- def_str << " default #{val}"
856
- end
857
- def_str
858
- end
859
-
860
- end
861
-
862
- class MysqlDumpGenerator
863
- # host, port, username, password, database, tables
864
- MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
865
- EXTRA_MYSQLDUMP_PARAMS = ""
866
- def initialize(conf)
867
- password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
868
- tables = if conf['tables']
869
- conf['tables'].split(',').join(' ')
870
- else
871
- ''
872
- end
873
- @dump_cmd = MYSQL_DUMP_CMD_TEMPLATE %
874
- [conf['host'], conf['port'], conf['username'], password, self.class::EXTRA_MYSQLDUMP_PARAMS, conf['database'], tables]
875
- @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
876
- end
877
- def dump(file_path)
878
- raise "subclass must implement the method"
879
- end
880
- end
881
-
882
- class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
883
- EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
884
- def dump(file_path)
885
- cmd = "#{@dump_cmd} -r #{file_path}"
886
- o, e, s = Open3.capture3(cmd)
887
- e.to_s.each_line {|l| puts l unless /^Warning:/ =~ l } unless e.to_s.empty?
888
- unless s.exitstatus == 0
889
- if File.exists?(file_path)
890
- File.open(file_path, 'r') {|f| f.each_line{|l| puts l}}
891
- FileUtils.rm(file_path)
892
- end
893
- raise "Failed to run mysqldump command."
894
- end
895
- unless File.exists?(file_path)
896
- raise "mysqldump file does not exist. Something wrong..."
897
- end
898
- if File.size(file_path) == 0
899
- raise "mysqldump file is empty. Something wrong..."
900
- end
901
- true
902
- end
903
- end
904
-
905
- class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
906
- EXTRA_MYSQLDUMP_PARAMS = ""
907
- CHANGE_MASTER_TEMPLATE = <<EOS
908
- --
909
- -- Position to start replication or point-in-time recovery from
910
- --
911
-
912
- -- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
913
-
914
- EOS
915
-
916
- def dump(file_path)
917
- # RDS doesn't allow obtaining binlog position using mysqldump. Get it separately and insert it into the dump file.
918
- table_locker = Fiber.new do
919
- client = Mysql2::Client.new(@db_opts)
920
- # Lock tables
921
- client.query "FLUSH LOCAL TABLES;"
922
- q = flush_tables_with_read_lock_query(client)
923
- puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
924
- client.query q
925
- begin
926
- Fiber.yield # Lock is done. Let dump to start
927
- # obtain binlog pos
928
- result = client.query "SHOW MASTER STATUS;"
929
- row = result.first
930
- if row.nil?
931
- raise "MySQL DB has no replication master status. Check if the DB is set up as a replication master. In case of RDS, make sure that Backup Retention Period is set to more than 0."
932
- end
933
- ensure
934
- # unlock tables
935
- client.query "UNLOCK TABLES;"
936
- client.close
937
- end
938
-
939
- [row["File"], row['Position']]
940
- end
941
-
942
- table_locker.resume # Lock tables
943
- begin
944
- # start dump
945
- Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
946
- cmd_in.close_write
947
- cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
948
- File.open(file_path, "w", encoding: "utf-8") do |f|
949
- find_insert_pos = :not_started
950
- cmd_out.each_line do |line|
951
- if find_insert_pos == :not_started && /^-- Server version/ === line
952
- find_insert_pos = :finding
953
- elsif find_insert_pos == :finding && /^--/ === line
954
- # wait before writing the first database queries
955
- file, pos = table_locker.resume # Get binlog pos
956
- # insert binlog pos
957
- change_master = CHANGE_MASTER_TEMPLATE % [file, pos]
958
- f.print change_master
959
-
960
- find_insert_pos = :found
961
- # resume dump
962
- end
963
- f.print line
964
- end
965
- end
966
- cmd_err.each_line do |line|
967
- $stderr.print line unless /^Warning:/ === line
968
- end
969
- end
970
- rescue
971
- # Cleanup
972
- FileUtils.rm(file_path) if File.exists?(file_path)
973
- raise
974
- ensure
975
- # Let table_locker finish its task even if an exception happened
976
- table_locker.resume if table_locker.alive?
977
- end
978
- end
979
-
980
- private
981
- # This query generates a query which flushes user tables with read lock
982
- FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
983
- USER_TABLES_QUERY = <<EOS
984
- SELECT CONCAT('`',
985
- REPLACE(TABLE_SCHEMA, '`', '``'), '`.`',
986
- REPLACE(TABLE_NAME, '`', '``'), '` ')
987
- AS tables
988
- FROM INFORMATION_SCHEMA.TABLES
989
- WHERE TABLE_TYPE = 'BASE TABLE'
990
- AND ENGINE NOT IN ('MEMORY', 'CSV', 'PERFORMANCE_SCHEMA');
991
- EOS
992
- def flush_tables_with_read_lock_query(client)
993
- tables = ""
994
- if mysql_server_version(client) >= "5.5"
995
- # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
996
- result = client.query(USER_TABLES_QUERY)
997
- tables = result.collect{|r| r['tables']}.join(", ")
998
- end
999
- FLUSH_TABLES_QUERY_TEMPLATE % [tables]
1000
- end
1001
-
1002
- VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
1003
- def mysql_server_version(client)
1004
- result = client.query(VERSION_QUERY)
1005
- result.first['Value']
1006
- end
1007
- end
1008
-
1009
- class MysqlDumpParser
1010
-
1011
- module State
1012
- START = 'START'
1013
- CREATE_TABLE = 'CREATE_TABLE'
1014
- CREATE_TABLE_COLUMNS = 'CREATE_TABLE_COLUMNS'
1015
- CREATE_TABLE_CONSTRAINTS = 'CREATE_TABLE_CONSTRAINTS'
1016
- INSERT_RECORD = 'INSERT_RECORD'
1017
- PARSING_INSERT_RECORD = 'PARSING_INSERT_RECORD'
1018
- end
1019
-
1020
- attr_accessor :binlog_pos
1021
-
1022
- def initialize(file_path, option = {})
1023
- @file_path = file_path
1024
- raise "Dump file does not exist. file_path:#{file_path}" unless File.exist?(file_path)
1025
- @binlog_pos = option[:binlog_pos]
1026
- @option = option
1027
- end
1028
-
1029
- def parse(create_table_block, insert_record_block, check_point_block)
1030
- invalid_file = false
1031
- current_state = State::START
1032
- substate = nil
1033
-
1034
- state_start = Proc.new do |f|
1035
- line = f.readline.strip
1036
- # -- CHANGE MASTER TO MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=120;
1037
- m = /^\-\- CHANGE MASTER TO MASTER_LOG_FILE='(?<binfile>[^']+)', MASTER_LOG_POS=(?<pos>\d+)/.match(line)
1038
- if m
1039
- @binlog_pos = {binfile: m[:binfile], pos: m[:pos].to_i}
1040
- current_state = State::CREATE_TABLE
1041
- check_point_block.call(nil, f.pos, @binlog_pos, current_state)
1042
- end
1043
- end
1044
-
1045
- current_table = nil
1046
- state_create_table = Proc.new do |f|
1047
- line = f.readline.strip
1048
- # CREATE TABLE `active_admin_comments` (
1049
- m = /^CREATE TABLE `(?<table_name>[^`]+)`/.match(line)
1050
- if m
1051
- current_table = MysqlTable.new(m[:table_name])
1052
- current_state = State::CREATE_TABLE_COLUMNS
1053
- end
1054
- end
1055
-
1056
- state_create_table_constraints = Proc.new do |f|
1057
- line = f.readline.strip
1058
- # PRIMARY KEY (`id`),
1059
- if line.start_with?(')')
1060
- create_table_block.call(current_table)
1061
- current_state = State::INSERT_RECORD
1062
- check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
1063
- elsif m = /^PRIMARY KEY \((?<primary_keys>[^\)]+)\)/.match(line)
1064
- current_table.primary_keys = m[:primary_keys].split(',').collect do |pk_str|
1065
- pk_str[1..-2]
1066
- end
1067
- end
1068
- end
1069
-
1070
- state_create_table_columns = Proc.new do |f|
1071
- start_pos = f.pos
1072
- line = f.readline.strip
1073
- # `author_type` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
1074
- if line.start_with?("\`")
1075
- column = {}
1076
-
1077
- # parse column line
1078
- line = line[0..-2] if line.end_with?(',')
1079
- items = line.split
1080
- column[:column_name] = items.shift[1..-2]
1081
- column[:format_type_str] = format_type_str = items.shift
1082
- pos = format_type_str.index('(')
1083
- if pos
1084
- ft = column[:format_type] = format_type_str[0..pos-1]
1085
- if ft == 'decimal'
1086
- precision, scale = format_type_str[pos+1..-2].split(',').collect{|v| v.to_i}
1087
- column[:decimal_precision] = precision
1088
- column[:decimal_scale] = scale
1089
- else
1090
- column[:format_size] = format_type_str[pos+1..-2].to_i
1091
- end
1092
- else
1093
- column[:format_type] = format_type_str
1094
- end
1095
- while (item = items.shift) do
1096
- case item
1097
- when 'DEFAULT'
1098
- value = items.shift
1099
- value = value.start_with?('\'') ? value[1..-2] : value
1100
- value = nil if value == 'NULL'
1101
- column[:default] = value
1102
- when 'NOT'
1103
- if items[1] == 'NULL'
1104
- items.shift
1105
- column[:not_null] = true
1106
- end
1107
- when 'unsigned'
1108
- column[:unsigned] = true
1109
- else
1110
- #ignore other options
1111
- end
1112
- end
1113
-
1114
- current_table.add_column(column)
1115
- else
1116
- current_state = State::CREATE_TABLE_CONSTRAINTS
1117
- f.pos = start_pos
1118
- state_create_table_constraints.call(f)
1119
- end
1120
- end
1121
-
1122
- state_insert_record = Proc.new do |f|
1123
- original_pos = f.pos
1124
- command = f.read(6)
1125
- if command == 'INSERT'
1126
- current_state = State::PARSING_INSERT_RECORD
1127
- else
1128
- f.pos = original_pos
1129
- f.readline
1130
- if command == 'UNLOCK'
1131
- current_state = State::CREATE_TABLE
1132
- check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
1133
- end
1134
- end
1135
- end
1136
-
1137
- state_parsing_insert_record = Proc.new do |f|
1138
- values_set = InsertParser.new(f).parse
1139
- current_state = State::INSERT_RECORD
1140
- if insert_record_block.call(current_table, values_set)
1141
- check_point_block.call(current_table, f.pos, @binlog_pos, current_state)
1142
- end
1143
- end
1144
-
1145
- # Start reading file from top
1146
- File.open(@file_path, 'r', encoding: "utf-8") do |f|
1147
- last_saved_pos = 0
1148
-
1149
- # resume
1150
- if @option[:last_pos]
1151
- f.pos = @option[:last_pos].to_i
1152
- current_state = @option[:state]
1153
- substate = @option[:substate]
1154
- current_table = @option[:mysql_table]
1155
- end
1156
-
1157
- until f.eof? do
1158
- case current_state
1159
- when State::START
1160
- state_start.call(f)
1161
- when State::CREATE_TABLE
1162
- state_create_table.call(f)
1163
- when State::CREATE_TABLE_COLUMNS
1164
- state_create_table_columns.call(f)
1165
- when State::CREATE_TABLE_CONSTRAINTS
1166
- state_create_table_constraints.call(f)
1167
- when State::INSERT_RECORD
1168
- state_insert_record.call(f)
1169
- when State::PARSING_INSERT_RECORD
1170
- state_parsing_insert_record.call(f)
1171
- end
1172
- end
1173
- end
1174
- @binlog_pos
1175
- end
1176
-
1177
- # Parse the insert line containing multiple values. (max line size is 1kb)
1178
- # ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
1179
- class InsertParser
1180
- #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
1181
- module State
1182
- IN_VALUE = 'IN_VALUE'
1183
- NEXT_VALUES = 'NEXT_VALUES'
1184
- end
1185
-
1186
- def initialize(file)
1187
- @file = file
1188
- @values = []
1189
- @values_set = []
1190
- end
1191
-
1192
- def start_ruby_prof
1193
- RubyProf.start if defined?(RubyProf) and not RubyProf.running?
1194
- end
1195
-
1196
- def stop_ruby_prof
1197
- if defined?(RubyProf) and RubyProf.running?
1198
- result = RubyProf.stop
1199
- #printer = RubyProf::GraphPrinter.new(result)
1200
- printer = RubyProf::GraphHtmlPrinter.new(result)
1201
- #printer.print(STDOUT)
1202
- printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
1203
- end
1204
- end
1205
-
1206
- def parse
1207
- start_ruby_prof
1208
- bench_start_time = Time.now
1209
- target_line = @file.readline
1210
- _parse(target_line)
1211
- ensure
1212
- stop_ruby_prof
1213
- if ENV['FLYDATA_BENCHMARK']
1214
- puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
1215
- end
1216
- end
1217
-
1218
- private
1219
-
1220
- def _parse(target_line)
1221
- target_line = target_line.strip
1222
- start_index = target_line.index('(')
1223
- target_line = target_line[start_index..-2]
1224
-
1225
- # Split insert line text with ',' and take care of ',' inside of the values later.
1226
- #
1227
- # We are using the C native method that is like 'split', 'start_with?', 'regexp'
1228
- # instead of 'String#each_char' and string comparision for the performance.
1229
- # 'String#each_char' is twice as slow as the current storategy.
1230
- items = target_line.split(',')
1231
- index = 0
1232
- cur_state = State::NEXT_VALUES
1233
-
1234
- loop do
1235
- case cur_state
1236
- when State::NEXT_VALUES
1237
- chars = items[index]
1238
- break unless chars
1239
- items[index] = chars[1..-1]
1240
- cur_state = State::IN_VALUE
1241
- when State::IN_VALUE
1242
- chars = items[index]
1243
- index += 1
1244
- if chars.start_with?("'")
1245
- # single item (not last item)
1246
- # size check added below otherwise end_with? matches the single quote which was also used by start_with?
1247
- if chars.size > 1 and chars.end_with?("'") and !last_char_escaped?(chars)
1248
- @values << replace_escape_char(chars[1..-2])
1249
- # single item (last item)
1250
- # size check added below otherwise end_with? matches the single quote which was also used by start_with?
1251
- elsif chars.size > 2 and chars.end_with?("')") and !last_char_escaped?(chars[0..-2])
1252
- @values << replace_escape_char(chars[1..-3])
1253
- @values_set << @values
1254
- @values = []
1255
- cur_state = State::NEXT_VALUES
1256
- # multi items
1257
- else
1258
- cur_value = chars[1..-1]
1259
- loop do
1260
- next_chars = items[index]
1261
- index += 1
1262
- if next_chars.end_with?('\'') and !last_char_escaped?(next_chars)
1263
- cur_value << ','
1264
- cur_value << next_chars[0..-2]
1265
- @values << replace_escape_char(cur_value)
1266
- break
1267
- elsif next_chars.end_with?("')") and !last_char_escaped?(next_chars[0..-2])
1268
- cur_value << ','
1269
- cur_value << next_chars[0..-3]
1270
- @values << replace_escape_char(cur_value)
1271
- @values_set << @values
1272
- @values = []
1273
- cur_state = State::NEXT_VALUES
1274
- break
1275
- else
1276
- cur_value << ','
1277
- cur_value << next_chars
1278
- end
1279
- end
1280
- end
1281
- else
1282
- if chars.end_with?(')')
1283
- chars = chars[0..-2]
1284
- @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
1285
- @values_set << @values
1286
- @values = []
1287
- cur_state = State::NEXT_VALUES
1288
- else
1289
- @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
1290
- end
1291
- end
1292
- else
1293
- raise "Invalid state: #{cur_state}"
1294
- end
1295
- end
1296
- return @values_set
1297
- end
1298
-
1299
- ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
1300
-
1301
- def replace_escape_char(original)
1302
- original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
1303
- end
1304
-
1305
- # This method assume that the last character is '(single quotation)
1306
- # abcd\' -> true
1307
- # abcd\\' -> false (back slash escape back slash)
1308
- # abcd\\\' -> true
1309
- def last_char_escaped?(text)
1310
- flag = false
1311
- (text.length - 2).downto(0) do |i|
1312
- if text[i] == '\\'
1313
- flag = !flag
1314
- else
1315
- break
1316
- end
1317
- end
1318
- flag
1319
- end
1320
-
1321
- def remove_leading_zeros(number_string)
1322
- if number_string.start_with?('0')
1323
- number_string.sub(/^0*([1-9][0-9]*(\.\d*)?|0(\.\d*)?)$/,'\1')
1324
- else
1325
- number_string
1326
- end
1327
- end
1328
- end
1329
- end
1330
- class CompatibilityCheck
1331
-
1332
- class CompatibilityError < StandardError
1333
- end
1334
-
1335
- SELECT_QUERY_TMPLT = "SELECT %s"
1336
-
1337
- def initialize(de_hash, dump_dir=nil)
1338
- @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
1339
- @dump_dir = dump_dir
1340
- @errors=[]
1341
- end
1342
-
1343
- def check
1344
- self.methods.grep(/^check_/).each do |m|
1345
- begin
1346
- send(m)
1347
- rescue CompatibilityError => e
1348
- @errors << e
1349
- end
1350
- end
1351
- print_errors
1352
- end
1353
-
1354
- def print_errors
1355
- return if @errors.empty?
1356
- puts "There may be some compatibility issues with your MySQL credentials: "
1357
- @errors.each do |error|
1358
- puts " * #{error.message}"
1359
- end
1360
- raise "Please correct these errors if you wish to run FlyData Sync"
1361
- end
1362
-
1363
- def check_mysql_user_compat
1364
- client = Mysql2::Client.new(@db_opts)
1365
- grants_sql = "SHOW GRANTS"
1366
- correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
1367
- necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
1368
- all_privileges_field= ["ALL PRIVILEGES"]
1369
- result = client.query(grants_sql)
1370
- # Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
1371
- client.close
1372
- missing_priv = []
1373
- result.each do |res|
1374
- # SHOW GRANTS should only return one column
1375
- res_value = res.values.first
1376
- if correct_db.all? {|perm| res_value.match(perm)}
1377
- necessary_permission_fields.each do |priv|
1378
- missing_priv << priv unless res_value.match(priv)
1379
- end
1380
- return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
1381
- end
1382
- end
1383
- raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
1384
- end
1385
-
1386
- def check_mysql_protocol_tcp_compat
1387
- query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
1388
- query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
1389
-
1390
- Open3.popen3(query) do |stdin, stdout, stderr|
1391
- stdin.close
1392
- while !stderr.eof?
1393
- line = stderr.gets
1394
- unless /Warning: Using a password on the command line interface can be insecure./ === line
1395
- raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
1396
- end
1397
- end
1398
- end
1399
- end
1400
-
1401
- def check_mysql_row_mode_compat
1402
- sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
1403
- errors={}
1404
-
1405
- client = Mysql2::Client.new(@db_opts)
1406
-
1407
- begin
1408
- sys_var_to_check.each_key do |sys_var|
1409
- sel_query = SELECT_QUERY_TMPLT % sys_var
1410
- begin
1411
- result = client.query(sel_query)
1412
- unless result.first[sys_var] == sys_var_to_check[sys_var]
1413
- errors[sys_var]=result.first[sys_var]
1414
- end
1415
- rescue Mysql2::Error => e
1416
- if e.message =~ /Unknown system variable/
1417
- unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
1418
- errors[sys_var] = false
1419
- end
1420
- else
1421
- raise e
1422
- end
1423
- end
1424
- end
1425
- ensure
1426
- client.close
1427
- end
1428
- unless errors.empty?
1429
- error_explanation = ""
1430
- errors.each_key do |err_key|
1431
- error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
1432
- end
1433
- raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
1434
- end
1435
- end
1436
-
1437
- def check_writing_permissions
1438
- write_errors = []
1439
- paths_to_check = ["~/.flydata"]
1440
- paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
1441
- paths_to_check.each do |path|
1442
- full_path = File.expand_path(path)
1443
- full_path = File.dirname(full_path) unless File.directory?(full_path)
1444
- write_errors << full_path unless File.writable?(full_path)
1445
- end
1446
- unless write_errors.empty?
1447
- error_dir = write_errors.join(", ")
1448
- raise CompatibilityError, "We cannot access the directories: #{error_dir}"
1449
- end
1450
- end
1451
- end
1452
- end
1453
554
  end