flydata 0.6.3 → 0.6.4

data/lib/flydata/source_mysql/mysql_compatibility_check.rb

@@ -0,0 +1,114 @@
+require 'mysql2'
+require 'flydata/compatibility_check'
+require 'flydata-core/mysql/config'
+require 'flydata-core/mysql/command_generator'
+require 'flydata-core/mysql/compatibility_checker'
+require 'flydata-core/errors'
+
+module Flydata
+module SourceMysql
+
+  class MysqlCompatibilityCheck < CompatibilityCheck
+    include InitialSyncChecks
+
+    def initialize(dp_hash, de_hash, options={})
+      super
+      @db_opts = FlydataCore::Mysql::Config.build_mysql_db_opts(de_hash)
+      @dump_dir = options[:dump_dir] || nil
+      @backup_dir = options[:backup_dir] || nil
+      @tables = de_hash['tables']
+    end
+
+    def print_errors
+      return if @errors.empty?
+      log_error_stderr "There may be some compatibility issues with your MySQL credentials: "
+      @errors.each do |error|
+        log_error_stderr "  * #{error.message}"
+      end
+      raise "Please correct these errors if you wish to run FlyData Sync"
+    end
+
+    def check_compatibility56_variable
+      FlydataCore::Mysql::MySqlVersion57CompabilityChecker.new(@db_opts).do_check
+    end
+
+    def check_mysql_user_compat
+      FlydataCore::Mysql::SyncPermissionChecker.new(@db_opts).do_check
+    end
+
+    def check_mysql_protocol_tcp_compat
+      query = FlydataCore::Mysql::CommandGenerator.generate_mysql_show_grants_cmd(@db_opts)
+
+      Open3.popen3(query) do |stdin, stdout, stderr|
+        stdin.close
+        while !stderr.eof?
+          lines = []
+          while line = stderr.gets; lines << line.strip; end
+          err_reason = lines.join(" ")
+          log_error("Error occured during access to mysql server.", {err: err_reason})
+
+          unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
+            raise FlydataCore::MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
+          end
+        end
+      end
+    end
+
+    def check_mysql_parameters_compat
+      begin
+        FlydataCore::Mysql::OptionalBinlogParameterChecker.new(@db_opts).do_check
+      rescue FlydataCore::MysqlCompatibilityError => e
+        log_warn_stderr(e.to_s)
+      end
+      FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
+    end
+
+    def check_rds_master_status
+      if is_rds?
+        FlydataCore::Mysql::RdsMasterStatusChecker.new(@db_opts).do_check
+      end
+    end
+
+    def check_mysql_binlog_retention
+      if is_rds?
+        run_rds_retention_check
+      else
+        run_mysql_retention_check
+      end
+    end
+
+    # If table_type='VIEW' or engine='MEMORY', raise error.
+    def check_mysql_table_types
+      return if @tables.empty?
+      option = @db_opts.dup.merge(tables: @tables)
+      FlydataCore::Mysql::TableTypeChecker.new(option).do_check
+    end
+
+    def run_mysql_retention_check
+      FlydataCore::Mysql::NonRdsRetentionChecker.new(@db_opts).do_check
+    end
+
+    def run_rds_retention_check
+      FlydataCore::Mysql::RdsRetentionChecker.new(@db_opts).do_check
+    rescue Mysql2::Error => e
+      if e.message =~ /command denied to user/
+        retention_hours = FlydataCore::Mysql::RdsRetentionChecker::BINLOG_RETENTION_HOURS
+        log_warn_stderr("[WARNING]Cannot verify RDS retention period on current MySQL user account.\n" +
+                        "To see retention period, please run this on your RDS:\n" +
+                        "  $> call mysql.rds_show_configuration;\n" +
+                        "Please verify that the hours is not nil and is at least #{retention_hours} hours\n" +
+                        "To set binlog retention hours, you can run this on your RDS:\n" +
+                        "  $> call mysql.rds_set_configuration('binlog retention hours', #{retention_hours});\n"
+                       )
+      else
+        raise e
+      end
+    end
+
+    def is_rds?(hostname = @db_opts[:host])
+      hostname.match(/rds.amazonaws.com$/) != nil
+    end
+  end
+
+end
+end

data/lib/flydata/source_mysql/parse_dump_and_send.rb

@@ -0,0 +1,28 @@
+require 'flydata/source/parse_dump_and_send'
+require 'flydata-core/table_def/mysql_table_def'
+require 'flydata/source_mysql/parser/dump_parser'
+require 'flydata-core/mysql/binlog_pos'
+
+module Flydata
+module SourceMysql
+
+class ParseDumpAndSend < Source::ParseDumpAndSend
+  def value_converters
+    FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS
+  end
+
+  def parse_dump(dump_pos_info, dmpio,
+                 create_table_block, insert_record_block, check_point_source_pos_block)
+
+    check_point_binlog_block = Proc.new { |source_table, last_pos, bytesize, binlog_hash, state, substate|
+      source_pos = FlydataCore::Mysql::BinlogPos.new(binlog_hash) # SourcePos object is BinlogPos in SourceMysql
+
+      check_point_source_pos_block.call(source_table, last_pos, bytesize, source_pos, state, substate)
+    }
+    Parser::MysqlDumpParser.new(dump_pos_info).parse(dmpio,
+                 create_table_block, insert_record_block, check_point_binlog_block)
+  end
+end
+
+end
+end

data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb

@@ -1,13 +1,16 @@
 require 'fiber'
 require 'io/wait'
 require 'mysql2'
+require 'flydata/parser'
+require 'flydata/parser/source_table'
 require 'flydata-core/mysql/config'
 require 'flydata-core/mysql/command_generator'
+require 'flydata-core/mysql/binlog_pos'
 #require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
 module Flydata
-  module Parser
-    module Mysql
+  module SourceMysql
+    module Parser
 
       module MysqlAccessible
         def mysql_conf(conf)
@@ -29,38 +32,6 @@ module Flydata
         end
       end
 
-      class MysqlTable
-        def initialize(table_name, columns = {}, primary_keys = [])
-          @table_name = table_name
-          @columns = columns
-          @column_names = columns.collect{|k,v| v[:column_name]}
-          @primary_keys = primary_keys
-          @value_converters = {}
-        end
-
-        attr_accessor :table_name, :columns, :column_names, :primary_keys, :value_converters
-
-        def add_column(column)
-          cn = column[:column_name]
-          @columns[cn] = column
-          @column_names << cn
-        end
-
-        def set_value_converters(converter_hash)
-          @value_converters ||= {}  # for backward compatibility with an old marshal dump object
-          @columns.each_with_index do |(k, v), i|
-            type = v[:format_type]
-            if converter_hash.has_key?(type)
-              @value_converters[i] = converter_hash[type]
-            end
-          end
-        end
-
-        def set_column_names
-          @column_names = columns.collect{|k,v| v[:column_name]}
-        end
-      end
-
       class MysqlDumpGenerator
         def initialize(conf)
           @conf = conf
@@ -289,21 +260,15 @@ EOS
 
       class MysqlDumpParser
 
-        module State
-          START = 'START'
-          CREATE_TABLE = 'CREATE_TABLE'
-          CREATE_TABLE_COLUMNS = 'CREATE_TABLE_COLUMNS'
-          CREATE_TABLE_CONSTRAINTS = 'CREATE_TABLE_CONSTRAINTS'
-          INSERT_RECORD = 'INSERT_RECORD'
-          PARSING_INSERT_RECORD = 'PARSING_INSERT_RECORD'
-        end
-
         attr_accessor :binlog_pos
         BINLOG_INV_ERROR_CHUNK_SIZE = 250
 
-        def initialize(option = {})
-          @binlog_pos = option[:binlog_pos] or raise ArgumentError.new("binlog position is required")
-          @option = option
+        def initialize(dump_pos_info = {})
+          binlog_pos_object = dump_pos_info[:source_pos]
+          raise ArgumentError.new("source position is required") unless binlog_pos_object
+
+          @binlog_pos = { binfile: binlog_pos_object.filename, pos: binlog_pos_object.pos }
+          @dump_pos_info = dump_pos_info
         end
 
         def parse(dmpio, create_table_block, insert_record_block, check_point_block)
@@ -313,7 +278,7 @@ EOS
 
           dump_io = nil
           invalid_file = false
-          current_state = State::START
+          current_state = Flydata::Parser::State::START
           substate = nil
           buffered_line = nil
           bytesize = 0
@@ -336,7 +301,7 @@ EOS
 
             # -- Server version 5.6.21-log
             if line.start_with?('-- Server version')
-              current_state = State::CREATE_TABLE
+              current_state = Flydata::Parser::State::CREATE_TABLE
               check_point_block.call(nil, dump_io.pos, bytesize, @binlog_pos, current_state)
             end
           end
@@ -348,8 +313,8 @@ EOS
             # CREATE TABLE `active_admin_comments` (
             m = /^CREATE TABLE `(?<table_name>[^`]+)`/.match(line)
             if m
-              current_table = MysqlTable.new(m[:table_name])
-              current_state = State::CREATE_TABLE_COLUMNS
+              current_table = Flydata::Parser::SourceTable.new(m[:table_name])
+              current_state = Flydata::Parser::State::CREATE_TABLE_COLUMNS
             end
           end
 
@@ -359,7 +324,7 @@ EOS
             #  PRIMARY KEY (`id`),
             if line.start_with?(')')
               create_table_block.call(current_table)
-              current_state = State::INSERT_RECORD
+              current_state = Flydata::Parser::State::INSERT_RECORD
               check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
             elsif m = /^PRIMARY KEY \((?<primary_keys>[^\)]+)\)/.match(line)
               current_table.primary_keys = m[:primary_keys].split(',').collect do |pk_str|
@@ -417,7 +382,7 @@ EOS
 
               current_table.add_column(column)
             else
-              current_state = State::CREATE_TABLE_CONSTRAINTS
+              current_state = Flydata::Parser::State::CREATE_TABLE_CONSTRAINTS
               buffered_line = line
               state_create_table_constraints.call
             end
@@ -428,9 +393,9 @@ EOS
 
             if line.start_with?('INSERT')
               buffered_line = line
-              current_state = State::PARSING_INSERT_RECORD
+              current_state = Flydata::Parser::State::PARSING_INSERT_RECORD
             elsif line.start_with?('UNLOCK')
-              current_state = State::CREATE_TABLE
+              current_state = Flydata::Parser::State::CREATE_TABLE
               check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
             end
           end
@@ -446,7 +411,7 @@ EOS
               newe.set_backtrace(e.backtrace)
               raise newe
             end
-            current_state = State::INSERT_RECORD
+            current_state = Flydata::Parser::State::INSERT_RECORD
 
             if insert_record_block.call(current_table, values_set)
               values_set = nil
@@ -457,11 +422,11 @@ EOS
           # Start reading file from top
           begin
             # resume(only when using dump file)
-            if @option[:last_pos] && (@option[:last_pos].to_i != -1)
-              dmpio.pos = @option[:last_pos].to_i
-              current_state = @option[:state]
-              substate = @option[:substate]
-              current_table = @option[:mysql_table]
+            if @dump_pos_info[:last_pos] && (@dump_pos_info[:last_pos].to_i != -1)
+              dmpio.pos = @dump_pos_info[:last_pos].to_i
+              current_state = @dump_pos_info[:state]
+              substate = @dump_pos_info[:substate]
+              current_table = @dump_pos_info[:source_table]
               bytesize = dmpio.pos
             end
 
@@ -469,17 +434,17 @@ EOS
 
             until dump_io.eof? do
               case current_state
-              when State::START
+              when Flydata::Parser::State::START
                 state_start.call
-              when State::CREATE_TABLE
+              when Flydata::Parser::State::CREATE_TABLE
                 state_create_table.call
-              when State::CREATE_TABLE_COLUMNS
+              when Flydata::Parser::State::CREATE_TABLE_COLUMNS
                 state_create_table_columns.call
-              when State::CREATE_TABLE_CONSTRAINTS
+              when Flydata::Parser::State::CREATE_TABLE_CONSTRAINTS
                 state_create_table_constraints.call
-              when State::INSERT_RECORD
+              when Flydata::Parser::State::INSERT_RECORD
                 state_insert_record.call
-              when State::PARSING_INSERT_RECORD
+              when Flydata::Parser::State::PARSING_INSERT_RECORD
                 state_parsing_insert_record.call
               end
             end
@@ -747,4 +712,4 @@ EOT
   end
 end
 
-require 'flydata/parser/mysql/dump_parser_ext'
+require 'flydata/source_mysql/parser/dump_parser_ext'

data/lib/flydata/source_mysql/setup.rb

@@ -0,0 +1,24 @@
+require 'flydata/source/setup'
+require 'flydata/sync_file_manager'
+
+module Flydata
+module SourceMysql
+
+class Setup < Source::Setup
+  def initial_run_need_restart?
+    overinstall?
+  end
+
+  def initial_run_complete_message
+    overinstall? ? :all_done : :initial_sync
+  end
+
+  private
+
+  def overinstall?
+    File.exists?(Flydata::SyncFileManager.new(de).source_pos_path)
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/source_pos.rb

@@ -0,0 +1,21 @@
+require 'flydata/source/source_pos'
+require 'flydata/compatibility_check'
+require 'flydata-core/mysql/command_generator'
+require 'flydata-core/mysql/binlog_pos'
+
+module Flydata
+module SourceMysql
+
+class SourcePos < Source::SourcePos
+  def create_source_pos(source_pos_str)
+    FlydataCore::Mysql::BinlogPos.new(source_pos_str)
+  end
+
+  def resume_pos(source_pos)
+    #TODO check Binlog file (source_pos.filename) is available
+    FlydataCore::Mysql::BinlogPos.new(source_pos.filename, 4) # 4 is the first position of binlog file
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/sync.rb

@@ -0,0 +1,45 @@
+require 'flydata/source/sync'
+
+module Flydata
+module SourceMysql
+
+class Sync < Source::Sync
+  def setup
+    mp = de['mysql_data_entry_preference']
+
+    if mp['tables_append_only']
+      mp['tables_append_only'] = mp['tables_append_only'].split(",").uniq
+      mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only']).uniq
+    else
+      mp['tables'] = mp['tables'].split(",").uniq
+    end
+    mp['invalid_tables'] = mp['invalid_tables'].kind_of?(String) ? mp['invalid_tables'].split(",").uniq : []
+    mp['new_tables'] = mp['new_tables'].kind_of?(String) ? mp['new_tables'].split(",").uniq : []
+
+    unless mp['ssl_ca_content'].to_s.strip.empty?
+      sync_fm = SyncFileManager.new(de)
+      sync_fm.save_ssl_ca(mp['ssl_ca_content'])
+      mp['ssl_ca'] = sync_fm.ssl_ca_path
+      mp['sslca'] = mp['ssl_ca']
+    end
+  end
+
+  def supported?
+    true
+  end
+
+  def table_lists
+    de['mysql_data_entry_preference'].select {|key, value| %w(tables new_tables invalid_tables tables_append_only).include?(key)}
+  end
+
+  def data_servers
+    de['mysql_data_entry_preference']['data_servers']
+  end
+
+  def forwarder
+    de['mysql_data_entry_preference']['forwarder']
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/sync_generate_table_ddl.rb

@@ -0,0 +1,40 @@
+require 'flydata/source/sync_generate_table_ddl'
+require 'flydata/source_mysql/mysql_compatibility_check'
+require 'flydata-core/mysql/command_generator'
+
+module Flydata
+module SourceMysql
+
+class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
+  def run_compatibility_check
+    MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
+    if `which mysqldump`.empty?
+      raise "mysqldump is not installed.  mysqldump is required to run the command"
+    end
+    %w(host username database).each do |conf_name|
+      raise "MySQL `#{conf_name}` is neither defined in the data entry nor the local config file" if de['mysql_data_entry_preference'][conf_name].to_s.empty?
+    end
+  end
+
+  def generate_flydata_tabledef(tables, options)
+    mp = de['mysql_data_entry_preference']
+    options = options.merge(mp)
+    flydata_tabledefs = []
+    error_list = []
+    missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, options) do |mysql_tabledef, error|
+      if error
+        error_list << error.err_hash
+        next
+      end
+      flydata_tabledefs << mysql_tabledef.to_flydata_tabledef
+    end
+    if missing_tables
+      missing_tables.each {|missing_table| error_list << { error: 'table does not exist in the MySQL database', table: missing_table } }
+    end
+
+    [flydata_tabledefs, error_list]
+  end
+end
+
+end
+end