flydata 0.4.0 → 0.4.1

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.4.0 ruby lib
+# stub: flydata 0.4.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-06-17"
+  s.date = "2015-06-19"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -47,6 +47,10 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb",
     "flydata-core/lib/flydata-core/fluent/config_helper.rb",
     "flydata-core/lib/flydata-core/logger.rb",
+    "flydata-core/lib/flydata-core/mysql/command_generator.rb",
+    "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
+    "flydata-core/lib/flydata-core/query_job.rb",
+    "flydata-core/lib/flydata-core/query_job/redshift.rb",
     "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/redshift/string.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
@@ -58,6 +62,9 @@ Gem::Specification.new do |s|
     "flydata-core/spec/config/user_maintenance_spec.rb",
     "flydata-core/spec/fluent/config_helper_spec.rb",
     "flydata-core/spec/logger_spec.rb",
+    "flydata-core/spec/mysql/command_generator_spec.rb",
+    "flydata-core/spec/mysql/compatibility_checker.rb",
+    "flydata-core/spec/query_job/redshift_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
     "flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
@@ -145,7 +152,6 @@ Gem::Specification.new do |s|
     "lib/flydata/heroku/instance_methods.rb",
     "lib/flydata/log_monitor.rb",
     "lib/flydata/mysql/binlog_position.rb",
-    "lib/flydata/mysql/mysql_util.rb",
     "lib/flydata/mysql/table_ddl.rb",
     "lib/flydata/output/forwarder.rb",
     "lib/flydata/parser/mysql/dump_parser.rb",
@@ -201,7 +207,6 @@ Gem::Specification.new do |s|
     "spec/flydata/helper/worker_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/mysql/binlog_position_spec.rb",
-    "spec/flydata/mysql/mysql_util_spec.rb",
     "spec/flydata/mysql/table_ddl_spec.rb",
     "spec/flydata/output/forwarder_spec.rb",
     "spec/flydata/parser/mysql/alter_table_parser_spec.rb",
@@ -216,7 +221,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.4.6"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then
@@ -235,7 +240,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_runtime_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+      s.add_runtime_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
       s.add_runtime_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
@@ -259,7 +264,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+      s.add_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
       s.add_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
@@ -284,7 +289,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-    s.add_dependency(%q<kodama>, [">= 0.1.5", "~> 0.1.2"])
+    s.add_dependency(%q<kodama>, [">= 0.1.6", "~> 0.1.2"])
     s.add_dependency(%q<serverengine>, ["~> 1.5"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])

data/lib/flydata/api/data_entry.rb

@@ -1,3 +1,4 @@
+require 'json'
 require 'flydata/api/base'
 
 module Flydata
@@ -17,6 +18,14 @@ module Flydata
       def cleanup_sync(data_entry_id, tables)
         @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})
       end
+
+      # Update validity of tables
+      #  table_validity_hash: { "bad_table": "error reason", "good_table": nil }
+      #    table "bad_table" will be marked invalid with reason "error reason"
+      #    table "good table" will be marked valid, that is, clear its error reason if it's set.
+      def update_table_validity(data_entry_id, table_validity_hash)
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_validity_hash.to_json)
+      end
     end
   end
 end

data/lib/flydata/command/mysql.rb

@@ -1,5 +1,6 @@
 require 'open3'
 require 'flydata/command/sync'
+require 'flydata-core/mysql/command_generator'
 
 module Flydata
   module Command
@@ -8,7 +9,7 @@ module Flydata
       def run
         de = retrieve_sync_data_entry
         de['mysql_data_entry_preference'].delete('tables')
-        cmd = Flydata::Mysql::MysqlUtil::generate_mysql_cmd(de['mysql_data_entry_preference'])
+        cmd = FlydataCore::Mysql::CommandGenerator::generate_mysql_cmd(de['mysql_data_entry_preference'])
         if $stdin.tty?
           # interactive shell
           system cmd

data/lib/flydata/command/sync.rb

@@ -13,9 +13,9 @@ require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
-require 'flydata/mysql/mysql_util'
 require 'flydata-core/table_def'
 require 'flydata/mysql/table_ddl'
+require 'flydata-core/mysql/command_generator'
 #require 'ruby-prof'
 
 module Flydata
@@ -74,11 +74,8 @@ module Flydata
           exit 1
         end
 
-        # Setup instance variables
-        set_current_tables(tables)
-
         # Start initial sync with check
-        handle_mysql_sync
+        handle_mysql_sync(tables)
 
         # Start continuous sync by starting fluentd process
         unless opts.no_flydata_start?
@@ -97,9 +94,6 @@ module Flydata
       # Public method
       #  - Called from Sender#start/restart
       def try_mysql_sync
-        # Setup instance variables
-        set_current_tables
-
         # Start initial sync
         handle_mysql_sync
       rescue SyncDataEntryError
@@ -233,7 +227,7 @@ EOS
         Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
 
         # Set instance variables
-        set_current_tables(tables)
+        set_current_tables(tables, include_invalid_tables: true)
 
         do_generate_table_ddl(de)
       end
@@ -290,10 +284,18 @@ EOS
 
       # Initial sync
 
-      def handle_mysql_sync
+      def handle_mysql_sync(tables = nil)
         de = data_entry
 
-        unless @new_tables.empty?
+        # Setup instance variables
+        sync_resumed = set_current_tables(tables)
+
+        if sync_resumed
+          # skip confirmation prompts and resume sync right away.  #initial_sync knows
+          # where to resume from.
+          log_info_stdout("Resuming the initial sync...")
+          initial_sync(de, sync_resumed: true)
+        elsif !@new_tables.empty?
           show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}\n"
           unless @ddl_tables.empty?
@@ -305,7 +307,7 @@ EOS
           end
           log_info_stdout(unsynced_table_message)
           if ask_yes_no("Do you want to run initial sync on all of these tables now?")
-            initial_sync(de)
+            initial_sync(de, sync_resumed: false)
           else
             #If generate_table_ddl has not been run for these tables, warn user
             unless @ddl_tables.empty?
@@ -318,13 +320,16 @@ EOS
         end
       end
 
-      def initial_sync(de)
+      def initial_sync(de, opt)
         # Load sync information from file
-        log_info_stdout("Checking FlyData Servers for existing buffer data...")
-        load_sync_info(de)
         validate_initial_sync_status
         begin
-          flush_buffer_and_stop(target_tables_for_api)  unless @full_initial_sync
+          unless @full_initial_sync || opt[:sync_resumed]
+            # flush is unnecessary for full initial sync or sync resume because in either
+            # case it's guaranteed that agent is stopped with no leftover buffer.
+            log_info_stdout("Sending the existing buffer data...")
+            flush_buffer_and_stop(target_tables_for_api)
+          end
           sync_mysql_to_redshift(de)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
@@ -748,6 +753,7 @@ EOM
         mp = de['mysql_data_entry_preference']
 
         tables = opts.all_tables? ? @full_tables : (@input_tables.empty? ? @new_tables : @input_tables)
+
         raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
 
         %w(host username database).each do |conf_name|
@@ -759,7 +765,7 @@ EOM
 
         create_flydata_ctl_table = true
         option = {skip_parimay_key_check: opts.skip_primary_key_check?}.merge(mp)
-        Mysql::MysqlUtil.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
+        missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
           if error
             error_list << error.err_hash
             next
@@ -768,18 +774,37 @@ EOM
           puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
           create_flydata_ctl_table = false
         end
+        if missing_tables
+          missing_tables.each {|missing_table| error_list << { error: 'table does not exist in the MySQL database', table: missing_table } }
+        end
+        table_validity_hash = {}
+        tables_without_error = tables
         unless error_list.empty?
-          log_error_stderr("We have noticed the following error(s):")
+          log_error_stderr("\n\nERROR: FlyData Sync will not sync the following table(s) due to an error.")
           group_error =  error_list.group_by {|d| d[:error]}
-          group_error.each_key do |a|
-            log_error_stderr("The following table(s) have #{a}:")
-            group_error[a].each do |hash|
-              log_error_stderr("  - #{hash[:table]}") if hash[:table]
+          group_error.each_key do |error|
+            group_error[error].each do |hash|
+              if table = hash[:table]
+                log_error_stderr("  - #{table} (#{error})")
+                table_validity_hash[table] = error
+              end
             end
           end
-          log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
+          log_error_stderr(<<EOS)
+To sync these table(s), please fix the error(s) and run "flydata sync:generate_table_ddl" again.
+EOS
+          tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
+          unless tables_without_error.empty?
+            log_error_stderr(<<EOS)
+
+The other tables are ready to sync.  To start sync, run the generated script on the Redshift cluster and run "flydata start".
+EOS
+          end
         end
-        tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
+
+        fixed_tables = @invalid_tables & tables_without_error
+        fixed_tables.each {|table| table_validity_hash[table] = nil }
+        flydata.data_entry.update_table_validity(de['id'], {updated_tables: table_validity_hash}) unless table_validity_hash.empty?
 
         sync_fm = create_sync_file_manager(de)
         sync_fm.save_generated_ddl(tables_without_error, Mysql::TableDdl::VERSION)
@@ -819,20 +844,25 @@ Thank you for using FlyData!
 
       # Utility methods
 
-      def set_current_tables(input_tables = nil)
+      def set_current_tables(input_tables = nil, options = {})
         de = data_entry
         sync_fm = create_sync_file_manager(de)
         @input_tables = input_tables || []
-        @full_tables = de['mysql_data_entry_preference']['tables']
+        @invalid_tables = de['mysql_data_entry_preference']['invalid_tables'] # tables marked as invalid as a result of previous check
+        @full_tables = options[:include_invalid_tables] ? de['mysql_data_entry_preference']['tables'] + @invalid_tables : de['mysql_data_entry_preference']['tables']
 
         @new_tables = sync_fm.get_new_table_list(@full_tables, "pos")
         @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
 
         @full_initial_sync = (@new_tables == @full_tables)
 
+        sync_resumed = load_sync_info(sync_fm)
+
         sync_fm.close
 
         verify_input_tables(@input_tables, @full_tables)
+
+        sync_resumed
       end
 
       def validate_initial_sync_status
@@ -847,17 +877,17 @@ Thank you for using FlyData!
         end
       end
 
-      def load_sync_info(de)
+      def load_sync_info(sync_fm)
+        sync_resumed = false
         # for debug
         raise "!AssertionError. set_current_tables needs to be called in advance" if @full_tables.nil?
 
-        sync_fm = create_sync_file_manager(de)
         if (rs = sync_fm.load_sync_info)
           @full_initial_sync = rs[:initial_sync]
           @input_tables = rs[:tables]
+          sync_resumed = true
         end
-        sync_fm.close
-        de
+        sync_resumed
       end
 
       def target_tables
@@ -886,7 +916,7 @@ Thank you for using FlyData!
       end
 
       def retrieve_sync_data_entry
-        de = retrieve_data_entries.first unless de
+        de = retrieve_data_entries.first
         raise "There are no data entries." unless de
         case de['type']
         when 'RedshiftMysqlDataEntry'
@@ -897,6 +927,7 @@ Thank you for using FlyData!
           else
             mp['tables'] = mp['tables'].split(",").uniq
           end
+          mp['invalid_tables'] = mp['invalid_tables'].kind_of?(String) ? mp['invalid_tables'].split(",").uniq : []
 
           unless mp['ssl_ca_content'].to_s.strip.empty?
             sync_fm = create_sync_file_manager(de)

data/lib/flydata/compatibility_check.rb

@@ -1,15 +1,14 @@
 require 'mysql2'
 require 'flydata/command_loggable'
-require 'flydata/mysql/mysql_util'
+require 'flydata-core/mysql/command_generator'
+require 'flydata-core/mysql/compatibility_checker'
+require 'flydata-core/errors'
 
 module Flydata
 
   class CompatibilityCheck
     include CommandLoggable
 
-    class CompatibilityError < StandardError
-    end
-
     def initialize(dp_hash, de_hash=nil, options={})
       @dp = dp_hash
       @errors=[]
@@ -21,7 +20,7 @@ module Flydata
           send(m)
         rescue ArgumentError => e
           # ignore
-        rescue CompatibilityError => e
+        rescue FlydataCore::CompatibilityError => e
           @errors << e
         end
       end
@@ -40,8 +39,6 @@ module Flydata
   end
 
   class AgentCompatibilityCheck < CompatibilityCheck
-    class AgentCompatibilityError < StandardError
-    end
 
     TCP_PORT=45326
     SSL_PORT=45327
@@ -66,21 +63,13 @@ module Flydata
         errors.each do |port, e|
           message += "  Port #{port}, Error #{e.class.name}: #{e.to_s}"
         end
-        raise AgentCompatibilityError, message
+        raise FlydataCore::AgentCompatibilityError, message
       end
     end
   end
 
   class MysqlCompatibilityCheck < CompatibilityCheck
 
-    class MysqlCompatibilityError < StandardError
-    end
-
-    SELECT_QUERY_TMPLT = "SELECT %s"
-    BINLOG_RETENTION_HOURS = 24
-    SELECT_TABLE_INFO_TMPLT = "SELECT table_name, table_type, engine FROM information_schema.tables WHERE table_schema = '%s' and table_name in(%s)"
-
-    #def initialize(de_hash, dump_dir=nil)
     def initialize(dp_hash, de_hash, options={})
       super
       @db_opts = [:host, :port, :username, :password, :database, :ssl_ca].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
@@ -100,47 +89,11 @@ module Flydata
     end
 
     def check_mysql_user_compat
-      databases = ['mysql', @db_opts[:database]]
-      get_grant_regex = /GRANT (?<privs>.*) ON (`)?(?<db_name>[^`]*)(`)?\.\* TO '#{@db_opts[:username]}/
-      necessary_permission_fields = ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
-      all_privileges_field = ["ALL PRIVILEGES"]
-
-      # Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
-      grants_sql = "SHOW GRANTS"
-      client = Mysql2::Client.new(@db_opts)
-      result = client.query(grants_sql)
-      client.close
-
-      found_priv = Hash[databases.map {|d| [d,[]]}]
-      missing_priv = {}
-
-      result.each do |res|
-        # SHOW GRANTS should only return one column
-        res_value = res.values.first
-        matched_values = res_value.match(get_grant_regex)
-        next unless matched_values
-        line_priv = matched_values["privs"].split(", ")
-        if matched_values["db_name"] == "*"
-          return true if (all_privileges_field - line_priv).empty?
-          databases.each {|d| found_priv[d] << line_priv }
-        elsif databases.include? matched_values["db_name"] 
-          if (all_privileges_field - line_priv).empty?
-            found_priv[matched_values["db_name"]] = necessary_permission_fields
-          else
-            found_priv[matched_values["db_name"]] << line_priv
-          end
-        end
-        missing_priv = get_missing_privileges(found_priv, necessary_permission_fields)
-        return true if missing_priv.empty?
-      end
-      error_text = "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n"
-      error_text << "    * These privileges are missing...\n"
-      missing_priv.each_key {|db| error_text << "      for the database '#{db}': #{missing_priv[db].join(", ")}\n"}
-      raise MysqlCompatibilityError, error_text
+      FlydataCore::Mysql::SyncPermissionChecker.new(@db_opts).do_check
     end
 
     def check_mysql_protocol_tcp_compat
-      query = Mysql::MysqlUtil.generate_mysql_show_grants_cmd(@db_opts)
+      query = FlydataCore::Mysql::CommandGenerator.generate_mysql_show_grants_cmd(@db_opts)
 
       Open3.popen3(query) do |stdin, stdout, stderr|
         stdin.close
@@ -151,64 +104,27 @@ module Flydata
           log_error("Error occured during access to mysql server.", {err: err_reason})
 
           unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
-            raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
+            raise FlydataCore::MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
           end
         end
       end
     end
 
     def check_mysql_parameters_compat
-      sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1, '@@log_slave_updates'=>1}
-      errors={}
-
-      client = Mysql2::Client.new(@db_opts)
-
-      begin
-        sys_var_to_check.each_key do |sys_var|
-          sel_query = SELECT_QUERY_TMPLT % sys_var
-          begin
-            result = client.query(sel_query)
-            unless result.first[sys_var] == sys_var_to_check[sys_var]
-              errors[sys_var]=result.first[sys_var]
-            end
-          rescue Mysql2::Error => e
-            if e.message =~ /Unknown system variable/
-              unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
-                errors[sys_var] = false
-              end
-            else
-              raise e
-            end
-          end
-        end
-      ensure
-        client.close
-      end
-      unless errors.empty?
-        error_explanation = ""
-        errors.each_key do |err_key|
-          error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
-        end
-        raise MysqlCompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n   Please change these system variables for FlyData Sync to run correctly"
-      end
+      FlydataCore::Mysql::BinlogParameterChecker.new(@db_opts).do_check
     end
 
     def check_mysql_binlog_retention
-      client = Mysql2::Client.new(@db_opts)
-      begin
-        if is_rds?(@db_opts[:host])
-          run_rds_retention_check(client)
-        else
-          run_mysql_retention_check(client)
-        end
-      ensure
-        client.close
+      if is_rds?(@db_opts[:host])
+        run_rds_retention_check
+      else
+        run_mysql_retention_check
       end
     end
 
     def check_writing_permissions
       write_errors = []
-      paths_to_check = ["~/.flydata"]
+      paths_to_check = [FLYDATA_HOME]
       paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
       paths_to_check << @backup_dir unless @backup_dir.to_s.empty?
       paths_to_check.each do |path|
@@ -218,71 +134,36 @@ module Flydata
       end
       unless write_errors.empty?
         error_dir = write_errors.join(", ")
-        raise MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
+        raise FlydataCore::MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
       end
     end
 
     # If table_type='VIEW' or engine='MEMORY', raise error.
     def check_mysql_table_types
       return if @tables.empty?
-      client = Mysql2::Client.new(@db_opts)
-      sel_query = SELECT_TABLE_INFO_TMPLT % [client.escape(@db_opts[:database]), @tables.collect{|t| "'#{client.escape(t)}'"}.join(", ")]
-      begin
-        invalid_tables = []
-        client.query(sel_query).each do |r|
-          invalid_tables.push(r['table_name']) if r['table_type'] == 'VIEW' || r['engine'] == 'MEMORY'
-        end
-        raise MysqlCompatibilityError, "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: #{invalid_tables.join(", ")}" unless invalid_tables.empty?
-      ensure
-        client.close
-      end
-    end
-
-    def get_missing_privileges(found_priv, all_priv)
-      return_hash = {}
-      found_priv.each_key do |key|
-        missing_priv = all_priv - found_priv[key].flatten.uniq
-        return_hash[key] = missing_priv unless missing_priv.empty?
-      end
-      return_hash
+      option = @db_opts.dup.merge(tables: @tables)
+      FlydataCore::Mysql::TableTypeChecker.new(option).do_check
     end
 
-    def run_mysql_retention_check(mysql_client)
-      expire_logs_days_limit = BINLOG_RETENTION_HOURS / 24
-      sel_query = SELECT_QUERY_TMPLT % '@@expire_logs_days'
-      result = mysql_client.query(sel_query)
-      if result.first["@@expire_logs_days"]!=0 and result.first["@@expire_logs_days"] <= expire_logs_days_limit
-        raise MysqlCompatibilityError, "Binary log retention is too short\n " +
-          "  We recommend the system variable '@@expire_logs_days' to be either set to 0 or at least #{expire_logs_days_limit} days"
-      end
+    def run_mysql_retention_check
+      FlydataCore::Mysql::NonRdsRetentionChecker.new(@db_opts).do_check
     end
 
-    def run_rds_retention_check(mysql_client)
-      sql_query =  "call mysql.rds_show_configuration;"
-
-      begin
-        result = mysql_client.query(sql_query)
-        if result.first["name"]=="binlog retention hours"
-          if result.first["value"].nil? or result.first["value"].to_i <= BINLOG_RETENTION_HOURS
-            raise MysqlCompatibilityError, "Binary log retention is too short\n" +
-                                           "  We recommend setting RDS binlog retention to be at least #{BINLOG_RETENTION_HOURS} hours. To do this, run this on your RDS MySQL database:\n" +
-                                           "  $> call mysql.rds_set_configuration('binlog retention hours', 94);"
-          end
-        end
-      rescue Mysql2::Error => e
-        if e.message =~ /command denied to user/
-          log_warn_stderr("[WARNING]Cannot verify RDS retention period on current MySQL user account.\n" +
-                          "To see retention period, please run this on your RDS:\n" +
-                          "  $> call mysql.rds_show_configuration;\n" +
-                          "Please verify that the hours is not nil and is at least #{BINLOG_RETENTION_HOURS} hours\n" +
-                          "To set binlog retention hours, you can run this on your RDS:\n" +
-                          "  $> call mysql.rds_set_configuration('binlog retention hours', #{BINLOG_RETENTION_HOURS});\n"
-                         )
-        else
-          raise e
-        end
+    def run_rds_retention_check
+      FlydataCore::Mysql::RdsRetentionChecker.new(@db_opts).do_check
+    rescue Mysql2::Error => e
+      if e.message =~ /command denied to user/
+        retention_hours = FlydataCore::Mysql::RdsRetentionChecker::BINLOG_RETENTION_HOURS
+        log_warn_stderr("[WARNING]Cannot verify RDS retention period on current MySQL user account.\n" +
+                        "To see retention period, please run this on your RDS:\n" +
+                        "  $> call mysql.rds_show_configuration;\n" +
+                        "Please verify that the hours is not nil and is at least #{retention_hours} hours\n" +
+                        "To set binlog retention hours, you can run this on your RDS:\n" +
+                        "  $> call mysql.rds_set_configuration('binlog retention hours', #{retention_hours});\n"
+                       )
+      else
+        raise e
       end
-
     end
 
     def is_rds?(hostname)