flydata 0.2.8 → 0.2.9

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -56,7 +56,12 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       raise "No position file(#{@position_file}). Initial synchronization is required before starting."
     end
     load_custom_conf
-    $log.info "mysql host:\"#{@host}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
+    $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
+    $log.info "mysql client version: #{`mysql -V`}"
+    server_version = `echo 'select version();' | mysql -h #{@host} --port #{@port} -u #{@username} -p#{@password} 2>/dev/null`
+    server_version = server_version[(server_version.index("\n") + 1)..-1]
+    $log.info "mysql server version: #{server_version}"
+
     @tables = @tables.split(/,\s*/)
     @omit_events = Hash.new
     @tables_append_only.split(/,\s*/).each do |table|

data/lib/flydata/helpers.rb

@@ -102,5 +102,16 @@ Usage: flydata COMMAND
         end
       end
     end
+
+    UNIT_PREFIX = %W(TB GB MB KB B).freeze
+    def as_size( s )
+      s = s.to_f
+      i = UNIT_PREFIX.length - 1
+      while s > 512 && i > 0
+        s /= 1024
+        i -= 1
+      end
+      ((s > 9 || s.modulo(1) < 0.1 ? '%d' : '%.1f') % s) + ' ' + UNIT_PREFIX[i]
+    end
   end
 end

data/lib/flydata/output/forwarder.rb

@@ -0,0 +1,166 @@
+require 'socket'
+
+module Flydata
+  module Output
+    class ForwarderFactory
+      def self.create(forwarder_key, tag, servers, options = {})
+        case forwarder_key
+        when nil, "tcpforwarder"
+          puts "Creating TCP connection" if FLYDATA_DEBUG
+          forward = TcpForwarder.new(tag, servers, options)
+        when "sslforwarder"
+          puts "Creating SSL connection" if FLYDATA_DEBUG
+          forward = SslForwarder.new(tag, servers, options)
+        else
+          raise "Unsupported Forwarding type #{forwarder_key}"
+        end
+        forward
+      end
+    end
+
+    class TcpForwarder
+      FORWARD_HEADER = [0x92].pack('C')
+      BUFFER_SIZE = 1024 * 1024 * 32 # 32M
+      DEFUALT_SEND_TIMEOUT = 60 # 1 minute
+      RETRY_INTERVAL = 2
+      RETRY_LIMIT = 10
+
+      def initialize(tag, servers, options = {})
+        @tag = tag
+        unless servers and servers.kind_of?(Array) and not servers.empty?
+          raise "Servers must not be empty."
+        end
+        @servers = servers
+        @server_index = 0
+        set_options(options)
+        reset
+      end
+
+      def set_options(options)
+        if options[:buffer_size_limit]
+          @buffer_size_limit = options[:buffer_size_limit]
+        else
+          @buffer_size_limit = BUFFER_SIZE
+        end
+      end
+
+      attr_reader :buffer_record_count, :buffer_size
+
+      def emit(records, time = Time.now.to_i)
+        records = [records] unless records.kind_of?(Array)
+        records.each do |record|
+          event_data = [time,record].to_msgpack
+          @buffer_records << event_data
+          @buffer_record_count += 1
+          @buffer_size += event_data.bytesize
+        end
+        if @buffer_size > @buffer_size_limit
+          send
+        else
+          false
+        end
+      end
+
+      #TODO retry logic
+      def send
+        if @buffer_size > 0
+        else
+          return false
+        end
+        if ENV['FLYDATA_BENCHMARK']
+          reset
+          return true
+        end
+        sock = nil
+        retry_count = 0
+        begin
+          sock = connect(pickup_server)
+
+          # Write header
+          sock.write FORWARD_HEADER
+          # Write tag
+          sock.write @tag.to_msgpack
+          # Write records
+          sock.write [0xdb, @buffer_records.bytesize].pack('CN')
+          StringIO.open(@buffer_records) do |i|
+            FileUtils.copy_stream(i, sock)
+          end
+        rescue => e
+          retry_count += 1
+          if retry_count > RETRY_LIMIT
+            puts "! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}"
+            raise e
+          end
+          puts "! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}"
+          wait_time = RETRY_INTERVAL ** retry_count
+          puts "  Now waiting for next retry. time=#{wait_time}sec"
+          sleep wait_time
+          retry
+        ensure
+          if sock
+            sock.close rescue nil
+          end
+        end
+        reset
+        true
+      end
+
+      #TODO: Check server status
+      def pickup_server
+        ret_server = @servers[@server_index]
+        @server_index += 1
+        if @server_index >= (@servers.count)
+          @server_index = 0
+        end
+        ret_server
+      end
+
+      def connect(server)
+        host, port = server.split(':')
+        sock = TCPSocket.new(host, port.to_i)
+
+        # Set options
+        opt = [1, DEFUALT_SEND_TIMEOUT].pack('I!I!')
+        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        opt = [DEFUALT_SEND_TIMEOUT, 0].pack('L!L!')
+        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+        sock
+      end
+
+      def reset
+        @buffer_records = ''
+        @buffer_record_count = 0
+        @buffer_size = 0
+      end
+
+      def flush
+        send
+      end
+
+      def close
+        flush
+      end
+    end
+    class SslForwarder < TcpForwarder
+      def connect(server)
+        tcp_sock = super
+        ssl_ctx = ssl_ctx_with_verification
+        ssl_sock = OpenSSL::SSL::SSLSocket.new(tcp_sock, ssl_ctx)
+        ssl_sock.sync_close = true
+        ssl_sock.connect
+        ssl_sock
+      end
+
+      private
+      def ssl_ctx_with_verification
+        cert_store = OpenSSL::X509::Store.new
+        cert_store.set_default_paths
+        ssl_ctx = OpenSSL::SSL::SSLContext.new
+        ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        ssl_ctx.cert_store = cert_store
+        ssl_ctx
+      end
+    end
+  end
+end

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -0,0 +1,729 @@
+module Flydata
+  module Parser
+    module Mysql
+
+      module MysqlAccessible
+        def mysql_conf(conf)
+          @mysql_conf = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+        end
+
+        def mysql_cli(conf = nil)
+          mysql_conf(conf) if conf
+          return  Mysql2::Client.new(@mysql_conf) if @mysql_conf
+          nil
+        end
+      end
+
+      module DumpStreamIO
+        # return position
+        # sync command doesn't resume if pos is -1 in dump position file
+        def pos
+          -1
+        end
+      end
+
+      class MysqlTable
+        def initialize(table_name, columns = {}, primary_keys = [])
+          @table_name = table_name
+          @columns = columns
+          @primary_keys = primary_keys
+        end
+
+        attr_accessor :table_name, :columns, :primary_keys
+
+        def add_column(column)
+          @columns[column[:column_name]] = column
+        end
+      end
+
+      class MysqlDumpGenerator
+        # host, port, username, password, database, tables
+        MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
+        EXTRA_MYSQLDUMP_PARAMS = ""
+        def initialize(conf)
+          password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
+          tables = if conf['tables']
+                     conf['tables'].split(',').join(' ')
+                   else
+                     ''
+                   end
+          @dump_cmd = MYSQL_DUMP_CMD_TEMPLATE %
+            [conf['host'], conf['port'], conf['username'], password, self.class::EXTRA_MYSQLDUMP_PARAMS, conf['database'], tables]
+          @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
+        end
+        def dump(file_path)
+          raise "subclass must implement the method"
+        end
+      end
+
+      class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
+        EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
+        def dump(file_path)
+          cmd = "#{@dump_cmd} -r #{file_path}"
+          o, e, s = Open3.capture3(cmd)
+          e.to_s.each_line {|l|  puts l unless /^Warning:/ =~ l } unless e.to_s.empty?
+          unless s.exitstatus == 0
+            if File.exists?(file_path)
+              File.open(file_path, 'r') {|f| f.each_line{|l| puts l}}
+              FileUtils.rm(file_path)
+            end
+            raise "Failed to run mysqldump command."
+          end
+          unless File.exists?(file_path)
+            raise "mysqldump file does not exist. Something wrong..."
+          end
+          if File.size(file_path) == 0
+            raise "mysqldump file is empty. Something wrong..."
+          end
+          true
+        end
+      end
+
+      class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
+        EXTRA_MYSQLDUMP_PARAMS = ""
+        CHANGE_MASTER_TEMPLATE = <<EOS
+--
+-- Position to start replication or point-in-time recovery from
+--
+
+-- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
+
+EOS
+
+        def dump(file_path = nil, &block)
+          unless file_path || block
+            raise ArgumentError.new("file_path or block must be given.")
+          end
+
+          # RDS doesn't allow obtaining binlog position using mysqldump.  Get it separately and insert it into the dump file.
+          table_locker = create_table_locker
+          table_locker.resume # Lock tables
+
+          begin
+            # create pipe for callback function
+            rd_io, wr_io = IO.pipe
+            wr_io.sync = true
+            wr_io.set_encoding("utf-8")
+            rd_io.extend(DumpStreamIO)
+
+            # start mysqldump
+            Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
+              cmd_in.close_write
+              cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
+
+              first_line = cmd_out.gets # wait until first line comes
+              binlog_file, binlog_pos = table_locker.resume
+
+              threads = []
+
+              # filter dump stream and write data to pipe
+              threads << Thread.new do
+                begin
+                  wr_io.print(first_line) # write a first line
+                  fileter_dump_stream(cmd_out, wr_io, binlog_file, binlog_pos)
+                ensure
+                  wr_io.close rescue nil
+                end
+              end
+
+              # show err message
+              threads << Thread.new do
+                cmd_err.each_line do |line|
+                  $stderr.print line unless /^Warning:/ === line
+                end
+              end
+
+              if block
+                # call callback function with io if block is given
+                block.call(rd_io)
+              elsif file_path
+                # store data to file
+                open_file_stream(file_path) {|f| rd_io.each_line{|l| f.print(l)}}
+              end
+
+              threads.each(&:join)
+            end
+          rescue
+            # Cleanup
+            FileUtils.rm(file_path) if file_path && File.exists?(file_path)
+            raise
+          ensure
+            # Let table_locker finish its task even if an exception happened
+            table_locker.resume if table_locker.alive?
+            rd_io.close rescue nil
+            wr_io.close rescue nil
+          end
+        end
+
+        private
+
+        def open_file_stream(file_path, &block)
+          File.open(file_path, "w", encoding: "utf-8") {|f| block.call(f)}
+        end
+
+        # This query generates a query which flushes user tables with read lock
+        FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
+        USER_TABLES_QUERY = <<EOS
+SELECT CONCAT('`',
+REPLACE(TABLE_SCHEMA, '`', '``'), '`.`',
+REPLACE(TABLE_NAME, '`', '``'), '` ')
+AS tables
+FROM INFORMATION_SCHEMA.TABLES
+WHERE TABLE_TYPE = 'BASE TABLE'
+AND ENGINE NOT IN ('MEMORY', 'CSV', 'PERFORMANCE_SCHEMA');
+EOS
+
+        def create_table_locker
+          Fiber.new do
+            client = Mysql2::Client.new(@db_opts)
+            # Lock tables
+            client.query "FLUSH LOCAL TABLES;"
+            q = flush_tables_with_read_lock_query(client)
+            puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
+            client.query q
+            begin
+              Fiber.yield  # Lock is done.  Let dump to start
+              # obtain binlog pos
+              result = client.query "SHOW MASTER STATUS;"
+              row = result.first
+              if row.nil?
+                raise "MySQL DB has no replication master status.  Check if the DB is set up as a replication master.  In case of RDS, make sure that Backup Retention Period is set to more than 0."
+              end
+            ensure
+              # unlock tables
+              client.query "UNLOCK TABLES;"
+              client.close
+            end
+
+            [row["File"], row['Position']]
+          end
+        end
+
+        def flush_tables_with_read_lock_query(client)
+          tables = ""
+          if mysql_server_version(client) >= "5.5"
+            # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
+            result = client.query(USER_TABLES_QUERY)
+            tables = result.collect{|r| r['tables']}.join(", ")
+          end
+          FLUSH_TABLES_QUERY_TEMPLATE % [tables]
+        end
+
+        VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
+        def mysql_server_version(client)
+          result = client.query(VERSION_QUERY)
+          result.first['Value']
+        end
+
+        def fileter_dump_stream(cmd_out, w_io, binlog_file, binlog_pos)
+          find_insert_pos = :not_started
+          cmd_out.each_line do |line|
+            if find_insert_pos == :not_started && /^-- Server version/ === line
+              find_insert_pos = :finding
+            elsif find_insert_pos == :finding && /^--/ === line
+              # wait before writing the first database queries
+              # insert binlog pos
+              change_master =  CHANGE_MASTER_TEMPLATE % [binlog_file, binlog_pos]
+              w_io.print change_master
+
+              find_insert_pos = :found
+            end
+            w_io.print line
+            w_io.puts unless line.end_with?("\n")
+            w_io.flush
+          end
+        end
+
+      end
+
+      class MysqlDumpParser
+
+        module State
+          START = 'START'
+          CREATE_TABLE = 'CREATE_TABLE'
+          CREATE_TABLE_COLUMNS = 'CREATE_TABLE_COLUMNS'
+          CREATE_TABLE_CONSTRAINTS = 'CREATE_TABLE_CONSTRAINTS'
+          INSERT_RECORD = 'INSERT_RECORD'
+          PARSING_INSERT_RECORD = 'PARSING_INSERT_RECORD'
+        end
+
+        attr_accessor :binlog_pos
+
+        def initialize(option = {})
+          @binlog_pos = option[:binlog_pos]
+          @option = option
+        end
+
+        def parse(dump_io, create_table_block, insert_record_block, check_point_block)
+          unless dump_io.kind_of?(IO)
+            raise ArgumentError.new("Invalid argument. The first parameter must be io.")
+          end
+
+          invalid_file = false
+          current_state = State::START
+          substate = nil
+          buffered_line = nil
+          bytesize = 0
+
+          readline_proc = Proc.new do
+            line = nil
+            if buffered_line
+              line = buffered_line
+              buffered_line = nil
+            else
+              rawline = dump_io.readline
+              bytesize += rawline.bytesize
+              line = rawline.strip
+            end
+            line
+          end
+
+          state_start = Proc.new do
+            line = readline_proc.call
+
+            # -- CHANGE MASTER TO MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=120;
+            m = /^\-\- CHANGE MASTER TO MASTER_LOG_FILE='(?<binfile>[^']+)', MASTER_LOG_POS=(?<pos>\d+)/.match(line)
+            if m
+              @binlog_pos = {binfile: m[:binfile], pos: m[:pos].to_i}
+              current_state = State::CREATE_TABLE
+              check_point_block.call(nil, dump_io.pos, bytesize, @binlog_pos, current_state)
+            end
+          end
+
+          current_table = nil
+          state_create_table = Proc.new do
+            line = readline_proc.call
+
+            # CREATE TABLE `active_admin_comments` (
+            m = /^CREATE TABLE `(?<table_name>[^`]+)`/.match(line)
+            if m
+              current_table = MysqlTable.new(m[:table_name])
+              current_state = State::CREATE_TABLE_COLUMNS
+            end
+          end
+
+          state_create_table_constraints = Proc.new do
+            line = readline_proc.call
+
+            #  PRIMARY KEY (`id`),
+            if line.start_with?(')')
+              create_table_block.call(current_table)
+              current_state = State::INSERT_RECORD
+              check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
+            elsif m = /^PRIMARY KEY \((?<primary_keys>[^\)]+)\)/.match(line)
+              current_table.primary_keys = m[:primary_keys].split(',').collect do |pk_str|
+                pk_str[1..-2]
+              end
+            end
+          end
+
+          state_create_table_columns = Proc.new do
+            start_pos = dump_io.pos
+            line = readline_proc.call
+
+            #  `author_type` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
+            if line.start_with?("\`")
+              column = {}
+
+              # parse column line
+              line = line[0..-2] if line.end_with?(',')
+              items = line.split
+              column[:column_name] = items.shift[1..-2]
+              column[:format_type_str] = format_type_str = items.shift
+              pos = format_type_str.index('(')
+              if pos
+                ft = column[:format_type] = format_type_str[0..pos-1]
+                if ft == 'decimal'
+                  precision, scale = format_type_str[pos+1..-2].split(',').collect{|v| v.to_i}
+                  column[:decimal_precision] = precision
+                  column[:decimal_scale] = scale
+                else
+                  column[:format_size] = format_type_str[pos+1..-2].to_i
+                end
+              else
+                column[:format_type] = format_type_str
+              end
+              while (item = items.shift) do
+                case item
+                when 'DEFAULT'
+                  value = items.shift
+                  value = value.start_with?('\'') ? value[1..-2] : value
+                  value = nil if value == 'NULL'
+                  column[:default] = value
+                when 'NOT'
+                  if items[1] == 'NULL'
+                    items.shift
+                    column[:not_null] = true
+                  end
+                when 'unsigned'
+                  column[:unsigned] = true
+                else
+                  #ignore other options
+                end
+              end
+
+              current_table.add_column(column)
+            else
+              current_state = State::CREATE_TABLE_CONSTRAINTS
+              buffered_line = line
+              state_create_table_constraints.call
+            end
+          end
+
+          state_insert_record = Proc.new do
+            line = readline_proc.call
+
+            if line.start_with?('INSERT')
+              buffered_line = line
+              current_state = State::PARSING_INSERT_RECORD
+            elsif line.start_with?('UNLOCK')
+              current_state = State::CREATE_TABLE
+              check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
+            end
+          end
+
+          state_parsing_insert_record = Proc.new do
+            line = readline_proc.call
+
+            values_set = InsertParser.new.parse(line)
+            current_state = State::INSERT_RECORD
+
+            if insert_record_block.call(current_table, values_set)
+              check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
+            end
+          end
+
+          # Start reading file from top
+          begin
+            # resume(only when using dump file)
+            if @option[:last_pos] && (@option[:last_pos].to_i != -1)
+              dump_io.pos = @option[:last_pos].to_i
+              current_state = @option[:state]
+              substate = @option[:substate]
+              current_table = @option[:mysql_table]
+              bytesize = dump_io.pos
+            end
+
+            until dump_io.eof? do
+              case current_state
+              when State::START
+                state_start.call
+              when State::CREATE_TABLE
+                state_create_table.call
+              when State::CREATE_TABLE_COLUMNS
+                state_create_table_columns.call
+              when State::CREATE_TABLE_CONSTRAINTS
+                state_create_table_constraints.call
+              when State::INSERT_RECORD
+                state_insert_record.call
+              when State::PARSING_INSERT_RECORD
+                state_parsing_insert_record.call
+              end
+            end
+          end
+          @binlog_pos
+        end
+
+        # Parse the insert line containing multiple values. (max line size is 1kb)
+        #   ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
+        class InsertParser
+          #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
+          module State
+            IN_VALUE = 'IN_VALUE'
+            NEXT_VALUES = 'NEXT_VALUES'
+          end
+
+          def initialize
+            @values = []
+            @values_set = []
+          end
+
+          def start_ruby_prof
+            RubyProf.start if defined?(RubyProf) and not RubyProf.running?
+          end
+
+          def stop_ruby_prof
+            if defined?(RubyProf) and RubyProf.running?
+              result = RubyProf.stop
+              #printer = RubyProf::GraphPrinter.new(result)
+              printer = RubyProf::GraphHtmlPrinter.new(result)
+              #printer.print(STDOUT)
+              printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
+            end
+          end
+
+          def parse(line)
+            start_ruby_prof
+            bench_start_time = Time.now
+            _parse(line)
+          ensure
+            stop_ruby_prof
+            if ENV['FLYDATA_BENCHMARK']
+              puts " -> time:#{Time.now.to_f - bench_start_time.to_f} size:#{target_line.size}"
+            end
+          end
+
+          private
+
+          def _parse(target_line)
+            target_line = target_line.strip
+            start_index = target_line.index('(')
+            target_line = target_line[start_index..-2]
+
+            # Split insert line text with ',' and take care of ',' inside of the values later.
+            #
+            # We are using the C native method that is like 'split', 'start_with?', 'regexp'
+            # instead of 'String#each_char' and string comparision for the performance.
+            # 'String#each_char' is twice as slow as the current storategy.
+            items = target_line.split(',')
+            index = 0
+            cur_state = State::NEXT_VALUES
+
+            loop do
+              case cur_state
+              when State::NEXT_VALUES
+                chars = items[index]
+                break unless chars
+                items[index] = chars[1..-1]
+                cur_state = State::IN_VALUE
+              when State::IN_VALUE
+                chars = items[index]
+                index += 1
+                if chars.start_with?("'")
+                  # single item (not last item)
+                  # size check added below otherwise end_with? matches the single quote which was also used by start_with?
+                  if chars.size > 1 and chars.end_with?("'") and !last_char_escaped?(chars)
+                    @values << replace_escape_char(chars[1..-2])
+                    # single item (last item)
+                    # size check added below otherwise end_with? matches the single quote which was also used by start_with?
+                  elsif chars.size > 2 and chars.end_with?("')") and !last_char_escaped?(chars[0..-2])
+                    @values << replace_escape_char(chars[1..-3])
+                    @values_set << @values
+                    @values = []
+                    cur_state = State::NEXT_VALUES
+                    # multi items
+                  else
+                    cur_value = chars[1..-1]
+                    loop do
+                      next_chars = items[index]
+                      index += 1
+                      if next_chars.end_with?('\'') and !last_char_escaped?(next_chars)
+                        cur_value << ','
+                        cur_value << next_chars[0..-2]
+                        @values << replace_escape_char(cur_value)
+                        break
+                      elsif next_chars.end_with?("')") and !last_char_escaped?(next_chars[0..-2])
+                        cur_value << ','
+                        cur_value << next_chars[0..-3]
+                        @values << replace_escape_char(cur_value)
+                        @values_set << @values
+                        @values = []
+                        cur_state = State::NEXT_VALUES
+                        break
+                      else
+                        cur_value << ','
+                        cur_value << next_chars
+                      end
+                    end
+                  end
+                else
+                  if chars.end_with?(')')
+                    chars = chars[0..-2]
+                    @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
+                    @values_set << @values
+                    @values = []
+                    cur_state = State::NEXT_VALUES
+                  else
+                    @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
+                  end
+                end
+              else
+                raise "Invalid state: #{cur_state}"
+              end
+            end
+            return @values_set
+          end
+
+          ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
+
+          def replace_escape_char(original)
+            original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
+          end
+
+          # This method assume that the last character is '(single quotation)
+          # abcd\' -> true
+          # abcd\\' -> false (back slash escape back slash)
+          # abcd\\\' -> true
+          def last_char_escaped?(text)
+            flag = false
+            (text.length - 2).downto(0) do |i|
+              if text[i] == '\\'
+                flag = !flag
+              else
+                break
+              end
+            end
+            flag
+          end
+
+          def remove_leading_zeros(number_string)
+            if number_string.start_with?('0')
+              number_string.sub(/^0*([1-9][0-9]*(\.\d*)?|0(\.\d*)?)$/,'\1')
+            else
+              number_string
+            end
+          end
+        end
+      end
+      class CompatibilityCheck
+
+        class CompatibilityError < StandardError
+        end
+
+        SELECT_QUERY_TMPLT = "SELECT %s"
+
+        def initialize(de_hash, dump_dir=nil)
+          @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
+          @dump_dir = dump_dir
+          @errors=[]
+        end
+
+        def check
+          self.methods.grep(/^check_/).each do |m|
+            begin
+              send(m)
+            rescue CompatibilityError => e
+              @errors << e
+            end
+          end
+          print_errors
+        end
+
+        def print_errors
+          return if @errors.empty?
+          puts "There may be some compatibility issues with your MySQL credentials: "
+          @errors.each do |error|
+            puts "  * #{error.message}"
+          end
+          raise "Please correct these errors if you wish to run FlyData Sync"
+        end
+
+        def check_mysql_user_compat
+          client = Mysql2::Client.new(@db_opts)
+          grants_sql = "SHOW GRANTS"
+          correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
+          necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
+          all_privileges_field= ["ALL PRIVILEGES"]
+          result = client.query(grants_sql)
+          # Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
+          client.close
+          missing_priv = []
+          result.each do |res|
+            # SHOW GRANTS should only return one column
+            res_value = res.values.first
+            if correct_db.all? {|perm| res_value.match(perm)}
+              necessary_permission_fields.each do |priv|
+                missing_priv << priv unless res_value.match(priv)
+              end
+              return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
+            end
+          end
+          raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n    * These privileges are missing: #{missing_priv.join(", ")}"
+        end
+
+        def check_mysql_protocol_tcp_compat
+          query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
+          query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
+
+          Open3.popen3(query) do |stdin, stdout, stderr|
+            stdin.close
+            while !stderr.eof?
+              line = stderr.gets
+              unless /Warning: Using a password on the command line interface can be insecure./ === line
+                raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
+              end
+            end
+          end
+        end
+
+        def check_mysql_row_mode_compat
+          sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
+          errors={}
+
+          client = Mysql2::Client.new(@db_opts)
+
+          begin
+            sys_var_to_check.each_key do |sys_var|
+              sel_query = SELECT_QUERY_TMPLT % sys_var
+              begin
+                result = client.query(sel_query)
+                unless result.first[sys_var] == sys_var_to_check[sys_var]
+                  errors[sys_var]=result.first[sys_var]
+                end
+              rescue Mysql2::Error => e
+                if e.message =~ /Unknown system variable/
+                  unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
+                    errors[sys_var] = false
+                  end
+                else
+                  raise e
+                end
+              end
+            end
+          ensure
+            client.close
+          end
+          unless errors.empty?
+            error_explanation = ""
+            errors.each_key do |err_key|
+              error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
+            end
+            raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n   Please change these system variables for FlyData Sync to run correctly"
+          end
+        end
+
+        def check_writing_permissions
+          write_errors = []
+          paths_to_check = ["~/.flydata"]
+          paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
+          paths_to_check.each do |path|
+            full_path = File.expand_path(path)
+            full_path = File.dirname(full_path) unless File.directory?(full_path)
+            write_errors << full_path unless File.writable?(full_path)
+          end
+          unless write_errors.empty?
+            error_dir = write_errors.join(", ")
+            raise CompatibilityError, "We cannot access the directories: #{error_dir}"
+          end
+        end
+      end
+
+      class DatabaseSizeCheck
+        include MysqlAccessible
+
+        SIZE_CHECK_QUERY = <<EOT
+SELECT
+  SUM(data_length) bytesize
+FROM
+  information_schema.tables
+WHERE
+  table_schema NOT IN ('information_schema','performance_schema','mysql') AND table_name in (%s);
+EOT
+
+        def initialize(de_conf)
+          @de_conf = de_conf
+          @tables = de_conf['tables'].split(',')
+          @query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
+        end
+
+        def get_db_bytesize
+          client = mysql_cli(@de_conf)
+          result = client.query(@query)
+          return result.first['bytesize'].to_i
+        ensure
+          client.close rescue nil
+        end
+      end
+    end
+  end
+end