flydata 0.2.29 → 0.2.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe45a465ce979b91e41ebfa3993cdd0b136de6bf
-  data.tar.gz: a929728a274a8fa8af440a77333f155c1a8d1442
+  metadata.gz: acad4ae7202b7fc525478de91925ef4ee0a24fdd
+  data.tar.gz: 63cf3ac658afa5be37ed7b2a85be476de280c56f
 SHA512:
-  metadata.gz: 37477dc1882b8faf6ad42459de4de796e0b3f78151d3ac095547fc9c9817c05ae72251962e71fb21f31d1d6eb9ea86fa9b2580b15b9106351030c8ef7d1f687a
-  data.tar.gz: 70832d4d94503232338e09e3f3f7eecceea1a96dab7d62b2cd17418c97dcf6b31c750918040c4e177b9124961e38deab6156f8221126939b6c8d43d7df820fe6
+  metadata.gz: fb7391185d4f9797817f0628ec46f5b9cce0ef1680fb843d0181f97dd18450c059d6b6614b7ef98e8ee88dfac171933561e448a2ca1b403665563b27b5d005a8
+  data.tar.gz: 7407fd538bbd7678ae19d5ddaa9dcc5b13159555b8c7b82a788c821fb70855594f49848eb141ec7a44982eb164dd63738e63a9714ed132b5b02ff2caab948d4f

data/VERSION

@@ -1 +1 @@
-0.2.29
+0.2.30

data/flydata-core/lib/flydata-core/logger.rb

@@ -196,10 +196,11 @@ module FlydataCore
 
     # Log an error for the given exception.  It does not log a RetryableError if
     # retry_count is less than retry_alert_limit.
-    # It returns the original exception of a RetryableError it has one.
+    # It logs a DataDeliverError as INFO.
+    # It returns the original exception of a RetryableError if it has one.
     # This method requires method 'logger' which returns a Ruby Logger instance or
     # a compatible one.
-    def log_retryable_error(exception, message = "unknown error occured.", retry_count = -1,
+    def log_flydata_error(exception, message = nil, retry_count = -1,
                   retry_alert_limit = 13)
       is_retryable_error = false
       if (exception.kind_of?(FlydataCore::RetryableError))
@@ -209,9 +210,19 @@ module FlydataCore
       end
 
       unless is_retryable_error && retry_count < retry_alert_limit
-        log_error_with_backtrace(message, retry_count: retry_count, error: exception)
+        if exception.kind_of?(FlydataCore::DataDeliveryError)
+          # Do not log a DataDeliveryError as ERROR.  It's a user error which
+          # should not cause a system alert
+          message ||= "a DataDeliveryError occured."
+          log_debug(message, {retry_count: retry_count, error: exception}, {backtrace: true})
+        else
+          message ||= "an unknown error occured."
+          log_error_with_backtrace(message, retry_count: retry_count, error: exception)
+        end
       end
       exception
     end
+    # log_retryable_error for backward compatibility
+    alias_method :log_retryable_error, :log_flydata_error
   end
 end

data/flydata-core/spec/logger_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+require 'flydata-core/logger'
+
+module FlydataCore
+
+describe Logger do
+  subject { d = double("logger"); d.extend Logger; d}
+  shared_examples "log_flydata_error" do
+    let(:message) { "my error" }
+    context "with a regular exception" do
+      let(:exception) { StandardError.new("stde") }
+      it do
+        expect(subject).to receive(:log_error_with_backtrace).with(message, {retry_count: -1, error: exception})
+
+        expect(subject.send(method, exception, message)).to eq(exception)
+      end
+      context "default message" do
+        it do
+          expect(subject).to receive(:log_error_with_backtrace).with("an unknown error occured.", {retry_count: -1, error: exception})
+
+          expect(subject.send(method, exception)).to eq(exception)
+        end
+      end
+    end
+    context "with a RetryableError" do
+      let(:original_exception) { StandardError.new("this will go away soon") }
+      let(:exception) { RetryableError.new(original_exception) }
+      it do
+        expect(subject).to receive(:log_debug).never
+        expect(subject).to receive(:log_error_with_backtrace).never
+
+        expect(subject.send(method, exception, message)).to eq original_exception
+      end
+    end
+    context "with a DataDeliveryError" do
+      let(:exception) { TableMissingError.new("tse") }
+      it do
+        expect(subject).to receive(:log_debug).with(message, {retry_count: -1, error: exception}, {backtrace: true})
+
+        expect(subject.send(method, exception, message)).to eq exception
+      end
+      context "default message" do
+        it do
+          expect(subject).to receive(:log_debug).with("a DataDeliveryError occured.", {retry_count: -1, error: exception}, {backtrace: true})
+
+          expect(subject.send(method, exception)).to eq exception
+        end
+      end
+    end
+    context "with a RetryableError wrapping a DataDeliveryError" do
+      let(:original_exception) { TableMissingError.new("no table info") }
+      let(:exception) { RetryableError.new(original_exception) }
+      it do
+        expect(subject).to receive(:log_debug).never
+        expect(subject).to receive(:log_error_with_backtrace).never
+
+        expect(subject.send(method, exception, message)).to eq original_exception
+      end
+    end
+  end
+  describe "#log_flydata_error" do
+    let(:method) { :log_flydata_error }
+    it_behaves_like "log_flydata_error"
+  end
+
+  describe "#log_retryable_error" do
+    let(:method) { :log_retryable_error }
+    it_behaves_like "log_flydata_error"
+  end
+end
+
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.29 ruby lib
+# stub: flydata 0.2.30 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.29"
+  s.version = "0.2.30"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-01-23"
+  s.date = "2015-01-26"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -48,6 +48,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
     "flydata-core/lib/flydata-core/thread_context.rb",
+    "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/spec_helper.rb",
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
@@ -121,6 +122,7 @@ Gem::Specification.new do |s|
     "lib/flydata/proxy.rb",
     "lib/flydata/sync_file_manager.rb",
     "lib/flydata/util/encryptor.rb",
+    "lib/flydata/util/mysql_util.rb",
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
@@ -140,6 +142,7 @@ Gem::Specification.new do |s|
     "spec/flydata/parser/mysql/dump_parser_spec.rb",
     "spec/flydata/sync_file_manager_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
+    "spec/flydata/util/mysql_util_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
     "tmpl/redshift_mysql_data_entry.conf.tmpl"

data/lib/flydata/command/sender.rb

@@ -30,6 +30,8 @@ module Flydata
         dp = flydata.data_port.get
         AgentCompatibilityCheck.new(dp).check
 
+        Flydata::Command::Sync.new.try_mysql_sync
+
         # Start sender(fluentd) process
         log_info_stdout("Starting sender process.") unless options[:quiet]
         Dir.chdir(FLYDATA_HOME){
@@ -84,16 +86,17 @@ EOF
       def restart(options = {})
         if process_exist?
           log_info_stdout('Restarting sender process.') unless options[:quiet]
-          if Kernel.system("kill -HUP `cat #{pid_file}`")
-            log_info_stdout('Done.') unless options[:quiet]
-            return true
-          else
-            raise 'Something has gone wrong..'
+          begin
+            stop(options)
+          rescue
+            log_out_stderr 'Something has gone wrong... force stopping the FlyData process...'
+            kill_all(options)
           end
         else
           log_info_stdout("Process doesn't exist.") unless options[:quiet]
-          start(options)
         end
+        start(options)
+        say('Process started.') unless options[:quiet]
       end
       def status
         if process_exist?
@@ -106,7 +109,7 @@ EOF
         # Returns true if the process is running
         `[ -f #{pid_file} ] && pgrep -P \`cat #{pid_file}\``.to_i > 0
       end
-      def kill_all(optiosn = {})
+      def kill_all(options = {})
         if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
           log_info_stdout("Done.") unless options[:quiet]
           return true

data/lib/flydata/command/sync.rb

@@ -7,6 +7,7 @@ require 'flydata/sync_file_manager'
 require 'flydata/compatibility_check'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
+require 'flydata/util/mysql_util'
 require 'flydata-core/table_def'
 #require 'ruby-prof'
 
@@ -21,6 +22,11 @@ module Flydata
       STATUS_WAITING = 'WAITING'
       STATUS_COMPLETE = 'COMPLETE'
 
+      attr_reader :initial_sync, :new_tables, :ddl_tables
+
+      class SyncDataEntryError < StandardError
+      end
+
       def self.slop
         Slop.new do
           on 'c', 'skip-cleanup', 'Skip server cleanup'
@@ -44,11 +50,61 @@ module Flydata
           exit 1
         end
 
-        de = retrieve_data_entry
-        de = load_sync_info(override_tables(de, tables))
+        handle_mysql_sync(tables)
+
+        unless opts.no_flydata_start?
+          log_info_stdout("Starting FlyData Agent...")
+          Flydata::Command::Sender.new.start(quiet: true)
+          log_info_stdout("  -> Done")
+        end
+
+        dashboard_url = "#{flydata.flydata_api_host}/dashboard"
+        redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
+        last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
+        log_info_stdout(last_message)
+      end
+
+      def try_mysql_sync
+        handle_mysql_sync()
+      rescue SyncDataEntryError
+        return
+      end
+
+      def handle_mysql_sync(tables=nil)
+        de = retrieve_sync_data_entry
+
+        set_current_tables(de)
+        verify_input_tables(tables, de['mysql_data_entry_preference']['tables'])
+
+        unless @new_tables.empty?
+          say("We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}")
+          unless @ddl_tables.empty?
+            say("  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.")
+            say("    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'")
+            say("    to generate SQL to run on Redshift to create the correct tables")
+            say("    Without running this sql on your Redshift cluster, there may be issues with your data")
+          end
+          if ask_yes_no("Do you want to run initial sync on all of these tables now?")
+            tables = @initial_sync ? [] : @new_tables
+            initial_sync(tables, de)
+          else
+            #If generate_table_ddl has not been run for these tables, warn user
+            unless @ddl_tables.empty?
+              say("  You can generate DDL SQL for your new tables by running this command")
+              say("    $> flydata sync:generate_table_ddl > create_table.sql")
+            end
+            puts "Without syncing these tables, we cannot start the flydata process"
+            raise "Please try again"
+          end
+        end
+      end
+      def initial_sync(tables, de=nil)
+        de = retrieve_sync_data_entry unless de
+        de = load_sync_info(de,tables)
         validate_initial_sync_status(de, tables)
-        flush_buffer_and_stop  unless de['mysql_data_entry_preference']['initial_sync']
+        flush_buffer_and_stop  unless @initial_sync
         sync_mysql_to_redshift(de)
+        complete
       end
 
       def self.slop_flush
@@ -76,7 +132,10 @@ module Flydata
         sender.flush_client_buffer # TODO We should rather delete buffer files
         sender.stop
 
-        de = retrieve_data_entry
+        de = retrieve_sync_data_entry
+        all_tables = de['mysql_data_entry_preference']['tables']
+        verify_input_tables(tables, all_tables)
+
         wait_for_server_buffer
         cleanup_sync_server(de, tables) unless opts.client?
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
@@ -88,9 +147,10 @@ module Flydata
           sync_fm.table_position_file_paths(*tables),
           sync_fm.table_binlog_pos_paths(*tables),
           sync_fm.table_binlog_pos_init_paths(*tables),
-          sync_fm.table_rev_file_paths(*tables)
+          sync_fm.table_rev_file_paths(*tables),
+          sync_fm.table_ddl_file_paths(*tables)
         ]
-        delete_files << sync_fm.binlog_path if tables.empty?
+        delete_files << sync_fm.binlog_path if tables.empty? or all_tables.empty?
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
         end
@@ -129,7 +189,7 @@ module Flydata
       end
 
       def check
-        de = retrieve_data_entry
+        de = retrieve_sync_data_entry
         retry_on(RestClient::Exception) do
           status = do_check(de)
           if status['complete']
@@ -142,7 +202,7 @@ module Flydata
 
       # skip initial sync
       def skip
-        de = retrieve_data_entry
+        de = retrieve_sync_data_entry
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         binlog_path = sync_fm.binlog_path
         sync_fm.close
@@ -157,18 +217,39 @@ module Flydata
           on 'c', 'ctl-only', 'Only generate FlyData Control definitions'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 's', 'skip-primary-key-check', 'Skip primary key check when generating DDL'
+          on 'all-tables', 'Generate all table schema'
         end
       end
 
       def generate_table_ddl(*tables)
-        de = retrieve_data_entry
+        de = retrieve_sync_data_entry
         dp = flydata.data_port.get
         Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
-        do_generate_table_ddl(override_tables(de, tables))
+        do_generate_table_ddl(de, tables)
       end
 
       private
 
+      def verify_input_tables(input_tables, all_tables)
+        return unless input_tables
+        inval_table = []
+        input_tables.each do |tab|
+          inval_table << tab unless all_tables.include?(tab)
+        end
+        raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
+      end
+
+      def set_current_tables(de)
+        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        full_tables = de['mysql_data_entry_preference']['tables']
+
+        @new_tables = sync_fm.get_new_table_list(full_tables, "pos")
+        @ddl_tables = sync_fm.get_new_table_list(full_tables, "generated_ddl")
+
+        @initial_sync = (@new_tables == full_tables)
+        sync_fm.close
+      end
+
       def validate_initial_sync_status(de, tables)
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         dump_pos_info = sync_fm.load_dump_pos
@@ -181,17 +262,19 @@ module Flydata
         end
       end
 
-      def retrieve_data_entry
-        de = retrieve_data_entries.first
-        raise "There are no data entry." unless de
+      def retrieve_sync_data_entry
+        de = retrieve_data_entries.first unless de
+        raise "There are no data entries." unless de
         case de['type']
         when 'RedshiftMysqlDataEntry'
           mp = de['mysql_data_entry_preference']
           if mp['tables_append_only']
-            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq.join(",")
+            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq
+          else
+            mp['tables'] = mp['tables'].split(",").uniq
           end
         else
-          raise "No supported data entry. Only mysql-redshift sync is supported."
+          raise SyncDataEntryError, "No supported data entry. Only mysql-redshift sync is supported."
         end
         de
       end
@@ -223,8 +306,7 @@ module Flydata
         log_info_stdout(message) unless message.nil? || message.empty?
       end
 
-      DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"
-      def do_generate_table_ddl(de)
+      def do_generate_table_ddl(de, tables=[])
         if `which mysqldump`.empty?
           raise "mysqldump is not installed.  mysqldump is required to run the command"
         end
@@ -234,20 +316,25 @@ module Flydata
         schema_name = (de['schema_name'] || nil)
 
         mp = de['mysql_data_entry_preference']
-        params = []
-        params << mp['password']
-        if !mp['host'].empty? then params << mp['host'] else raise "MySQL `host` is neither defined in the data entry nor the local config file" end
-        params << (mp['port'] or '3306')
-        if !mp['username'].empty? then params << mp['username'] else raise "MySQL `username` is neither defined in the data entry nor the local config file" end
-        if !mp['database'].empty? then params << mp['database'] else raise "`database` is neither defined in the data entry nor the local config file" end
-        if !mp['tables'].empty? then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file" end
 
-        command = DDL_DUMP_CMD_TEMPLATE % params
+        set_current_tables(de)
+
+        tables = opts.all_tables? ? mp['tables'] : (tables.empty? ? @new_tables : tables)
+        raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
+
+        %w(host username database).each do |conf_name|
+          raise "MySQL `#{conf_name}` is neither defined in the data entry nor the local config file" if mp[conf_name].to_s.empty?
+        end
+        if tables.empty?
+          raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
+        end
+
+        command = Util::MysqlUtil.generate_mysql_ddl_dump_cmd(mp.merge(tables: tables))
 
         Open3.popen3(command) do |stdin, stdout, stderr|
           stdin.close
           stdout.set_encoding("utf-8") # mysqldump output must be in UTF-8
-          create_flydata_ctl_table = mp['initial_sync']
+          create_flydata_ctl_table = @initial_sync
           while !stdout.eof?
             begin
               mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, skip_primary_key_check: opts.skip_primary_key_check?)
@@ -281,6 +368,11 @@ module Flydata
           end
           log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
         end
+        tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
+
+        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        sync_fm.mark_generated_tables(tables_without_error)
+        sync_fm.close
       end
 
       def sync_mysql_to_redshift(de)
@@ -288,7 +380,7 @@ module Flydata
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
 
         # Check client condition
-        if File.exists?(sync_fm.binlog_path) and de['mysql_data_entry_preference']['initial_sync']
+        if File.exists?(sync_fm.binlog_path) and @initial_sync
           raise "Already synchronized. If you want to do initial sync, run 'flydata sync:reset'"
         end
 
@@ -296,14 +388,11 @@ module Flydata
         unless Flydata::Preference::DataEntryPreference.conf_exists?(de)
           Flydata::Command::Conf.new.copy_templates
         end
-
         generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, db_bytesize|
-          sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
+          sync_fm.save_sync_info(@initial_sync, de['mysql_data_entry_preference']['tables'])
           parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
         end
         wait_for_mysqldump_processed(dp, de, sync_fm)
-        sync_fm.close
-        complete
       end
 
       def generate_mysqldump(de, sync_fm, file_dump = true, &block)
@@ -333,12 +422,12 @@ module Flydata
 
         confirmation_text = <<-EOM
 
-FlyData Sync will start synchonizing the following database tables
+FlyData Sync will start synchronizing the following database tables
   host:         #{de['mysql_data_entry_preference']['host']}
   port:         #{de['mysql_data_entry_preference']['port']}
   username:     #{de['mysql_data_entry_preference']['username']}
   database:     #{de['mysql_data_entry_preference']['database']}
-  tables:       #{tables}#{data_servers}
+  tables:       #{tables.join(", ")}#{data_servers}
 EOM
         confirmation_text << <<-EOM if file_dump
   dump file:    #{fp}
@@ -349,6 +438,8 @@ EOM
         log_info confirmation_text.strip
 
         if ask_yes_no('Start Sync?')
+          log_info_stdout("Checking MySQL server connection and configuration...")
+          Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
           log_info_stdout("Checking database size...")
 
           db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
@@ -370,7 +461,6 @@ EOM
 
           log_info_stdout("Exporting data from database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
-          Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
           if file_dump
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
               new(de['mysql_data_entry_preference']).dump(fp)
@@ -448,7 +538,7 @@ EOM
           log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
         else
           #If its a new sync, ensure server side resources are clean
-          cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(',')) unless opts.skip_cleanup?
+          cleanup_sync_server(de, de['mysql_data_entry_preference']['tables']) unless opts.skip_cleanup?
         end
         log_info_stdout("Sending data to FlyData Server...")
 
@@ -517,7 +607,7 @@ EOM
         binlog_pos = dump_pos_info[:binlog_pos]
 
         wait_for_server_data_processing
-        tables = de['mysql_data_entry_preference']['tables'].split(',').join(' ')
+        tables = de['mysql_data_entry_preference']['tables']
         sync_fm.save_table_binlog_pos(tables, binlog_pos)
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
       end
@@ -539,26 +629,17 @@ Thank you for using FlyData!
       EOM
 
       def complete
-        de = load_sync_info(retrieve_data_entry)
+        de = load_sync_info(retrieve_sync_data_entry)
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE
-          if de['mysql_data_entry_preference']['initial_sync']
+          if @initial_sync
             sync_fm.save_binlog(info[:binlog_pos])
           end
-          sync_fm.install_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
-          sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
+          sync_fm.install_table_binlog_files(de['mysql_data_entry_preference']['tables'])
+          sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'])
           sync_fm.delete_dump_file
           sync_fm.backup_dump_dir
-          unless opts.no_flydata_start?
-            log_info_stdout("Starting FlyData Agent...")
-            Flydata::Command::Sender.new.start(quiet: true)
-            log_info_stdout("  -> Done")
-          end
-          dashboard_url = "#{flydata.flydata_api_host}/dashboard"
-          redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
-          last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
-          log_info_stdout(last_message)
         else
           raise "Initial sync status is not complete. Try running 'flydata sync'."
         end
@@ -573,20 +654,14 @@ Thank you for using FlyData!
         h.to_json
       end
 
-      def override_tables(de, tables)
-        de['mysql_data_entry_preference']['initial_sync'] = tables.empty?
-        if ! de['mysql_data_entry_preference']['initial_sync']
-          de['mysql_data_entry_preference']['tables'] = tables.join(',')
-        end
-        de
-      end
-
-      def load_sync_info(de)
+      def load_sync_info(de ,additional_tables=[])
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         mp = de['mysql_data_entry_preference']
         unless (rs = sync_fm.load_sync_info).nil?
-          mp['initial_sync'] = rs[:initial_sync]
+          @initial_sync = rs[:initial_sync]
           mp['tables'] = rs[:tables]
+        else
+          mp['tables'] = additional_tables unless @initial_sync
         end
         sync_fm.close
         de

data/lib/flydata/compatibility_check.rb

@@ -1,4 +1,6 @@
+require 'mysql2'
 require 'flydata/command_logger'
+require 'flydata/util/mysql_util'
 
 module Flydata
 
@@ -118,7 +120,7 @@ module Flydata
     end
 
     def check_mysql_protocol_tcp_compat
-      query = "MYSQL_PWD=\"#{@db_opts[:password]}\" mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
+      query = Util::MysqlUtil.generate_mysql_show_grants_cmd(@db_opts)
 
       Open3.popen3(query) do |stdin, stdout, stderr|
         stdin.close

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -9,6 +9,7 @@ lib = File.absolute_path(File.dirname(__FILE__) + '/../..')
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'flydata'
 require 'flydata/sync_file_manager'
+require 'flydata/util/mysql_util'
 
 require_relative 'preference'
 require_relative 'mysql/binlog_position_file'
@@ -63,7 +64,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     load_custom_conf
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
-    server_version = `echo 'select version();' | MYSQL_PWD="#{@password}" mysql -h #{@host} --port #{@port} -u #{@username} 2>/dev/null`
+    server_version = `echo 'select version();' | #{Flydata::Util::MysqlUtil.generate_mysql_cmd(host: @host, port: @port, username: @username, password: @password)} 2>/dev/null`
     $log.info "mysql server version: #{server_version}"
 
     @tables = @tables.split(/,\s*/)

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,4 +1,5 @@
 require 'fiber'
+require 'flydata/util/mysql_util'
 
 module Flydata
   module Parser
@@ -39,26 +40,20 @@ module Flydata
       end
 
       class MysqlDumpGenerator
-        # host, port, username, password, database, tables
-        MYSQL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --default-character-set=utf8 --protocol=tcp -h %s -P %s -u%s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
-        EXTRA_MYSQLDUMP_PARAMS = ""
         def initialize(conf)
-          tables = if conf['tables']
-                     conf['tables'].split(',').join(' ')
-                   else
-                     ''
-                   end
-          @dump_cmd = MYSQL_DUMP_CMD_TEMPLATE %
-            [conf['password'], conf['host'], conf['port'], conf['username'], self.class::EXTRA_MYSQLDUMP_PARAMS, conf['database'], tables]
+          @dump_cmd = generate_dump_cmd(conf)
           @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
         end
         def dump(file_path)
           raise "subclass must implement the method"
         end
+
+        def generate_dump_cmd(conf)
+          raise "subclass must implement the method"
+        end
       end
 
       class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
-        EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
         def dump(file_path)
           cmd = "#{@dump_cmd} -r #{file_path}"
           o, e, s = Open3.capture3(cmd)
@@ -78,10 +73,13 @@ module Flydata
           end
           true
         end
+
+        def generate_dump_cmd(conf)
+          Util::MysqlUtil.generate_mysqldump_with_master_data_cmd(conf)
+        end
       end
 
       class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
-        EXTRA_MYSQLDUMP_PARAMS = ""
         CHANGE_MASTER_TEMPLATE = <<EOS
 --
 -- Position to start replication or point-in-time recovery from
@@ -164,6 +162,10 @@ EOS
           end
         end
 
+        def generate_dump_cmd(conf)
+          Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf)
+        end
+
         private
 
         def open_file_stream(file_path, &block)
@@ -596,7 +598,7 @@ EOT
 
         def initialize(de_conf)
           @de_conf = de_conf
-          @tables = de_conf['tables'].split(',')
+          @tables = de_conf['tables']
           @query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
         end
 

data/lib/flydata/sync_file_manager.rb

@@ -40,6 +40,26 @@ module Flydata
           state: items[5], substate: items[6], mysql_table: mysql_table}
       end
 
+      def mark_generated_tables(tables)
+        table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
+        #Create positions if dir does not exist
+        unless File.directory?(table_positions_dir_path)
+          FileUtils.mkdir_p(table_positions_dir_path)
+        end
+        tables.each do |tab|
+          File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write("1") }
+        end
+      end
+
+      def get_new_table_list(tables, file_type)
+        table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
+        new_tables = []
+        tables.each do |table|
+          new_tables << table unless  File.exists?(File.join(table_positions_dir_path, "#{table}.#{file_type}"))
+        end
+        new_tables
+      end
+
       # MysqlTable marshal file
       def mysql_table_marshal_dump_path
         dump_file_path + ".mysql_table"
@@ -79,6 +99,11 @@ module Flydata
           tables.map{|table| File.join(table_positions_dir_path, table + '.pos')}
       end
 
+      def table_ddl_file_paths(*tables)
+        tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.generated_ddl')) :
+          tables.map{|table| File.join(table_positions_dir_path, table + '.generated_ddl')}
+      end
+
       def table_binlog_pos_paths(*tables)
         tables.empty? ? Dir.glob(File.join(table_positions_dir_path, '*.binlog.pos')) :
           tables.map{|table| File.join(table_positions_dir_path, table + '.binlog.pos')}
@@ -128,7 +153,7 @@ module Flydata
 
       def save_sync_info(initial_sync, tables)
         File.open(sync_info_file, "w") do |f|
-          f.write([initial_sync, tables].join("\t"))
+          f.write([initial_sync, tables.join(" ")].join("\t"))
         end
       end
 
@@ -136,7 +161,7 @@ module Flydata
         return nil unless File.exists?(sync_info_file)
         items = File.open(sync_info_file, 'r').readline.split("\t")
         { initial_sync: (items[0] == 'true'),
-          tables: items[1] }
+          tables: items[1].split(" ") }
       end
 
       def get_table_binlog_pos(table_name)
@@ -186,7 +211,7 @@ module Flydata
       end
 
       def save_table_binlog_pos(tables, binlog_pos)
-        tables.split(" ").each do |table_name|
+        tables.each do |table_name|
           file = File.join(dump_dir, table_name + ".binlog.pos")
           File.open(file, "w") do |f|
             f.write(binlog_content(binlog_pos))

data/lib/flydata/util/mysql_util.rb

@@ -0,0 +1,79 @@
+module Flydata
+  module Util
+    class MysqlUtil
+      DEFAULT_MYSQL_CMD_OPTION = "--default-character-set=utf8 --protocol=tcp"
+
+      # Generate mysql/mysqldump command with options
+      # options must be hash
+      # - command # mysql(default) | mysqldump
+      # - host
+      # - port
+      # - username
+      # - password
+      # - database
+      # - tables  # array
+      # - custom_option   # string
+      def self.generate_mysql_cmd(option)
+        raise ArgumentError.new("option must be hash.") unless option.kind_of?(Hash)
+        option = convert_keys_to_sym(option)
+        command = option[:command] ? option[:command] : 'mysql'
+        host = option[:host] ? "-h #{option[:host]}" : nil
+        port = option[:port] ? "-P #{option[:port]}" : nil
+        username = option[:username] ? "-u#{option[:username]}" : nil
+        password = if !(option[:password].to_s.empty?)
+                     "-p\"#{option[:password].gsub('`', '\\\`')}\""
+                   else
+                     nil
+                   end
+        database = option[:database]
+        tables = option[:tables] ? option[:tables].join(' ') : nil
+        default_option = option[:no_default_option] ? nil : DEFAULT_MYSQL_CMD_OPTION
+        custom_option = option[:custom_option]
+
+        [command, host, port, username, password, default_option,
+         custom_option, database, tables].compact.join(' ')
+      end
+
+      # DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"
+      def self.generate_mysql_ddl_dump_cmd(option)
+        opt = option.dup
+        opt[:command] = 'mysqldump'
+        opt[:custom_option] = '-d'
+        generate_mysql_cmd(opt)
+      end
+
+      # MYSQL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --default-character-set=utf8 --protocol=tcp -h %s -P %s -u%s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
+      def self.generate_mysqldump_with_master_data_cmd(option)
+        opt = option.dup
+        opt[:command] = 'mysqldump'
+        opt[:custom_option] = '--skip-lock-tables --single-transaction --hex-blob --flush-logs --master-data=2'
+        generate_mysql_cmd(opt)
+      end
+
+      def self.generate_mysqldump_without_master_data_cmd(option)
+        opt = option.dup
+        opt[:command] = 'mysqldump'
+        opt[:custom_option] = '--skip-lock-tables --single-transaction --hex-blob'
+        generate_mysql_cmd(opt)
+      end
+
+      def self.generate_mysql_show_grants_cmd(option)
+        opt = option.dup
+        opt[:command] = 'mysql'
+        opt[:custom_option] = '-e "SHOW GRANTS;"'
+        generate_mysql_cmd(opt)
+      end
+
+      private
+
+      def self.convert_keys_to_sym(hash)
+        hash.inject(hash.dup) do |ret, (k, v)|
+          if k.kind_of?(String) && ret[k.to_sym].nil?
+            ret[k.to_sym] = v
+          end
+          ret
+        end
+      end
+    end
+  end
+end

data/spec/flydata/command/sender_spec.rb

@@ -18,11 +18,12 @@ module Flydata
           allow(Kernel).to receive(:sleep)
           allow_any_instance_of(Flydata::Api::DataPort).to receive(:get).and_return("Wibble")
           allow_any_instance_of(Flydata::AgentCompatibilityCheck).to receive(:check).and_return(true)
+          Flydata::Command::Sync.any_instance.should_receive(:handle_mysql_sync).and_return("Wobble")
         end
 
         context "as daemon" do
           let(:args) { [] }
-          it "starts fluend with daemon option" do
+          it "starts fluentd with daemon option" do
             expect(Kernel).to receive(:system).with( Regexp.new(
 "bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with( Regexp.new(

data/spec/flydata/command/sync_spec.rb

@@ -29,7 +29,7 @@ module Flydata
         "data_port_key"=>"a458c641",
         "mysql_data_entry_preference" =>
           { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
-            "password"=>"welcome", "database"=>"sync_test", "tables"=>"table1, table2",
+            "password"=>"welcome", "database"=>"sync_test", "tables"=>["table1", " table2"],
             "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
             "data_servers"=>"localhost:9905" }
          }
@@ -75,7 +75,7 @@ module Flydata
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
             expect(Open3).to receive(:popen3).with(
-             'MYSQL_PWD="welcome" mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -97,7 +97,7 @@ module Flydata
           end
           it "uses the default port" do
             expect(Open3).to receive(:popen3).with(
-             'MYSQL_PWD="welcome" mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1  table2')
+             'mysqldump -h localhost -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -107,7 +107,7 @@ module Flydata
           end
           it "uses the specified port" do
             expect(Open3).to receive(:popen3).with(
-             'MYSQL_PWD="welcome" mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi sync_test table1  table2')
+             'mysqldump -h localhost -P 1234 -umasashi -p"welcome" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -129,7 +129,7 @@ module Flydata
           end
           it "call mysqldump without MYSQL_PW set" do
             expect(Open3).to receive(:popen3).with(
-             'MYSQL_PWD="" mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -140,7 +140,7 @@ module Flydata
           end
           it "call mysqldump with MYSQL_PW set with correct symbols" do
             expect(Open3).to receive(:popen3).with(
-             'MYSQL_PWD="welcome&!@^@#^" mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1  table2')
+             'mysqldump -h localhost -P 3306 -umasashi -p"welcome&!@^@#^" --default-character-set=utf8 --protocol=tcp -d sync_test table1  table2')
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -24,7 +24,7 @@ module Flydata
         let(:default_conf) do
           {
             'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
-            'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
+            'password' => 'pass', 'database' => 'dev', 'tables' => ['users','groups'],
           }
         end
 
@@ -44,6 +44,12 @@ module Flydata
             m
           end
 
+          describe '#initialize' do
+            subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
+            it { is_expected.to eq('mysqldump -h localhost -P 3306 -uadmin -p"pass" --default-character-set=utf8 --protocol=tcp --skip-lock-tables ' +
+                                   '--single-transaction --hex-blob dev users groups') }
+          end
+
           describe '#dump' do
             before do
               expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
@@ -84,7 +90,7 @@ module Flydata
           describe '#initialize' do
             context 'with password' do
               subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
-              it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
+              it { is_expected.to eq('mysqldump -h localhost -P 3306 -uadmin -p"pass" --default-character-set=utf8 --protocol=tcp --skip-lock-tables ' +
                                      '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
             end
             context 'without password' do
@@ -92,7 +98,7 @@ module Flydata
                 MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
               end
               subject { dump_generator.instance_variable_get(:@dump_cmd) }
-              it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
+              it { is_expected.to eq('mysqldump -h localhost -P 3306 -uadmin --default-character-set=utf8 --protocol=tcp --skip-lock-tables ' +
                                      '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
             end
           end

data/spec/flydata/util/mysql_util_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+module Flydata
+  module Util
+
+    describe MysqlUtil do
+      let(:default_option) { {
+        command: 'mysqldump',
+        host: 'test-host',
+        port: 3306,
+        username: 'testuser',
+        password: 'testpassword',
+        database: 'testdb',
+      } }
+      let(:option) { default_option }
+
+      describe '.generate_mysql_cmd' do
+        subject { described_class.generate_mysql_cmd(option) }
+
+        context 'when option is not hash' do
+          let(:option) { nil }
+          it { expect{subject}.to raise_error(ArgumentError) }
+        end
+
+        context 'when option is empty' do
+          let(:option) { {} }
+          it { is_expected.to eq("mysql --default-character-set=utf8 --protocol=tcp") }
+        end
+
+        context 'when all option is specified' do
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp testdb'
+          ) }
+        end
+
+        context 'when option conatins string key' do
+          before {
+            option[:database] = nil
+            option['database'] = 'another_db'
+          }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp another_db'
+          ) }
+        end
+
+        context 'when password is empty' do
+          before { option[:password] = nil }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser --default-character-set=utf8 --protocol=tcp testdb'
+          ) }
+        end
+
+        context 'when password is empty' do
+          before { option[:password] = "abc`def" }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"abc\\`def" --default-character-set=utf8 --protocol=tcp testdb'
+          ) }
+        end
+
+        context 'when tables are specified' do
+          before { option[:tables] = %w(table_a table_b table_c) }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp testdb table_a table_b table_c'
+          ) }
+        end
+
+        context 'when no_default_option is turned on' do
+          before { option[:no_default_option] = true }
+          it { is_expected.to eq(
+            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" testdb'
+          ) }
+        end
+
+
+        context 'with options for mysql_protocol_tcp_compat' do
+          before {
+            option[:command] = 'mysql'
+            option[:custom_option] = '-e "SHOW GRANTS;"'
+          }
+          it { is_expected.to eq(
+            'mysql -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -e "SHOW GRANTS;" testdb'
+          ) }
+        end
+      end
+
+      describe '.generate_mysql_ddl_dump_cmd' do
+        subject { described_class.generate_mysql_ddl_dump_cmd(option) }
+
+        it { is_expected.to eq(
+          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -d testdb'
+        ) }
+      end
+
+      describe '.generate_mysqldump_with_master_data_cmd' do
+        subject { described_class.generate_mysqldump_with_master_data_cmd(option) }
+        before { option[:tables] = %w(table_a table_b) }
+        it { is_expected.to eq(
+          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob --flush-logs --master-data=2 testdb table_a table_b'
+        ) }
+      end
+
+      describe '.generate_mysqldump_without_master_data_cmd' do
+        subject { described_class.generate_mysqldump_without_master_data_cmd(option) }
+        before { option[:tables] = %w(table_a table_b) }
+        it { is_expected.to eq(
+          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob testdb table_a table_b'
+        ) }
+      end
+
+      describe '.generate_mysql_show_grants_cmd' do
+        subject { described_class.generate_mysql_show_grants_cmd(option) }
+        it { is_expected.to eq(
+          'mysql -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -e "SHOW GRANTS;" testdb'
+        ) }
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.29
+  version: 0.2.30
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-23 00:00:00.000000000 Z
+date: 2015-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -451,6 +451,7 @@ files:
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
+- flydata-core/spec/logger_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
@@ -524,6 +525,7 @@ files:
 - lib/flydata/proxy.rb
 - lib/flydata/sync_file_manager.rb
 - lib/flydata/util/encryptor.rb
+- lib/flydata/util/mysql_util.rb
 - spec/fluent_plugins_spec_helper.rb
 - spec/fly_data_model_spec.rb
 - spec/flydata/api/data_entry_spec.rb
@@ -543,6 +545,7 @@ files:
 - spec/flydata/parser/mysql/dump_parser_spec.rb
 - spec/flydata/sync_file_manager_spec.rb
 - spec/flydata/util/encryptor_spec.rb
+- spec/flydata/util/mysql_util_spec.rb
 - spec/flydata_spec.rb
 - spec/spec_helper.rb
 - tmpl/redshift_mysql_data_entry.conf.tmpl