flydata 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3eda267f32da9fd2ce6f6dd3ad583b2803d689ce
-  data.tar.gz: 43876505915eee1ef232973a23de0d432afcb7e3
+  metadata.gz: 3aabc1598aa6770f9f4238bae95ef42b1577c0aa
+  data.tar.gz: 5ba322b53b34ffaf0986542a8071c103c4d2d881
 SHA512:
-  metadata.gz: 927ee474b1b8567b25667d24e2bd876bbde521192566f59fd24c53c82ea1fe181fea48087183490674feaa283139f0379af481a07d6a6558720a719fe3ee39b4
-  data.tar.gz: 17dcdec903a1a1c9bf137005b57495f272655c291103f377005ca8cf30208ecb7dba68ed0a564f1a2c11556764e0ddab545277c990439bc72e67fc2885226267
+  metadata.gz: a1b6059f259512e9d7f190f9bc37c51905979adab22310f4782e92b3ebbc7156673ce48259d7f7ba6a43ae2d40755b2836b59adffd3b36d035a8ceb0ff1b9c4b
+  data.tar.gz: 129df374f7bbba7ed5dc70eb23bd52b46ee683521cd391c7fa0c9440f9c56649d2761210be0eca31cb8d1c05c5ed55a9ba8aadacec84085d73460af1b950303f

data/VERSION

@@ -1 +1 @@
-0.3.4
+0.3.5

data/flydata.gemspec

@@ -2,14 +2,12 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.4 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.4"
+  s.version = "0.3.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
   s.date = "2015-02-28"
   s.description = "FlyData Agent"
@@ -152,7 +150,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/api/data_entry.rb

@@ -7,12 +7,13 @@ module Flydata
         super
       end
 
-      def buffer_stat(data_entry_id, mode = nil)
-        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{mode}")
+      def buffer_stat(data_entry_id, params = {})
+        tables = params[:tables] ? params[:tables].join(',') : ''
+        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{params[:mode]}?tables=#{tables}")
       end
-      
+
       def cleanup_sync(data_entry_id, tables)
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})      
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})
       end
     end
   end

data/lib/flydata/api_client.rb

@@ -63,7 +63,7 @@ module Flydata
       if json_response.class == Hash and json_response["success"] == false
         err_msg = json_response['errors'] ? json_response['errors'].to_s : "Unkown error."
         raise err_msg
-      elsif json_response.class == Hash and json_response["auth_token"] 
+      elsif json_response.class == Hash and json_response["auth_token"]
         @credentials.write_credentials(json_response["auth_token"])
       end
       json_response
@@ -73,10 +73,11 @@ module Flydata
       parameters = {}
       parameters.merge(headers) if headers
     end
-    def generate_auth_url(path) 
+    def generate_auth_url(path)
       token = @credentials.token
       token = '' unless token
-      "#{@flydata_api_host}#{path}?auth_token=#{token}"
+      c = (path && path.include?('?')) ? '&' : '?'
+      "#{@flydata_api_host}#{path}#{c}auth_token=#{token}"
     end
   end
 end

data/lib/flydata/command/sync.rb

@@ -24,11 +24,13 @@ module Flydata
       STATUS_WAITING = 'WAITING'
       STATUS_COMPLETE = 'COMPLETE'
 
-      attr_reader :initial_sync, :new_tables, :ddl_tables
+      attr_reader :full_initial_sync, :full_tables, :new_tables, :ddl_tables, :input_tables
 
       class SyncDataEntryError < StandardError
       end
 
+      # Command: flydata sync
+      #  - Arguments
       def self.slop
         Slop.new do
           on 'c', 'skip-cleanup', 'Skip server cleanup'
@@ -42,7 +44,10 @@ module Flydata
         end
       end
 
+      # Command: flydata sync
+      #  - Entry method
       def run(*tables)
+        # Process check
         sender = Flydata::Command::Sender.new
         if (sender.process_exist?)
           if tables.empty?
@@ -55,76 +60,40 @@ module Flydata
           exit 1
         end
 
-        handle_mysql_sync(tables)
+        # Setup instance variables
+        set_current_tables(tables)
 
+        # Start initial sync with check
+        handle_mysql_sync
+
+        # Start continuous sync by starting fluentd process
         unless opts.no_flydata_start?
           log_info_stdout("Starting FlyData Agent...")
           Flydata::Command::Sender.new.start(quiet: true)
           log_info_stdout("  -> Done")
         end
 
+        # Show message
         dashboard_url = "#{flydata.flydata_api_host}/dashboard"
         redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
         last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
         log_info_stdout(last_message)
       end
 
+      # Public method
+      #  - Called from Sender#start/restart
       def try_mysql_sync
-        handle_mysql_sync()
+        # Setup instance variables
+        set_current_tables
+
+        # Start initial sync
+        handle_mysql_sync
       rescue SyncDataEntryError
         return
       end
 
-      def handle_mysql_sync(tables=nil)
-        de = retrieve_sync_data_entry
-
-        set_current_tables(de)
-        verify_input_tables(tables, de['mysql_data_entry_preference']['tables'])
-
-        unless @new_tables.empty?
-          say("We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}")
-          unless @ddl_tables.empty?
-            say("  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.")
-            say("    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'")
-            say("    to generate SQL to run on Redshift to create the correct tables")
-            say("    Without running this sql on your Redshift cluster, there may be issues with your data")
-          end
-          if ask_yes_no("Do you want to run initial sync on all of these tables now?")
-            tables = @initial_sync ? [] : @new_tables
-            initial_sync(tables, de)
-          else
-            #If generate_table_ddl has not been run for these tables, warn user
-            unless @ddl_tables.empty?
-              say("  You can generate DDL SQL for your new tables by running this command")
-              say("    $> flydata sync:generate_table_ddl > create_table.sql")
-            end
-            puts "Without syncing these tables, we cannot start the flydata process"
-            raise "Please try again"
-          end
-        end
-      end
-      def initial_sync(tables, de=nil)
-        de = retrieve_sync_data_entry unless de
-        de = load_sync_info(de,tables)
-        validate_initial_sync_status(de, tables)
-        begin
-          flush_buffer_and_stop  unless @initial_sync
-          sync_mysql_to_redshift(de)
-        rescue ServerDataProcessingTimeout => e
-          ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
-          ee.description = <<EOS
-  Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
-  Once checked, you can resume sync with the following command.
-
-    flydata start
-
-EOS
-          ee.set_backtrace e.backtrace
-          raise ee
-        end
-        complete
-      end
-
+      # Command: flydata sync:flush
+      #  - Arguments
       def self.slop_flush
         Slop.new do
           on 'f', 'skip-flush', 'Skip server flush'
@@ -132,9 +101,11 @@ EOS
         end
       end
 
-      def flush
+      # Command: flydata sync:flush
+      #  - Entry method
+      def flush(*tables)
         begin
-          flush_buffer_and_stop
+          flush_buffer_and_stop(tables)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
@@ -147,6 +118,8 @@ EOS
         log_info_stdout("Buffers have been flushed and the sender process has been stopped.")
       end
 
+      # Command: flydata sync:reset
+      #  - Arguments
       def self.slop_reset
         Slop.new do
           on 'c', 'client', 'Resets client only.'
@@ -154,19 +127,22 @@ EOS
         end
       end
 
+      # Command: flydata sync:reset
+      #  - Entry method
       def reset(*tables)
+        # Flush client buffer
         msg = tables.empty? ? '' : " for these tables : #{tables.join(" ")}"
         return unless ask_yes_no("This resets the current sync#{msg}.  Are you sure?")
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer # TODO We should rather delete buffer files
         sender.stop
 
-        de = retrieve_sync_data_entry
-        all_tables = de['mysql_data_entry_preference']['tables']
-        verify_input_tables(tables, all_tables)
+        # Set instance variables
+        de = data_entry
+        set_current_tables(tables)
 
         begin
-          wait_for_server_buffer(SERVER_DATA_PROCESSING_TIMEOUT)
+          wait_for_server_buffer(timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: target_tables_for_api)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
@@ -179,20 +155,24 @@ EOS
           ee.set_backtrace e.backtrace
           raise ee
         end
-        cleanup_sync_server(de, tables) unless opts.client?
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+
+        # Cleanup tables on server
+        cleanup_sync_server(de, @input_tables) unless opts.client?
+
+        # Delete local files
+        sync_fm = create_sync_file_manager(de)
         delete_files = [
           sync_fm.dump_file_path,
           sync_fm.dump_pos_path,
           sync_fm.mysql_table_marshal_dump_path,
           sync_fm.sync_info_file,
-          sync_fm.table_position_file_paths(*tables),
-          sync_fm.table_binlog_pos_paths(*tables),
-          sync_fm.table_binlog_pos_init_paths(*tables),
-          sync_fm.table_rev_file_paths(*tables),
-          sync_fm.table_ddl_file_paths(*tables)
+          sync_fm.table_position_file_paths(*@input_tables),
+          sync_fm.table_binlog_pos_paths(*@input_tables),
+          sync_fm.table_binlog_pos_init_paths(*@input_tables),
+          sync_fm.table_rev_file_paths(*@input_tables),
+          sync_fm.table_ddl_file_paths(*@input_tables)
         ]
-        delete_files << sync_fm.binlog_path if tables.empty? or all_tables.empty?
+        delete_files << sync_fm.binlog_path if @input_tables.empty? or @full_tables.empty?
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
         end
@@ -200,75 +180,11 @@ EOS
         log_info_stdout("Reset completed successfully.")
       end
 
-      def wait_for_server_buffer(timeout = 0)
-        start_time = Time.now
-        log_info_stdout("Waiting for the server buffer to get empty.")
-        prev_message =nil
-        while (status = check) && (status['state'] == 'processing')
-          prev_message = status['message']
-          if timeout > 0 && Time.now - start_time > timeout
-            raise ServerDataProcessingTimeout.new
-          end
-          print_progress(status)
-          sleep 10
-        end
-      end
-
-      def wait_for_server_data_processing(timeout = 0)
-        state = :PROCESS
-        start_time = Time.now
-        log_info_stdout("Uploading data to Redshift...")
-        sleep 10
-        status = nil
-        prev_message =nil
-        while (status = check)
-          if state == :PROCESS && status['state'] == 'uploading'
-            log_info_stdout("  -> Done")
-            state = :UPLOAD
-            log_info_stdout("Finishing data upload...")
-          end
-
-          #TODO This is based on a temporary option
-          if state == :UPLOAD && opts.ff?
-            log_info_stdout("Skip checking for pending uploads")
-            break
-          end
-
-          if status['message'] != prev_message
-            # making some progress.  Reset timer
-            start_time = Time.now
-          end
-          prev_message = status['message']
-          if timeout > 0 && Time.now - start_time > timeout
-            raise ServerDataProcessingTimeout.new
-          end
-          print_progress(status)
-          sleep 10
-        end
-        if (state == :PROCESS)
-          # :UPLOAD state was skipped due to no data
-          log_info_stdout("  -> Done")
-          log_info_stdout("Finishing data upload...")
-        end
-        log_info_stdout("  -> Done")
-      end
-
-      def check
-        de = retrieve_sync_data_entry
-        retry_on(RestClient::Exception) do
-          status = do_check(de)
-          if status['complete']
-            nil
-          else
-            status
-          end
-        end
-      end
-
+      # Depricated Command: flydata sync:skip
       # skip initial sync
       def skip
-        de = retrieve_sync_data_entry
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
         binlog_path = sync_fm.binlog_path
         sync_fm.close
         `touch #{binlog_path}`
@@ -277,6 +193,8 @@ EOS
         log_info_stdout("Run 'flydata start' to start continuous sync.")
       end
 
+      # Command: flydata sync:generate_table_ddl
+      #  - Arguments
       def self.slop_generate_table_ddl
         Slop.new do
           on 'c', 'ctl-only', 'Only generate FlyData Control definitions'
@@ -286,166 +204,77 @@ EOS
         end
       end
 
+      # Command: flydata sync:generate_table_ddl
+      #  - Entry method
       def generate_table_ddl(*tables)
-        de = retrieve_sync_data_entry
+        # Compatibility check
+        de = data_entry
         dp = flydata.data_port.get
         Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
-        do_generate_table_ddl(de, tables)
-      end
 
-      private
+        # Set instance variables
+        set_current_tables(tables)
 
-      def verify_input_tables(input_tables, all_tables)
-        return unless input_tables
-        inval_table = []
-        input_tables.each do |tab|
-          inval_table << tab unless all_tables.include?(tab)
-        end
-        raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
+        do_generate_table_ddl(de)
       end
 
-      def set_current_tables(de)
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
-        full_tables = de['mysql_data_entry_preference']['tables']
-
-        @new_tables = sync_fm.get_new_table_list(full_tables, "pos")
-        @ddl_tables = sync_fm.get_new_table_list(full_tables, "generated_ddl")
-
-        @initial_sync = (@new_tables == full_tables)
-        sync_fm.close
-      end
+      private
 
-      def validate_initial_sync_status(de, tables)
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
-        dump_pos_info = sync_fm.load_dump_pos
-        fp = sync_fm.dump_file_path
-        sync_fm.close
+      # Initial sync
 
-        # status is parsing but dumpfile doesn't exist due to streaming -> raise error
-        if dump_pos_info[:status] == STATUS_PARSING && !File.exists?(fp)
-          raise "FlyData Sync was interrupted with invalid state. Run 'flydata sync:reset#{tables.empty? ? '' : ' ' + tables.join(' ')}' first."
-        end
-      end
+      def handle_mysql_sync
+        de = data_entry
 
-      def retrieve_sync_data_entry
-        de = retrieve_data_entries.first unless de
-        raise "There are no data entries." unless de
-        case de['type']
-        when 'RedshiftMysqlDataEntry'
-          mp = de['mysql_data_entry_preference']
-          if mp['tables_append_only']
-            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq
-          else
-            mp['tables'] = mp['tables'].split(",").uniq
+        unless @new_tables.empty?
+          say("We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}")
+          unless @ddl_tables.empty?
+            say("  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.")
+            say("    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'")
+            say("    to generate SQL to run on Redshift to create the correct tables")
+            say("    Without running this sql on your Redshift cluster, there may be issues with your data")
           end
-        else
-          raise SyncDataEntryError, "No supported data entry. Only mysql-redshift sync is supported."
-        end
-        de
-      end
-
-      def cleanup_sync_server(de, tables = [])
-        print "Cleaning the server."
-        log_info("Cleaning the server.")
-        worker = Thread.new do
-          begin
-            flydata.data_entry.cleanup_sync(de['id'], tables)
-          rescue RestClient::RequestTimeout, RestClient::GatewayTimeout
-            # server is taking time to cleanup.  Try again
-            retry
+          if ask_yes_no("Do you want to run initial sync on all of these tables now?")
+            initial_sync(de)
+          else
+            #If generate_table_ddl has not been run for these tables, warn user
+            unless @ddl_tables.empty?
+              say("  You can generate DDL SQL for your new tables by running this command")
+              say("    $> flydata sync:generate_table_ddl > create_table.sql")
+            end
+            puts "Without syncing these tables, we cannot start the flydata process"
+            raise "Please try again"
           end
         end
-        until worker.join(5)
-          print "."
-        end
-        puts
-        log_info_stdout("Done.")
-      end
-
-      def do_check(de)
-        flydata.data_entry.buffer_stat(de['id'], env_mode)
       end
 
-      def print_progress(buffer_stat)
-        message = buffer_stat['message']
-        log_info_stdout(message) unless message.nil? || message.empty?
-      end
-
-      def do_generate_table_ddl(de, tables=[])
-        if `which mysqldump`.empty?
-          raise "mysqldump is not installed.  mysqldump is required to run the command"
-        end
-
-        error_list = []
-
-        schema_name = (de['schema_name'] || nil)
-
-        mp = de['mysql_data_entry_preference']
-
-        set_current_tables(de)
-
-        tables = opts.all_tables? ? mp['tables'] : (tables.empty? ? @new_tables : tables)
-        raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
-
-        %w(host username database).each do |conf_name|
-          raise "MySQL `#{conf_name}` is neither defined in the data entry nor the local config file" if mp[conf_name].to_s.empty?
-        end
-        if tables.empty?
-          raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
-        end
+      def initial_sync(de)
+        # Load sync information from file
+        load_sync_info(de)
+        validate_initial_sync_status
+        begin
+          flush_buffer_and_stop(target_tables_for_api)  unless @full_initial_sync
+          sync_mysql_to_redshift(de)
+        rescue ServerDataProcessingTimeout => e
+          ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
+          ee.description = <<EOS
+  Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
+  Once checked, you can resume sync with the following command.
 
-        command = Util::MysqlUtil.generate_mysql_ddl_dump_cmd(mp.merge(tables: tables))
+    flydata start
 
-        Open3.popen3(command) do |stdin, stdout, stderr|
-          stdin.close
-          stdout.set_encoding("utf-8") # mysqldump output must be in UTF-8
-          create_flydata_ctl_table = @initial_sync
-          while !stdout.eof?
-            begin
-              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, skip_primary_key_check: opts.skip_primary_key_check?)
-              if mysql_tabledef.nil?
-                # stream had no more create table definition
-                break
-              end
-              flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-              puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
-            rescue FlydataCore::TableDefError=> e
-              error_list << e.err_hash
-              next
-            end
-            create_flydata_ctl_table = false
-          end
-          errors = ""
-          while !stderr.eof?
-            line = stderr.gets.gsub('mysqldump: ', '')
-            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
-          end
-          raise errors unless errors.empty?
-        end
-        unless error_list.empty?
-          log_error_stderr("We have noticed the following error(s):")
-          group_error =  error_list.group_by {|d| d[:error]}
-          group_error.each_key do |a|
-            log_error_stderr("The following table(s) have #{a}:")
-            group_error[a].each do |hash|
-              log_error_stderr("  - #{hash[:table]}") if hash[:table]
-            end
-          end
-          log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
+EOS
+          ee.set_backtrace e.backtrace
+          raise ee
         end
-        tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
-
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
-        sync_fm.mark_generated_tables(tables_without_error)
-        sync_fm.close
+        complete(de)
       end
 
       def sync_mysql_to_redshift(de)
         dp = flydata.data_port.get
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        sync_fm = create_sync_file_manager(de)
 
         # Check client condition
-        if File.exists?(sync_fm.binlog_path) and @initial_sync
+        if File.exists?(sync_fm.binlog_path) and @full_initial_sync
           raise "Already synchronized. If you want to do initial sync, run 'flydata sync:reset'"
         end
 
@@ -454,7 +283,7 @@ EOS
           Flydata::Command::Conf.new.copy_templates
         end
         generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, db_bytesize|
-          sync_fm.save_sync_info(@initial_sync, de['mysql_data_entry_preference']['tables'])
+          sync_fm.save_sync_info(@full_initial_sync, de['mysql_data_entry_preference']['tables'])
           parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
         end
         wait_for_mysqldump_processed(dp, de, sync_fm)
@@ -481,7 +310,7 @@ EOS
           return call_block_or_return_io(fp, &block)
         end
 
-        tables = de['mysql_data_entry_preference']['tables']
+        tables = target_tables
         tables ||= '<all tables>'
         data_servers = de['mysql_data_entry_preference']['data_servers'] ? "\n  data servers: #{de['mysql_data_entry_preference']['data_servers']}" : ""
 
@@ -507,7 +336,7 @@ EOM
           Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
           log_info_stdout("Checking database size...")
 
-          db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
+          db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).get_db_bytesize
           log_info_stdout("  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
 
           if file_dump
@@ -523,17 +352,16 @@ EOM
             end
           end
 
-
           log_info_stdout("Exporting data from database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           if file_dump
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference']).dump(fp)
+              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
             log_info_stdout("  -> Done")
             call_block_or_return_io(fp, &block)
           else
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
+              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io| block.call(io, db_bytesize)}
           end
         else
           exit 1
@@ -603,7 +431,7 @@ EOM
           log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
         else
           #If its a new sync, ensure server side resources are clean
-          cleanup_sync_server(de, de['mysql_data_entry_preference']['tables']) unless opts.skip_cleanup?
+          cleanup_sync_server(de, target_tables_for_api) unless opts.skip_cleanup?
         end
         log_info_stdout("Sending data to FlyData Server...")
 
@@ -671,38 +499,85 @@ EOM
         return unless dump_pos_info[:status] == STATUS_WAITING
         binlog_pos = dump_pos_info[:binlog_pos]
 
-        wait_for_server_data_processing(SERVER_DATA_PROCESSING_TIMEOUT)
-        tables = de['mysql_data_entry_preference']['tables']
-        sync_fm.save_table_binlog_pos(tables, binlog_pos)
+        wait_for_server_data_processing(
+          timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: target_tables_for_api)
+        sync_fm.save_table_binlog_pos(target_tables, binlog_pos)
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
       end
 
-      ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
+      # option: timeout, tables
+      def wait_for_server_data_processing(option = {})
+        timeout = option[:timeout] || 0
+        tables = option[:tables] || []
 
-Congratulations! FlyData has started synchronizing your database tables.
+        state = :PROCESS
+        start_time = Time.now
+        log_info_stdout("Uploading data to Redshift...")
+        sleep 10
+        status = nil
+        prev_message =nil
+        while (status = check_server_status(tables))
+          if state == :PROCESS && status['state'] == 'uploading'
+            log_info_stdout("  -> Done")
+            state = :UPLOAD
+            log_info_stdout("Finishing data upload...")
+          end
 
-What's next?
+          #TODO This is based on a temporary option
+          if state == :UPLOAD && opts.ff?
+            log_info_stdout("Skip checking for pending uploads")
+            break
+          end
 
-    - Check data on Redshift (%s)
-    - Check your FlyData usage on the FlyData Dashboard (%s)
-    - To manage the FlyData Agent, use the 'flydata' command (type 'flydata' for help)
-    - If you encounter an issue,
-        please check our documentation (https://www.flydata.com/docs/) or
-        contact our customer support team (support@flydata.com)
+          if status['message'] != prev_message
+            # making some progress.  Reset timer
+            start_time = Time.now
+          end
+          prev_message = status['message']
+          if timeout > 0 && Time.now - start_time > timeout
+            raise ServerDataProcessingTimeout.new
+          end
+          print_progress(status)
+          sleep 10
+        end
+        if (state == :PROCESS)
+          # :UPLOAD state was skipped due to no data
+          log_info_stdout("  -> Done")
+          log_info_stdout("Finishing data upload...")
+        end
+        log_info_stdout("  -> Done")
+      end
 
-Thank you for using FlyData!
-      EOM
+      def check_server_status(tables = [])
+        de = data_entry
+        retry_on(RestClient::Exception) do
+          status = do_check_server_status(de, tables)
+          if status['complete']
+            nil
+          else
+            status
+          end
+        end
+      end
 
-      def complete
-        de = load_sync_info(retrieve_sync_data_entry)
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+      def do_check_server_status(de, tables = [])
+        flydata.data_entry.buffer_stat(de['id'], mode: env_mode, tables: tables)
+      end
+
+      def print_progress(buffer_stat)
+        message = buffer_stat['message']
+        log_info_stdout(message) unless message.nil? || message.empty?
+      end
+
+      def complete(de)
+        sync_fm = create_sync_file_manager(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE
-          if @initial_sync
+          if @full_initial_sync
             sync_fm.save_binlog(info[:binlog_pos])
           end
-          sync_fm.install_table_binlog_files(de['mysql_data_entry_preference']['tables'])
-          sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'])
+          sync_fm.install_table_binlog_files(target_tables)
+          sync_fm.reset_table_position_files(target_tables)
           sync_fm.delete_dump_file
           sync_fm.backup_dump_dir
         else
@@ -719,29 +594,238 @@ Thank you for using FlyData!
         h.to_json
       end
 
-      def load_sync_info(de ,additional_tables=[])
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+      # Sync reset
+
+      def wait_for_server_buffer(option = {})
+        timeout = option[:timeout] || 0
+        tables = option[:tables] || []
+
+        start_time = Time.now
+        log_info_stdout("Waiting for the server buffer to get empty.")
+        prev_message =nil
+        while (status = check_server_status(tables)) && (status['state'] == 'processing')
+          prev_message = status['message']
+          if timeout > 0 && Time.now - start_time > timeout
+            raise ServerDataProcessingTimeout.new
+          end
+          print_progress(status)
+          sleep 10
+        end
+      end
+
+      def cleanup_sync_server(de, tables = [])
+        print "Cleaning the server."
+        log_info("Cleaning the server.")
+        worker = Thread.new do
+          begin
+            flydata.data_entry.cleanup_sync(de['id'], tables)
+          rescue RestClient::RequestTimeout, RestClient::GatewayTimeout
+            # server is taking time to cleanup.  Try again
+            retry
+          end
+        end
+        until worker.join(5)
+          print "."
+        end
+        puts
+        log_info_stdout("Done.")
+      end
+
+      # Generate table ddl
+
+      def do_generate_table_ddl(de)
+        if `which mysqldump`.empty?
+          raise "mysqldump is not installed.  mysqldump is required to run the command"
+        end
+
+        error_list = []
+        schema_name = (de['schema_name'] || nil)
         mp = de['mysql_data_entry_preference']
-        unless (rs = sync_fm.load_sync_info).nil?
-          @initial_sync = rs[:initial_sync]
-          mp['tables'] = rs[:tables]
-        else
-          mp['tables'] = additional_tables unless @initial_sync
+
+        tables = opts.all_tables? ? @full_tables : (@input_tables.empty? ? @new_tables : @input_tables)
+        raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
+
+        %w(host username database).each do |conf_name|
+          raise "MySQL `#{conf_name}` is neither defined in the data entry nor the local config file" if mp[conf_name].to_s.empty?
         end
+        if tables.empty?
+          raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
+        end
+
+        command = Util::MysqlUtil.generate_mysql_ddl_dump_cmd(mp.merge(tables: tables))
+
+        Open3.popen3(command) do |stdin, stdout, stderr|
+          stdin.close
+          stdout.set_encoding("utf-8") # mysqldump output must be in UTF-8
+          create_flydata_ctl_table = @full_initial_sync
+          while !stdout.eof?
+            begin
+              mysql_tabledef = FlydataCore::TableDef::MysqlTableDef.create(stdout, skip_primary_key_check: opts.skip_primary_key_check?)
+              if mysql_tabledef.nil?
+                # stream had no more create table definition
+                break
+              end
+              flydata_tabledef = mysql_tabledef.to_flydata_tabledef
+              puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
+            rescue FlydataCore::TableDefError=> e
+              error_list << e.err_hash
+              next
+            end
+            create_flydata_ctl_table = false
+          end
+          errors = ""
+          while !stderr.eof?
+            line = stderr.gets.gsub('mysqldump: ', '')
+            errors << line unless /Warning: Using a password on the command line interface can be insecure./ === line
+          end
+          raise errors unless errors.empty?
+        end
+        unless error_list.empty?
+          log_error_stderr("We have noticed the following error(s):")
+          group_error =  error_list.group_by {|d| d[:error]}
+          group_error.each_key do |a|
+            log_error_stderr("The following table(s) have #{a}:")
+            group_error[a].each do |hash|
+              log_error_stderr("  - #{hash[:table]}") if hash[:table]
+            end
+          end
+          log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
+        end
+        tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
+
+        sync_fm = create_sync_file_manager(de)
+        sync_fm.mark_generated_tables(tables_without_error)
         sync_fm.close
-        de
       end
 
-      def flush_buffer_and_stop
+
+      ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
+
+Congratulations! FlyData has started synchronizing your database tables.
+
+What's next?
+
+    - Check data on Redshift (%s)
+    - Check your FlyData usage on the FlyData Dashboard (%s)
+    - To manage the FlyData Agent, use the 'flydata' command (type 'flydata' for help)
+    - If you encounter an issue,
+        please check our documentation (https://www.flydata.com/docs/) or
+        contact our customer support team (support@flydata.com)
+
+Thank you for using FlyData!
+      EOM
+
+      # Sync flush
+
+      def flush_buffer_and_stop(tables = [])
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer
         if opts.skip_flush?
           log_info_stdout("Skip waiting for server data processing.")
         else
-          wait_for_server_data_processing(SERVER_DATA_PROCESSING_TIMEOUT)
+          wait_for_server_data_processing(
+            timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: tables)
         end
         sender.stop(quiet: true)
       end
+
+      # Utility methods
+
+      def set_current_tables(input_tables = nil)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        @input_tables = input_tables || []
+        @full_tables = de['mysql_data_entry_preference']['tables']
+
+        @new_tables = sync_fm.get_new_table_list(@full_tables, "pos")
+        @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
+
+        @full_initial_sync = (@new_tables == @full_tables)
+
+        sync_fm.close
+
+        verify_input_tables(@input_tables, @full_tables)
+      end
+
+      def validate_initial_sync_status
+        sync_fm = create_sync_file_manager
+        dump_pos_info = sync_fm.load_dump_pos
+        fp = sync_fm.dump_file_path
+        sync_fm.close
+
+        # status is parsing but dumpfile doesn't exist due to streaming -> raise error
+        if dump_pos_info[:status] == STATUS_PARSING && !File.exists?(fp)
+          raise "FlyData Sync was interrupted with invalid state. Run 'flydata sync:reset#{@input_tables.join(',')}' first."
+        end
+      end
+
+      def load_sync_info(de)
+        # for debug
+        raise "!AssertionError. set_current_tables needs to be called in advance" if @full_tables.nil?
+
+        sync_fm = create_sync_file_manager(de)
+        if (rs = sync_fm.load_sync_info)
+          @full_initial_sync = rs[:initial_sync]
+          @input_tables = rs[:tables]
+        end
+        sync_fm.close
+        de
+      end
+
+      def target_tables
+        if @full_initial_sync
+          @full_tables
+        elsif !@input_tables.empty?
+          @input_tables
+        else
+          @new_tables
+        end
+      end
+
+      # return empty array if full sync
+      def target_tables_for_api
+        if @full_initial_sync
+          []
+        elsif !@input_tables.empty?
+          @input_tables
+        else
+          @new_tables
+        end
+      end
+
+      def data_entry
+        @de ||= retrieve_sync_data_entry
+      end
+
+      def retrieve_sync_data_entry
+        de = retrieve_data_entries.first unless de
+        raise "There are no data entries." unless de
+        case de['type']
+        when 'RedshiftMysqlDataEntry'
+          mp = de['mysql_data_entry_preference']
+          if mp['tables_append_only']
+            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq
+          else
+            mp['tables'] = mp['tables'].split(",").uniq
+          end
+        else
+          raise SyncDataEntryError, "No supported data entry. Only mysql-redshift sync is supported."
+        end
+        de
+      end
+
+      def create_sync_file_manager(de = data_entry)
+        Flydata::FileUtil::SyncFileManager.new(de)
+      end
+
+      def verify_input_tables(input_tables, all_tables)
+        return unless input_tables
+        inval_table = []
+        input_tables.each do |tab|
+          inval_table << tab unless all_tables.include?(tab)
+        end
+        raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
+      end
     end
   end
 end

data/spec/flydata/command/sender_spec.rb

@@ -18,7 +18,7 @@ module Flydata
           allow(Kernel).to receive(:sleep)
           allow_any_instance_of(Flydata::Api::DataPort).to receive(:get).and_return("Wibble")
           allow_any_instance_of(Flydata::AgentCompatibilityCheck).to receive(:check).and_return(true)
-          Flydata::Command::Sync.any_instance.should_receive(:handle_mysql_sync).and_return("Wobble")
+          Flydata::Command::Sync.any_instance.should_receive(:try_mysql_sync).and_return("Wobble")
         end
 
         context "as daemon" do

data/spec/flydata/command/sync_spec.rb

@@ -65,6 +65,10 @@ module Flydata
         end
       end
       describe '#do_generate_table_ddl' do
+        before do
+          allow(subject).to receive(:data_entry).and_return(default_data_entry)
+          subject.send(:set_current_tables)
+        end
         shared_examples 'throws an error' do
           it "throws an error" do
             expect {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -592,7 +592,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FlyData Agent