flydata 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 325ec216ca30d8548887146aa97bde6daff26476
-  data.tar.gz: 59a60554b1eea3751b0fc904d9df55fbbc1fab4d
+  metadata.gz: 0bbdc5f5871df8b3fec4c8c92abd1fdb6b6b24f6
+  data.tar.gz: 0be09c4771db44cac9c3229251598f6f5c3c23fd
 SHA512:
-  metadata.gz: 34cf256c13392454624c461b5920bbf8f77540fbe99a470b4acc1249a31e49d3ab627934c1f6b54b32083edf520c8e2836622bb71228febd5236b3b02a36abba
-  data.tar.gz: b2004689294e57c0dee428ea852fe421fbd945b1bf47ec130757c097212d4c0e0eb469543df32994365b5b21cd90b4721cc843f195b80c0f15437119c205fa06
+  metadata.gz: 3bdacc04e29100c441e7e1eb1660eb9763e2566f0ac05846a7cc85ac2067e4f0171c41d95e4a12ae5fa7b9c3720b5583040191a68b669b392e360fa58be79abb
+  data.tar.gz: 932a0de391228f576f389c731f312108931be9473875fe8265cc982d58771a6ea20671a762eeaf7b30a5aefbdfa0d60bb4572f409635f757e2bd900fad65e966

data/VERSION

@@ -1 +1 @@
-0.4.3
+0.5.0

data/flydata-core/lib/flydata-core/errors.rb

@@ -289,6 +289,10 @@ class RedshiftAccessError < UserResourceError
   def err_code; ErrorCode::REDSHIFT_ACCESS_ERROR; end
 end
 
+# error_content[:errors] has a list of errors
+class MultipleErrors < DataDeliveryError
+  def err_code; -1; end
+end
 
 ## Depricated errors
 

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -156,6 +156,8 @@ module FlydataCore
         "FROM information_schema.tables " +
         "WHERE table_schema = '%s' and table_name in (%s)"
 
+      UNSUPPORTED_ENGINES = %w(MEMORY BLACKHOLE)
+
       # option[:client] : Mysql2::Client object
       # option[:tables] : target table list
       def create_query(option = @option)
@@ -168,12 +170,12 @@ module FlydataCore
       def check_result(result, option = @option)
         invalid_tables = []
         result.each do |r|
-          invalid_tables.push(r['table_name']) if r['table_type'] == 'VIEW' || r['engine'] == 'MEMORY'
+          invalid_tables.push(r['table_name']) if r['table_type'] == 'VIEW' || UNSUPPORTED_ENGINES.include?(r['engine'])
         end
 
         unless invalid_tables.empty?
           raise FlydataCore::MysqlCompatibilityError,
-            "FlyData does not support VIEW and MEMORY ENGINE table. " +
+            "FlyData does not support VIEW and #{UNSUPPORTED_ENGINES.join(',')} STORAGE ENGINE table. " +
             "Remove following tables from data entry: #{invalid_tables.join(", ")}"
         end
       end

data/flydata-core/lib/flydata-core/option_validator.rb

@@ -0,0 +1,38 @@
+module FlydataCore
+
+module OptionValidator
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  def validate_options(o, options = nil)
+    self.class.validate_options(o, options)
+  end
+
+  module ClassMethods
+    def validate_options(o, options = nil)
+      if (options.nil?)
+        unless defined?(self::OPTIONS)
+          raise "validate_options needs OPTIONS hash defined"
+        end
+        options = self::OPTIONS
+      end
+
+      result = o.clone
+
+      options.keys.each do |k|
+        unless (result.has_key?(k))
+          if (options[k] != :mandatory)
+            result[k] = options[k]
+          else
+            raise "Mandatory option #{k} is missing"
+          end
+        end
+      end
+
+      result
+    end
+  end
+end
+
+end

data/flydata-core/lib/flydata-core/query_job.rb

@@ -2,6 +2,13 @@
 # known as Copy Handler)
 # SimpleDB related classes and methods defined in flydata and flydata-web will
 # be eventually consolidated into this library.
+module FlydataCore
+module QueryJob
+  # A special "sequence number" which kicks off sync handoff between init and cont syncs.
+  SYNC_FIRST_SEQ = '1.sync'
+end
+end
+
 Dir[File.join(File.dirname(__FILE__), "query_job/**/*.rb")].each do |f|
   require f
 end

data/flydata-core/lib/flydata-core/query_job/table_status.rb

@@ -0,0 +1,44 @@
+require 'flydata-core/query_job/redshift'
+require 'flydata-core/table_def/redshift_table_def'
+
+module FlydataCore
+module QueryJob
+
+class TableStatus
+  @@domain = 'flydata_redshift_table_status'
+
+  def self.domain=(domain)
+    @@domain = domain
+  end
+
+  def self.sdb=(sdb)
+    @@sdb = sdb
+  end
+
+  def self.where(*args)
+    table_status_items = {}
+    @@sdb.domains[@@domain].items.where(*args).each(select: :all) do |item|
+      class << item
+        # returns the corect (Redshift) table name
+        def table_name
+          FlydataCore::TableDef::RedshiftTableDef.convert_to_valid_table_name(attributes['table_name'].first)
+        end
+
+        def target_table_names
+          FlydataCore::QueryJob::Redshift.target_table_names(attributes['table_name'].first)
+        end
+      end
+      tn = item.attributes['table_name'].first
+      redshift_table_name = FlydataCore::TableDef::RedshiftTableDef.convert_to_valid_table_name(tn)
+      if redshift_table_name == tn
+        table_status_items[redshift_table_name] ||= item
+      else
+        table_status_items[redshift_table_name] = item
+      end
+    end
+    table_status_items.values
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -285,7 +285,6 @@ EOS
   end
 
   MAX_COLUMNNAME_LENGTH = 127
-
   VALID_IDENTIFIER_CHARACTERS = [" "] + %w"a-z 0-9 ! # $ % & ' ( ) * + , . \\- \\/ : ; < = > ? @ \\[ \\\\ \\] ^ { \\| } ~ `"
 
   def self.convert_to_valid_name(key)
@@ -335,7 +334,7 @@ EOS
     elsif time_match = TIME_REGEXP.match(value_str)
       t = convert_time_into_timestamp(time_match)
     else
-      t = DateTime.parse(value_str)
+      t = DateTime.parse(convert_date(value_str))
     end
     t = t.new_offset(0) # Redshift Plug-in uses UTC
     t.strftime('%Y-%m-%d %H:%M:%S.%6N')

data/flydata-core/spec/option_validator_spec.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+require 'flydata-core/option_validator'
+
+module FlydataCore
+
+describe OptionValidator do
+  describe '#validate_options' do
+    context 'with OPTIONS' do
+      class GoodBoy
+        include OptionValidator
+        OPTIONS = {
+          :a => :mandatory,
+          :b => 'some default value'
+        }
+
+        def initialize(o)
+          @o = validate_options(o)
+        end
+        attr_reader :o
+      end
+
+      context 'valid options are passed through' do
+        subject{ GoodBoy.new(a: 'me', b: 'you') }
+
+        describe '#o' do
+          subject { super().o }
+          it { is_expected.to eq({a: 'me', b: 'you'}) }
+        end
+      end
+      context 'optional option may be omitted' do
+        subject{ GoodBoy.new(a: 'me') }
+
+        describe '#o' do
+          subject { super().o }
+          it { is_expected.to eq({a: 'me', b: 'some default value' }) }
+        end
+      end
+      context 'mandatory option must be there' do
+        it { expect{ GoodBoy.new(b: 'you') }.to raise_error }
+      end
+      context 'unknown options are passed through' do
+        subject{ GoodBoy.new(a: 'm', c: 'who') }
+
+        describe '#o' do
+          subject { super().o }
+          it { is_expected.to eq({a: 'm', b: 'some default value', c: 'who'}) }
+        end
+      end
+    end
+    context 'without OPTIONS' do
+      class BadBoy
+        include OptionValidator
+
+        def initialize(o)
+          @o = validate_options(o)
+        end
+        attr_reader :o
+      end
+
+      context 'OPTIONS must exist' do
+        it { expect{ BadBoy.new(a: 'me', b: 'you') }.to raise_error }
+      end
+    end
+    context 'nested classes' do
+      class Parent
+        include OptionValidator
+        OPTIONS = { :a => :mandatory }
+      end
+      context "Child's OPTIONS overrides parent's" do
+        class Child < Parent
+          OPTIONS = {:b => :mandatory}
+
+          def initialize(o)
+            @o = validate_options(o)
+          end
+          attr_reader :o
+        end
+        subject { Child.new(b: 'test') }
+
+        describe '#o' do
+          subject { super().o }
+          it { is_expected.to eq({b: 'test'}) }
+        end
+      end
+      context "Parent's OPTIONS carries over if child doesn't have its own" do
+        class Child2 < Parent
+          def initialize(o)
+            @o = validate_options(o)
+          end
+          attr_reader :o
+        end
+        it { expect { Child2.new(b: 'test') }.to raise_error }
+      end
+    end
+  end
+  describe "::validate_options" do
+    context 'with OPTIONS' do
+      class GoodBoy2
+        include OptionValidator
+        OPTIONS = {
+          :a => :mandatory,
+          :b => 'default value'
+        }
+
+        def self.test(o)
+          validate_options(o)
+        end
+      end
+
+      context 'valid options are passed through' do
+        it { expect(GoodBoy2.test(a: 'me')).to eq(a: 'me', b: 'default value') }
+      end
+    end
+  end
+end
+
+end

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -662,6 +662,21 @@ EOS
       end
     end
 
+    context 'with zero date' do
+      let(:value) { '0000-00-00 10:24:28' }
+      it { is_expected.to eq('0001-01-01 10:24:28.000000') }
+    end
+
+    context 'with zero month' do
+      let(:value) { '2005-00-19 10:19:25' }
+      it { is_expected.to eq('2005-01-19 10:19:25.000000') }
+    end
+
+    context 'with zero day and month' do
+      let(:value) { '2010-00-00 00:00:00' }
+      it { is_expected.to eq('2010-01-01 00:00:00.000000') }
+    end
+
     context 'with time value' do
       context 'when value is positive' do
         context 'with sec' do

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.4.3 ruby lib
+# stub: flydata 0.5.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.4.3"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-07-08"
+  s.date = "2015-07-24"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -49,8 +49,10 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/logger.rb",
     "flydata-core/lib/flydata-core/mysql/command_generator.rb",
     "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
+    "flydata-core/lib/flydata-core/option_validator.rb",
     "flydata-core/lib/flydata-core/query_job.rb",
     "flydata-core/lib/flydata-core/query_job/redshift.rb",
+    "flydata-core/lib/flydata-core/query_job/table_status.rb",
     "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/redshift/string.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
@@ -64,6 +66,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/mysql/command_generator_spec.rb",
     "flydata-core/spec/mysql/compatibility_checker.rb",
+    "flydata-core/spec/option_validator_spec.rb",
     "flydata-core/spec/query_job/redshift_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",

data/lib/flydata/api/data_entry.rb

@@ -20,12 +20,19 @@ module Flydata
       end
 
       # Update validity of tables
-      #  table_validity_hash: { "bad_table": "error reason", "good_table": nil }
+      #  table_validity_hash: {"updated_tables":{ "bad_table": "error reason", "good_table": nil }}
       #    table "bad_table" will be marked invalid with reason "error reason"
       #    table "good table" will be marked valid, that is, clear its error reason if it's set.
       def update_table_validity(data_entry_id, table_validity_hash)
         @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_validity_hash.to_json)
       end
+
+      # Tells the server that an initial sync has completed
+      #  stats_hash: {"init_sync_stats":{"Table1": 100, "Table2": 12345}}
+      #    Sent 100 records for Table1, Sent 12345 records for Table2
+      def complete_init_sync(data_entry_id, stats_hash)
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
+      end
     end
   end
 end

data/lib/flydata/command/helper.rb

@@ -48,7 +48,6 @@ module Flydata
 
       def stop
         if running?
-          log_info_stdout("Stopping Helper.")
           run_command(stop_cmd)
         else
           log_info_stdout("Helper is not running.")
@@ -136,10 +135,8 @@ EOS
       end
 
       def raw_start
-        log_info_stdout("Starting Helper.")
         Flydata::Helper::Server.run(
             DEFAULT_OPTIONS.merge(create_config_with_args))
-        log_info_stdout("Done")
       end
 
       def create_helper_dirs

data/lib/flydata/command/sender.rb

@@ -32,14 +32,24 @@ module Flydata
         dp = flydata.data_port.get
         AgentCompatibilityCheck.new(dp).check
 
-        Flydata::Command::Sync.new.try_mysql_sync
-
-        # Start sender(fluentd) process
-        log_info_stdout("Starting FlyData Agent sender process.") unless options[:quiet]
-        raw_start(options)
+        fluentd_started = false
+        start_fluentd = Proc.new do
+          # Start sender(fluentd) process
+          log_info_stdout("Starting FlyData Agent sender process.") unless options[:quiet]
+          raw_start(options)
+  
+          wait_until_client_ready(options)
+          #wait_until_logs_uploaded
+          fluentd_started = true
+        end
 
-        wait_until_client_ready(options)
-        #wait_until_logs_uploaded
+        quiet_option = options[:quiet]
+        # surpress messages if fluentd is started in #try_mysql_sync
+        options[:quiet] = true
+        Flydata::Command::Sync.new.try_mysql_sync(
+                                        binlog_ready_callback: start_fluentd)
+        options[:quiet] = quiet_option
+        start_fluentd.call unless fluentd_started
         if options[:show_final_message] && !options[:quiet]
           data_port = flydata.data_port.get
           log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")

data/lib/flydata/command/sync.rb

@@ -28,7 +28,7 @@ module Flydata
 
       # for dump.pos file
       STATUS_PARSING = 'PARSING'
-      STATUS_WAITING = 'WAITING'
+      STATUS_PARSED = 'WAITING'    # the value is different from the constant name on purpose for backward compatibility.
       STATUS_COMPLETE = 'COMPLETE'
 
       attr_reader :full_initial_sync,     # true if full initial sync
@@ -74,16 +74,27 @@ module Flydata
           exit 1
         end
 
-        # Start initial sync with check
-        handle_mysql_sync(tables)
+        fluentd_started = false
 
-        # Start continuous sync by starting fluentd process
-        unless opts.no_flydata_start?
-          log_info_stdout("Starting FlyData Agent...")
-          Flydata::Command::Sender.new.start(quiet: true)
-          log_info_stdout("  -> Done")
+        quiet_option = false
+
+        start_fluentd = Proc.new do |binlog_pos|
+          # Start continuous sync by starting fluentd process
+          unless opts.no_flydata_start?
+            log_info_stdout("Starting FlyData Agent...") unless quiet_option
+            Flydata::Command::Sender.new.start(quiet: true)
+            log_info_stdout("  -> Done") unless quiet_option
+          end
+          fluentd_started = true
         end
 
+        quiet_option = true
+        # Start initial sync with check
+        handle_mysql_sync(tables, binlog_ready_callback: start_fluentd)
+        quiet_option = false
+
+        start_fluentd.call unless fluentd_started
+
         # Show message
         dashboard_url = "#{flydata.flydata_api_host}/dashboard"
         redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
@@ -93,9 +104,9 @@ module Flydata
 
       # Public method
       #  - Called from Sender#start/restart
-      def try_mysql_sync
+      def try_mysql_sync(options)
         # Start initial sync
-        handle_mysql_sync
+        handle_mysql_sync(nil, options)
       rescue SyncDataEntryError
         return
       end
@@ -113,7 +124,7 @@ module Flydata
       #  - Entry method
       def flush(*tables)
         begin
-          flush_buffer_and_stop(tables)
+          flush_buffer_and_stop(tables, skip_flush: opts.skip_flush?)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
@@ -139,17 +150,27 @@ EOS
       #  - Entry method
       def reset(*tables)
         show_purpose_name
+
+        # Set instance variables
+        sync_resumed = set_current_tables(tables, resume: true)
+
+        message = ''
+        if sync_resumed && !tables.empty?
+          log_info_stdout <<EOS
+Initial sync is in progress.  In this case, you can only reset the initial sync.To reset specific table(s), please resume and complete the initial sync by running the 'flydata start' command first.
+If you'd like to reset the initial sync in progress, run the 'flydata reset' command with no arguments.
+EOS
+          return
+        end
+
         # Flush client buffer
-        msg = tables.empty? ? '' : " for these tables : #{tables.join(" ")}"
-        return unless ask_yes_no("This resets the current sync#{msg}.  Are you sure?")
+        msg_tables = @input_tables.empty? ? '' : " for these tables : #{@input_tables.join(" ")}"
+        msg_sync_type = sync_resumed ? "the current initial sync" : "the current sync"
+        return unless ask_yes_no("This resets #{msg_sync_type}#{msg_tables}.  Are you sure?")
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer # TODO We should rather delete buffer files
         sender.stop
 
-        # Set instance variables
-        de = data_entry
-        set_current_tables(tables)
-
         begin
           wait_for_server_buffer(timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: target_tables_for_api)
         rescue ServerDataProcessingTimeout => e
@@ -166,6 +187,7 @@ EOS
         end
 
         # Cleanup tables on server
+        de = data_entry
         cleanup_sync_server(de, @input_tables) unless opts.client?
 
         # Delete local files
@@ -284,17 +306,17 @@ EOS
 
       # Initial sync
 
-      def handle_mysql_sync(tables = nil)
+      def handle_mysql_sync(tables = nil, options = {})
         de = data_entry
 
         # Setup instance variables
-        sync_resumed = set_current_tables(tables)
+        sync_resumed = set_current_tables(tables, resume: true)
 
         if sync_resumed
-          # skip confirmation prompts and resume sync right away.  #initial_sync knows
-          # where to resume from.
+          # skip confirmation prompts and resume sync right away.
+          # #initial_sync knows where to resume from.
           log_info_stdout("Resuming the initial sync...")
-          initial_sync(de, sync_resumed: true)
+          initial_sync(de, options.merge(sync_resumed: true))
         elsif !@new_tables.empty?
           show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@new_tables.join(", ")}\n"
@@ -307,7 +329,7 @@ EOS
           end
           log_info_stdout(unsynced_table_message)
           if ask_yes_no("Do you want to run initial sync on all of these tables now?")
-            initial_sync(de, sync_resumed: false)
+            initial_sync(de, options.merge(sync_resumed: false))
           else
             #If generate_table_ddl has not been run for these tables, warn user
             unless @ddl_tables.empty?
@@ -324,13 +346,20 @@ EOS
         # Load sync information from file
         validate_initial_sync_status
         begin
-          unless @full_initial_sync || opt[:sync_resumed]
-            # flush is unnecessary for full initial sync or sync resume because in either
-            # case it's guaranteed that agent is stopped with no leftover buffer.
+          if opt[:sync_resumed]
+            # parallel cont sync has sent buffer data by now so server buffer
+            # data will exist.  Skip server data flush.
+            flush_buffer_and_stop(target_tables_for_api, skip_flush: true)
+          elsif !@full_initial_sync
+            # flush leftover data for tables being added.
             log_info_stdout("Sending the existing buffer data...")
-            flush_buffer_and_stop(target_tables_for_api)
+            flush_buffer_and_stop(target_tables_for_api,
+                                  skip_flush: opts.skip_flush?)
+          else
+            # flush is unnecessary for full initial sync because it's guaranteed
+            # that agent is stopped with no leftover buffer.
           end
-          sync_mysql_to_redshift(de)
+          sync_mysql_to_redshift(de, opt)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
@@ -346,12 +375,12 @@ EOS
         complete(de)
       end
 
-      def sync_mysql_to_redshift(de)
+      def sync_mysql_to_redshift(de, options = {})
         dp = flydata.data_port.get
         sync_fm = create_sync_file_manager(de)
 
         # Check client condition
-        if File.exists?(sync_fm.binlog_path) and @full_initial_sync
+        if @full_initial_sync && !options[:sync_resumed] && File.exists?(sync_fm.binlog_path)
           raise "Already synchronized. If you want to do initial sync, run 'flydata sync:reset'"
         end
 
@@ -359,14 +388,20 @@ EOS
         unless Flydata::Preference::DataEntryPreference.conf_exists?(de)
           Flydata::Command::Conf.new.copy_templates
         end
-        generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, binlog_pos, db_bytesize|
-          sync_fm.save_sync_info(@full_initial_sync, target_tables)
+
+        # initialize stats
+        target_tables.each do |table_name|
+          sync_fm.save_record_count_stat(table_name, 0)
+        end
+
+        generate_mysqldump(de, sync_fm, !opts.dump_stream?, options) do |mysqldump_io, binlog_pos, db_bytesize|
           parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize)
         end
-        wait_for_mysqldump_processed(dp, de, sync_fm)
+        complete_mysqldump_processing(dp, de, sync_fm)
       end
 
-      def generate_mysqldump(de, sync_fm, file_dump = true, &block)
+      def generate_mysqldump(de, sync_fm, file_dump = true, options = {},
+                             &block)
         # validate parameter
         %w(host username database).each do |k|
           if de['mysql_data_entry_preference'][k].to_s.empty?
@@ -375,15 +410,19 @@ EOS
           end
         end
 
-        # Status is waiting or complete -> skip dump and parse
+        # Status is parsed or complete -> skip dump and parse
         dump_pos_info = sync_fm.load_dump_pos
-        return if dump_pos_info[:status] == STATUS_WAITING || dump_pos_info[:status] == STATUS_COMPLETE
+        if dump_pos_info[:status] == STATUS_PARSED || dump_pos_info[:status] == STATUS_COMPLETE
+          initialize_binlog_positions_and_call_callback(
+                  nil, options[:binlog_ready_callback], sync_fm)
+        end
 
         # mysqldump file exists -> skip dump
         dp = flydata.data_port.get
         fp = sync_fm.dump_file_path
         if file_dump && File.exists?(fp) && File.size(fp) > 0
-          log_info_stdout("  -> Skip")
+          initialize_binlog_positions_and_call_callback(
+                  nil, options[:binlog_ready_callback], sync_fm)
           return call_block_or_return_io(fp, &block)
         end
 
@@ -435,10 +474,16 @@ EOM
 
           log_info_stdout("Setting binary log position and exporting data from the database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
+          sync_fm.save_sync_info(@full_initial_sync, target_tables)
           if file_dump
+            binlog_pos = nil
             begin
-              binlog_pos = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-                new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
+              Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
+                new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp) do |_io, _binlog_pos|
+                binlog_pos = _binlog_pos
+                initialize_binlog_positions_and_call_callback(
+                  binlog_pos, options[:binlog_ready_callback], sync_fm)
+              end
               log_info_stdout("  -> Database dump done")
             rescue Exception => e
               #Catch all exceptions including SystemExit and Interrupt.
@@ -450,13 +495,31 @@ EOM
             call_block_or_return_io(fp, binlog_pos, &block)
           else
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io, binlog_pos| block.call(io, binlog_pos, db_bytesize)}
+              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump do |io, binlog_pos|
+              initialize_binlog_positions_and_call_callback(
+                binlog_pos, options[:binlog_ready_callback], sync_fm)
+
+              block.call(io, binlog_pos, db_bytesize)
+            end
           end
         else
           exit 1
         end
       end
 
+      def initialize_binlog_positions_and_call_callback(binlog_pos, callback, sync_fm)
+        if binlog_pos
+          initialize_positions(sync_fm, binlog_pos)
+        else
+          # no binlog_pos was given because dump was completed in the
+          # previous init sync attempt.  Position files must be there already
+          # so no initialization is necessary.
+        end
+        if callback
+          callback.call(binlog_pos)
+        end
+      end
+
       # return available disk size(byte)
       def free_disk_space(dump_path)
         stat = Sys::Filesystem.stat(dump_path)
@@ -584,7 +647,7 @@ EOM
         log_info_stdout("  -> Done")
         #log_info_stdout("  -> Records sent to the server")
         #log_info_stdout("  -> #{sync_fm.load_stats}")
-        sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
+        sync_fm.save_dump_pos(STATUS_PARSED, '', dump_file_size, binlog_pos)
 
         if ENV['FLYDATA_BENCHMARK']
           bench_end_time = Time.now
@@ -593,17 +656,14 @@ EOM
         end
       end
 
-      def wait_for_mysqldump_processed(dp, de, sync_fm)
+      def complete_mysqldump_processing(dp, de, sync_fm)
         return if ENV['FLYDATA_BENCHMARK']
 
-        # Status is not waiting -> skip waiting
+        # Status is not parsed -> don't complete
         dump_pos_info = sync_fm.load_dump_pos
-        return unless dump_pos_info[:status] == STATUS_WAITING
+        return unless dump_pos_info[:status] == STATUS_PARSED
         binlog_pos = dump_pos_info[:binlog_pos]
 
-        wait_for_server_data_processing(
-          timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: target_tables_for_api)
-        sync_fm.save_table_binlog_pos(target_tables, binlog_pos)
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
       end
 
@@ -676,19 +736,48 @@ EOM
         sync_fm = create_sync_file_manager(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE
-          if @full_initial_sync
-            sync_fm.save_binlog(info[:binlog_pos])
-          end
-          sync_fm.install_table_binlog_files(target_tables)
-          sync_fm.reset_table_position_files(target_tables)
+          send_record_counts(de, sync_fm)
           sync_fm.delete_dump_file
           sync_fm.backup_dump_dir
         else
-          raise "Initial sync status is not complete. Try running 'flydata sync'."
+          raise "Initial sync status is not complete. Try running 'flydata start' again."
         end
         sync_fm.close
       end
 
+      NUM_TABLES_IN_CHUNK = 30
+      def send_record_counts(de, sync_fm)
+        stats = sync_fm.load_stats
+        stats.each_slice(NUM_TABLES_IN_CHUNK) do |slice|
+          h = Hash[slice]
+          send_record_counts_chunk(de, h)
+        end
+      end
+
+      def send_record_counts_chunk(de, stats)
+        retry_count = 0
+        retry_interval = 3
+        begin
+          flydata.data_entry.complete_init_sync(de['id'],
+                                      {init_sync_stats: {record_counts:stats}})
+        rescue
+          retry_count += 1
+          raise if retry_count > 3
+          sleep retry_interval
+          retry_interval *= 2
+        end
+      end
+
+      def initialize_positions(sync_fm, binlog_pos)
+        sync_fm.save_table_binlog_pos(target_tables, binlog_pos)
+
+        if @full_initial_sync
+          sync_fm.save_binlog(binlog_pos)
+        end
+        sync_fm.install_table_binlog_files(target_tables)
+        sync_fm.reset_table_position_files(target_tables)
+      end
+
       def convert_to_flydata_values(mysql_table, values)
         types = mysql_table.columns.each_value.collect{|col_attrs| col_attrs[:format_type]}
         types.size.times.collect{|i| FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i]) }
@@ -828,16 +917,15 @@ Thank you for using FlyData!
 
       # Sync flush
 
-      def flush_buffer_and_stop(tables = [])
+      def flush_buffer_and_stop(tables = [], options = {})
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer
         sender.stop(quiet: true)
-        if opts.skip_flush?
-          log_info_stdout("Skip waiting for server data processing.")
-        else
-          wait_for_server_data_processing(
-            timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: tables)
-        end
+
+        return if options[:skip_flush]
+
+        wait_for_server_data_processing(
+          timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: tables)
       end
 
       # Utility methods
@@ -845,16 +933,19 @@ Thank you for using FlyData!
       def set_current_tables(input_tables = nil, options = {})
         de = data_entry
         sync_fm = create_sync_file_manager(de)
-        @input_tables = input_tables || []
+        sync_info = sync_fm.load_sync_info
+        sync_resumed = options[:resume] && !!sync_info
+
         @invalid_tables = de['mysql_data_entry_preference']['invalid_tables'] # tables marked as invalid as a result of previous check
         @full_tables = options[:include_invalid_tables] ? de['mysql_data_entry_preference']['tables'] + @invalid_tables : de['mysql_data_entry_preference']['tables']
 
         @new_tables = sync_fm.get_new_table_list(@full_tables, "pos")
         @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
 
-        @full_initial_sync = (@new_tables == @full_tables)
-
-        sync_resumed = load_sync_info(sync_fm)
+        @input_tables = sync_resumed ? sync_info[:tables] : input_tables
+        @input_tables ||= []
+        @full_initial_sync = sync_resumed ? sync_info[:initial_sync] :
+                                            (@new_tables == @full_tables)
 
         sync_fm.close
 
@@ -875,19 +966,6 @@ Thank you for using FlyData!
         end
       end
 
-      def load_sync_info(sync_fm)
-        sync_resumed = false
-        # for debug
-        raise "!AssertionError. set_current_tables needs to be called in advance" if @full_tables.nil?
-
-        if (rs = sync_fm.load_sync_info)
-          @full_initial_sync = rs[:initial_sync]
-          @input_tables = rs[:tables]
-          sync_resumed = true
-        end
-        sync_resumed
-      end
-
       def target_tables
         if @full_initial_sync
           @full_tables

data/lib/flydata/sync_file_manager.rb

@@ -1,4 +1,5 @@
 require 'mysql2'
+require 'flydata-core/query_job'
 
 module Flydata
   class SyncFileManager
@@ -69,7 +70,7 @@ module Flydata
       table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
       new_tables = []
       tables.each do |table|
-        new_tables << table unless  File.exists?(File.join(table_positions_dir_path, "#{table}.#{file_type}"))
+        new_tables << table unless File.exists?(File.join(table_positions_dir_path, "#{table}.#{file_type}"))
       end
       new_tables
     end
@@ -184,6 +185,7 @@ module Flydata
       f = @table_position_files[table_name]
       seq = f.read
       seq = seq.to_i + 1
+      seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
       begin
         yield(seq)
       ensure

data/spec/flydata/command/sync_spec.rb

@@ -108,6 +108,7 @@ module Flydata
         end
         context 'with no stream option' do
           before do
+            expect(default_sync_fm).to receive(:save_sync_info).once
             expect(subject).to receive(:free_disk_space).and_return(disk_byte)
           expect(File).to receive(:dirname)
           end
@@ -128,6 +129,7 @@ module Flydata
         end
         context 'with stream option' do
           it 'will export to io' do
+            expect(default_sync_fm).to receive(:save_sync_info).once
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.any_instance.should_receive(:dump)
 
             subject.send(:generate_mysqldump, default_data_entry, default_sync_fm, false)

data/spec/flydata/compatibility_check_spec.rb

@@ -234,11 +234,12 @@ module Flydata
       end
       let(:normal_table) { {"table_name"=>"normal_table", "table_type"=>"BASE TABLE", "engine"=>"InnoDB"} }
       let(:engine_table) { {"table_name"=>"engine_table", "table_type"=>"BASE TABLE", "engine"=>"MEMORY"} }
+      let(:blackhole_table) { {"table_name"=>"blackhole_table", "table_type"=>"BASE TABLE", "engine"=>"BLACKHOLE"} }
       let(:view) { {"table_name"=>"view_table", "table_type"=>"VIEW", "engine"=>nil} }
       let(:client) { double('client') }
       let(:subject_object) { MysqlCompatibilityCheck.new(default_data_port,test_data_entry, {}) }
       let(:error) { FlydataCore::MysqlCompatibilityError }
-      let(:base_error_msg) { "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: %s" }
+      let(:base_error_msg) { "FlyData does not support VIEW and MEMORY,BLACKHOLE STORAGE ENGINE table. Remove following tables from data entry: %s" }
       subject { subject_object.check_mysql_table_types }
       before do
         allow(Mysql2::Client).to receive(:new).and_return(client)
@@ -256,6 +257,11 @@ module Flydata
         let(:table_list) { [ engine_table ] }
         it { expect{subject}.to raise_error(error, /#{error_msg}/) }
       end
+      context "where data entry has BLACKHOLE engine table" do
+        let(:error_msg) { base_error_msg %  blackhole_table['table_name'] }
+        let(:table_list) { [ blackhole_table ] }
+        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
+      end
       context "where data entry has the VIEW" do
         let(:error_msg) { base_error_msg % view['table_name'] }
         let(:table_list) { [ view ] }

data/spec/flydata/sync_file_manager_spec.rb

@@ -236,8 +236,8 @@ module Flydata
       end
       subject { subject_object }
       context 'when an exception happens in the block' do
-        let(:first_pos) { 1 }
-        let(:last_pos) { first_pos + 1 }
+        let(:first_pos) { '1.sync' }
+        let(:last_pos) { first_pos.to_i + 1 }
         before do
           subject.increment_and_save_table_position(test_table) do |seq|
             expect(seq).to eq first_pos

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-08 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -486,8 +486,10 @@ files:
 - flydata-core/lib/flydata-core/logger.rb
 - flydata-core/lib/flydata-core/mysql/command_generator.rb
 - flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
+- flydata-core/lib/flydata-core/option_validator.rb
 - flydata-core/lib/flydata-core/query_job.rb
 - flydata-core/lib/flydata-core/query_job/redshift.rb
+- flydata-core/lib/flydata-core/query_job/table_status.rb
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
 - flydata-core/lib/flydata-core/table_def.rb
@@ -501,6 +503,7 @@ files:
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/mysql/command_generator_spec.rb
 - flydata-core/spec/mysql/compatibility_checker.rb
+- flydata-core/spec/option_validator_spec.rb
 - flydata-core/spec/query_job/redshift_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb