flydata 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8603d9b83a0d1f764610c72f23c822b8442322b
-  data.tar.gz: 653eb4641b6602f760156e8b8bd83168642e40a8
+  metadata.gz: 498996a0198e22b19c7417c3b40fe1be38f2826f
+  data.tar.gz: 1e069123fe4fead123563f63f43a4a5f0f663394
 SHA512:
-  metadata.gz: 0d3835c0914fcb12a951e58584a435528fc42f3aa7905420b02905ac9eca26354b122d2b82d45b8b83a8ef10a6b9b13ce3684b6336508aee8a207783fe2e693b
-  data.tar.gz: 6c0819ccc0c95efb01e6c6728c21e25d034ca7612ea02a2ed3671028c7c147065e1fd47f15f902c57cb5ce5efd39a47cadc3c5db30e07a736d94cd37e252bbbe
+  metadata.gz: 395895f16250333b5f7f094ff063cfafacc404c4f1628f29f10f3585844852e452f96ac18effc39043238ac770b3afbaeeeabb78095b3e3b6eae2aff44be0409
+  data.tar.gz: f7e2c589cdc767b5d0edc1f134755e379bb6808903ec411532a6d575388b7e2a823c0cef4766d501806f1e05dd13f526fe44a4ac85af893423499240a0367d6a

data/VERSION

@@ -1 +1 @@
-0.5.6
+0.5.7

data/flydata-core/lib/flydata-core/agent.rb

@@ -0,0 +1,8 @@
+module FlydataCore
+
+class Agent
+  # fluentd buffer path
+  BUFFER_PATH = 'buffer/client_buf'
+end
+
+end

data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb

@@ -0,0 +1,72 @@
+module FlydataCore
+module Mysql
+
+class BinlogPos
+  def initialize(binlog_str_or_filename_or_hash, binlog_pos = nil)
+    arg = binlog_str_or_filename_or_hash
+    if binlog_pos
+      @pos = binlog_pos
+      @filename = arg
+    elsif arg.kind_of?(String)
+      @filename, @pos = arg.split("\t")
+    elsif arg.kind_of?(Hash)
+      @pos = arg[:pos]
+      @filename = arg[:filename]
+      @filename ||= arg[:binfile]
+    end
+    if @filename.nil? || @pos.nil?
+      raise "Invalid initialize argument (#{arg}, #{binlog_pos})"
+    end
+    @pos = @pos.to_i
+  end
+
+  attr_reader :filename, :pos
+
+  def <=>(other_pos)
+    if other_pos.nil?
+      raise ArgumentError.new("comparison of BinlosPos with nil failed")
+    end
+
+    other_pos = BinlogPos.new(other_pos) unless other_pos.kind_of?(BinlogPos)
+
+    if @filename < other_pos.filename
+      -1
+    elsif @filename == other_pos.filename
+      if @pos < other_pos.pos
+        -1
+      elsif @pos == other_pos.pos
+        0
+      else
+        1
+      end
+    else
+      1
+    end
+  end
+  def <(other_pos)
+    self.<=>(other_pos) < 0
+  end
+  def <=(other_pos)
+    self.<=>(other_pos) <= 0
+  end
+  def ==(other_pos)
+    return false if other_pos.nil?
+    self.<=>(other_pos) == 0
+  end
+  def !=(other_pos)
+    !self.==(other_pos)
+  end
+  def >(other_pos)
+    self.<=>(other_pos) > 0
+  end
+  def >=(other_pos)
+    self.<=>(other_pos) >= 0
+  end
+
+  def to_s
+    "#{@filename}\t#{@pos}"
+  end
+end
+
+end
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.6 ruby lib
+# stub: flydata 0.5.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.6"
+  s.version = "0.5.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-09-22"
+  s.date = "2015-09-23"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
     "flydata-core/Gemfile.lock",
     "flydata-core/circle.yml",
     "flydata-core/lib/flydata-core.rb",
+    "flydata-core/lib/flydata-core/agent.rb",
     "flydata-core/lib/flydata-core/config/user_maintenance.rb",
     "flydata-core/lib/flydata-core/core_ext.rb",
     "flydata-core/lib/flydata-core/core_ext/module.rb",
@@ -47,6 +48,7 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb",
     "flydata-core/lib/flydata-core/fluent/config_helper.rb",
     "flydata-core/lib/flydata-core/logger.rb",
+    "flydata-core/lib/flydata-core/mysql/binlog_pos.rb",
     "flydata-core/lib/flydata-core/mysql/command_generator.rb",
     "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
     "flydata-core/lib/flydata-core/mysql/config.rb",
@@ -92,6 +94,7 @@ Gem::Specification.new do |s|
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
+    "lib/flydata/agent.rb",
     "lib/flydata/api/agent.rb",
     "lib/flydata/api/base.rb",
     "lib/flydata/api/data_entry.rb",
@@ -228,7 +231,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.6"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata.rb

@@ -29,6 +29,7 @@ module Flydata
   FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
   FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
   FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
+  FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata.lock')
   FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
 
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')

data/lib/flydata/agent.rb

@@ -0,0 +1,17 @@
+require 'flydata-core/agent'
+
+module Flydata
+
+class Agent
+  def initialize(flydata_home)
+    @flydata_home = flydata_home
+  end
+
+  def delete_buffer_files
+    files = Dir.glob("#{@flydata_home}/#{FlydataCore::Agent::BUFFER_PATH}*")
+    File.delete(*files)
+    files
+  end
+end
+
+end

data/lib/flydata/api/data_entry.rb

@@ -10,13 +10,19 @@ module Flydata
         super
       end
 
-      def buffer_stat(data_entry_id, params = {})
-        tables = params[:tables] ? params[:tables].join(',') : ''
-        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{params[:mode]}?tables=#{tables}")
+      def buffer_stat(data_entry_id, options = {})
+        tables = options[:tables] ? options[:tables].join(',') : ''
+        @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}?tables=#{tables}")
       end
 
-      def cleanup_sync(data_entry_id, tables)
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})
+      def table_status(data_entry_id, options = {})
+        tables = options[:tables] ? options[:tables].join(',') : ''
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
+      end
+
+      def cleanup_sync(data_entry_id, tables, options = {})
+        params = options.merge({tables: tables.join(',')})
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, params)
       end
 
       # Update validity of tables

data/lib/flydata/command/sender.rb

@@ -22,6 +22,11 @@ module Flydata
           return
         end
 
+        if agent_locked?
+          $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
+          raise "Agent was not shutdown properly. Agent is stopped temporarily in order to avoid sync consistency issues.  Please contact support@flydata.com to solve the issue."
+        end
+
         # Ends orphan_proceses if there is any
         orphan_processes.each do |pid|
           Process.kill(:TERM, pid)
@@ -72,8 +77,9 @@ EOF
         raise 'Something has gone wrong..'
       end
       def flush_client_buffer(options = {})
+        force_flush = options.has_key?(:force) ? options[:force] : true
         unless process_exist?
-          return true if client_buffer_empty?
+          return true if !force_flush || client_buffer_empty?
           log_info_stdout("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
           raw_start(options)
         end
@@ -124,6 +130,10 @@ EOF
           raise 'Something has gone wrong...'
         end
       end
+      # call the method only when no legit process is running.
+      def agent_locked?
+        File.exists?(FLYDATA_LOCK)
+      end
 
       private
       # Return a list of fluentd parent processes run by the same user for the

data/lib/flydata/command/sync.rb

@@ -2,6 +2,7 @@ require 'msgpack'
 require 'mysql2'
 require 'rest_client'
 require 'sys/filesystem'
+require 'flydata/agent'
 require 'flydata/command/base'
 require 'flydata/command/conf'
 require 'flydata/command/sender'
@@ -13,6 +14,7 @@ require 'flydata/parser/mysql/dump_parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
 require 'flydata-core/table_def'
+require 'flydata-core/mysql/binlog_pos'
 require 'flydata/mysql/table_ddl'
 require 'flydata-core/mysql/command_generator'
 #require 'ruby-prof'
@@ -159,7 +161,7 @@ EOS
         message = ''
         if sync_resumed && !tables.empty?
           log_info_stdout <<EOS
-Initial sync is in progress.  In this case, you can only reset the initial sync.To reset specific table(s), please resume and complete the initial sync by running the 'flydata start' command first.
+Initial sync is in progress.  In this case, you can only reset the initial sync.  To reset specific table(s), please resume and complete the initial sync by running the 'flydata start' command first.
 If you'd like to reset the initial sync in progress, run the 'flydata reset' command with no arguments.
 EOS
           return
@@ -308,6 +310,205 @@ EOS
       end
       run_exclusive :fix_binlogpos
 
+      def repair
+        de = data_entry
+        set_current_tables
+        # Stop agent.  Check sync and make sure the state is :STUCK_AT_UPLOAD
+        # Get table status for the tables.
+        status, pos_mismatch_tables, gap_tables, table_status_hash = _check(stop_agent:true)
+        if status.include? :STUCK_AT_PROCESS
+          e = AgentError.new("Data is stuck while processing")
+          e.description = <<EOS
+Data appears to be stuck while processing.  Contact FlyData Support (support@flydata.com) to solve the issue.
+EOS
+          raise e
+        end
+
+        if status.include? :OK
+          log_info_stdout
+          log_info_stdout "Sync is in good condition.  Nothing to repair."
+          return
+        end
+
+        tables = []
+        gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
+        pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
+        tables = gt | pt # position mismatch can be due to query queue items discarded by the copy handler so it also needs to be in the target table list.
+
+        log_info_stdout <<EOS
+
+The following issues have been found and will be repaired.
+
+EOS
+        log_info_stdout <<EOS unless tables.empty?
+ - Sync is broken due to missing data for the following tables
+     #{tables.join("\n     ")}
+
+EOS
+
+        log_info_stdout <<EOS if status.include? :ABNORMAL_SHUTDOWN
+ - Agent process was not shut down correctly.  Files may be corrupt.
+
+EOS
+
+        return unless ask_yes_no("Proceed?")
+
+        oldest_binlog = get_oldest_available_binlog
+        unrepairable_tables = []
+        # Determine the master binlog positions
+        sent_binlog_pos = nil
+        @full_tables.each do |table|
+          binlog_str = table_status_hash[table]["src_pos"]
+          unless binlog_str
+            raise "Table `#{table}` has no 'src_pos' in its table status"
+          end
+          binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
+          if tables.empty?
+            if sent_binlog_pos.nil? || sent_binlog_pos < binlog_pos
+              sent_binlog_pos = binlog_pos
+            end
+          else
+            if tables.include?(table)
+              if sent_binlog_pos.nil? || sent_binlog_pos > binlog_pos
+                if oldest_binlog && binlog_pos < oldest_binlog
+                  unrepairable_tables << table
+                else
+                  sent_binlog_pos = binlog_pos
+                end
+              end
+            end
+          end
+        end
+        if oldest_binlog && sent_binlog_pos < oldest_binlog
+          e = AgentError.new("Repair failed due to expired binlog")
+          e.description = <<EOS
+Repair failed because the starting binlog position `#{sent_binlog_pos} no longer exists.  Run full initial sync instead.
+EOS
+          raise e
+        end
+        master_binlog_pos = FlydataCore::Mysql::BinlogPos.new(sent_binlog_pos.filename, 4)
+
+        # Delete agent buffer
+        log_info_stdout "Deleting data in the agent buffer..."
+        files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
+        unless files.empty?
+          $log.debug "Deleted buffer files\n  " + files.join("\n  ")
+        end
+
+        # Delete query queue items for the tables
+        log_info_stdout "Deleting data stuck in the server buffer..."
+        cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
+
+        # Save the positions (binlog and seq)
+        log_info_stdout "Fixing table positions..."
+        @full_tables.each do |table|
+           binlog_str = table_status_hash[table]["src_pos"]
+           unless binlog_str
+             raise "Table `#{table}` has no 'src_pos' in its table status"
+           end
+           binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
+           pos = table_status_hash[table]["seq"]
+           old_binlog_pos, old_pos = save_table_positions(table, binlog_pos, pos)
+           if pos.to_i != old_pos.to_i && !tables.include?(table)
+             $log.debug "Fixed broken table position.  table:#{table} pos:#{old_pos} -> #{pos}"
+           end
+        end
+
+        log_info_stdout "Fixing the master position files..."
+        save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
+
+        # Remove the lock file if exists.
+        File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
+
+        log_info_stdout "Repair is done.  Start Agent with `flydata start` command."
+      end
+      run_exclusive :repair
+
+
+      def check(options = {})
+        status, pos_mismatch_tables, gap_tables = _check(options)
+
+        if status.include? :OK
+          message = "\nNo errors are found.  Sync is clean.\n"
+        else
+          message = "\nFollowing errors are found.\n"
+
+          if status.include? :STUCK_AT_PROCESS
+            message += " - Data is stuck while processing\n"
+          end
+          if status.include? :STUCK_AT_UPLOAD
+            message += " - Data is stuck while uploading\n"
+          end
+          if status.include? :ABNORMAL_SHUTDOWN
+            message += " - Agent was not shut down correctly\n"
+          end
+          if gap_tables
+            message += " - Sync data is missing for the following table(s)\n"
+            gap_tables.each do |bt|
+              message += "     table:#{bt[:table]}\n"
+            end
+            message += "\n"
+          end
+          if pos_mismatch_tables
+            message += " - Incorrect table position(s)\n"
+            pos_mismatch_tables.each do |bt|
+              message += "     table:#{bt[:table]}, agent position:#{bt[:agent_seq] ? bt[:agent_seq] : '(missing)'}, server position:#{bt[:server_seq]}\n"
+            end
+            message += "\n"
+          end
+        end
+        log_info_stdout message
+      end
+      run_exclusive :check
+
+      def _check(options = {})
+        options[:stop_agent] ||= false
+
+        set_current_tables
+        sender = Flydata::Command::Sender.new
+        start_process = !options[:stop_agent] && sender.process_exist?
+        pos_mismatch_tables = nil
+        gap_tables = nil
+        data_stuck_at = nil
+        abnormal_shutdown = false
+        begin
+          begin
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 55)
+          rescue ServerDataProcessingTimeout => e
+            data_stuck_at = e.state
+          end
+
+          # Agent is stopped but locked.  There was an abnormal shutdown.
+          abnormal_shutdown = sender.agent_locked?
+          table_status_hash = get_table_status(@full_tables)
+          pos_mismatch_tables = check_position_files(table_status_hash)
+          gap_tables = check_gaps(table_status_hash)
+        ensure
+          Flydata::Command::Sender.new.start(quiet: true) if start_process
+        end
+
+        status = []
+        if data_stuck_at == :PROCESS
+          status << :STUCK_AT_PROCESS
+        end
+        if data_stuck_at == :UPLOAD
+          status << :STUCK_AT_UPLOAD
+        end
+        if gap_tables
+          status << :TABLE_GAPS
+        end
+        if pos_mismatch_tables
+          status << :TABLE_POS_MISMATCH
+        end
+        if abnormal_shutdown
+          status << :ABNORMAL_SHUTDOWN
+        end
+        if status.empty?
+          status << :OK
+        end
+        [status, pos_mismatch_tables, gap_tables, table_status_hash]
+      end
+
       private
 
       # Initial sync
@@ -711,7 +912,7 @@ EOM
           end
           prev_message = status['message']
           if timeout > 0 && Time.now - start_time > timeout
-            raise ServerDataProcessingTimeout.new
+            raise ServerDataProcessingTimeout.new(nil, state: state)
           end
           print_progress(status)
           sleep 10
@@ -824,12 +1025,12 @@ EOM
         end
       end
 
-      def cleanup_sync_server(de, tables = [])
+      def cleanup_sync_server(de, tables = [], options = {})
         print("Cleaning the queued items on the FlyData Servers.")
         log_info("Cleaning the queued items on the FlyData Servers.")
         worker = Thread.new do
           begin
-            flydata.data_entry.cleanup_sync(de['id'], tables)
+            flydata.data_entry.cleanup_sync(de['id'], tables, options)
           rescue RestClient::RequestTimeout, RestClient::GatewayTimeout
             # server is taking time to cleanup.  Try again
             retry
@@ -932,13 +1133,14 @@ Thank you for using FlyData!
 
       def flush_buffer_and_stop(tables = [], options = {})
         sender = Flydata::Command::Sender.new
-        sender.flush_client_buffer
+        sender.flush_client_buffer(options)
         sender.stop(quiet: true)
 
         return if options[:skip_flush]
 
-        wait_for_server_data_processing(
-          timeout: SERVER_DATA_PROCESSING_TIMEOUT, tables: tables)
+        timeout = options.has_key?(:timeout) ? options[:timeout]
+                                             : SERVER_DATA_PROCESSING_TIMEOUT
+        wait_for_server_data_processing( timeout: timeout, tables: tables)
       end
 
       # Utility methods
@@ -1050,6 +1252,100 @@ Thank you for using FlyData!
         end
         raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
       end
+
+      def check_position_files(table_status_hash)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        pos_mismatch_tables = []
+        table_status_hash.keys.collect do |table|
+          table_status = table_status_hash[table]
+
+          server_sequence = table_status["seq"]
+          agent_sequence = sync_fm.get_table_position(table)
+          agent_sequence = agent_sequence.to_i if agent_sequence
+          if server_sequence != agent_sequence
+            pos_mismatch_tables << {table: table, agent_seq: agent_sequence, server_seq: server_sequence}
+          end
+        end
+        pos_mismatch_tables.empty? ? nil : pos_mismatch_tables
+      end
+
+      def check_gaps(table_status_hash)
+        gap_tables = table_status_hash.values.select {|table_status|
+          table_status["num_items"] > 0 && table_status["next_item"] == false
+        }.collect{|table_status| {table: table_status["table_name"]}}
+        gap_tables.empty? ? nil : gap_tables
+      end
+
+      def save_table_positions(table, binlog_pos, pos)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        s = sync_fm.get_table_binlog_pos(table)
+        old_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
+        old_pos = sync_fm.get_table_position(table)
+        if pos.to_i != old_pos.to_i
+          sync_fm.save_table_position(table, pos)
+          $log.debug "table pos updated.  table:#{table} pos:#{old_pos} -> #{pos}"
+        end
+        if binlog_pos != old_binlog_pos
+          sync_fm.save_table_binlog_pos(table, binlog_pos.to_s, destination: :positions)
+          $log.debug "table binlog updated.  table:#{table} binlog:`#{old_binlog_pos}` -> `#{binlog_pos}`"
+        end
+        [old_binlog_pos, old_pos]
+      end
+
+      def save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        s = sync_fm.load_binlog
+        old_master_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
+        s = sync_fm.load_sent_binlog
+        old_sent_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
+        if master_binlog_pos != old_master_binlog_pos
+          sync_fm.save_binlog(master_binlog_pos.to_s)
+          $log.debug "master binlog positions updated. `#{old_master_binlog_pos}` -> `#{master_binlog_pos}`"
+        end
+        if sent_binlog_pos != old_sent_binlog_pos
+          sync_fm.save_sent_binlog(sent_binlog_pos.to_s)
+          $log.debug "sent binlog positions updated. `#{old_sent_binlog_pos}` -> `#{sent_binlog_pos}`"
+        end
+        [old_master_binlog_pos, old_sent_binlog_pos]
+      end
+
+      def get_table_status(tables)
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        result = flydata.data_entry.table_status(de['id'], mode: env_mode, tables: tables)
+        result = result["table_status"]
+
+        table_status_hash = result.inject({}){|h, ts| h[ts["table_name"]] = ts; h}
+        missing_tables = tables - table_status_hash.keys
+        unless missing_tables.empty?
+          raise "table status is not available for these table(s): #{missing_tables.join(",")}"
+        end
+
+        populate_initial_binlog_positions(table_status_hash, sync_fm)
+        table_status_hash
+      end
+
+      # table_status has no binlog position for sequence "0".  Populate the info
+      # from 'table.binlog.pos.init' file.
+      def populate_initial_binlog_positions(table_status_hash, sync_fm)
+        table_status_hash.keys.each do |table|
+          if table_status_hash[table]["src_pos"] == "-"
+            init_binlog_pos =  sync_fm.get_table_binlog_pos_init(table)
+            unless init_binlog_pos
+              raise "File `#{table}.binlog.pos.init` is missing"
+            end
+            table_status_hash[table]["src_pos"] = init_binlog_pos
+          end
+        end
+      end
+
+      # TODO implement
+      def get_oldest_available_binlog
+        nil
+      end
     end
   end
 end

data/lib/flydata/errors.rb

@@ -29,6 +29,11 @@ EOM
 end
 
 class ServerDataProcessingTimeout < AgentError
+  def initialize(message, options)
+    super(message)
+    @state = options[:state]
+  end
+  attr_reader :state
 end
 
 class DumpParseError < AgentError

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -135,9 +135,17 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
     @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
+    @lock_file = Flydata::FLYDATA_LOCK
   end
 
   def start
+    if File.exists?(@lock_file)
+      $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
+      @abort = true
+      Process.kill(:TERM, Process.ppid)
+      return
+    end
+
     super
     @idle_event_detector.start do |reason, timestamp|
       case reason
@@ -155,6 +163,8 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
     positions_path = @context.sync_fm.table_positions_dir_path
     Dir.mkdir positions_path unless File.exists? positions_path
+
+    File.open(@lock_file, "w") {|f| f.write(Process.pid)}
   rescue Binlog::Error
     if (/basic_string::_M_replace_aux/ === $!.to_s)
       # TODO Fix the root cause in mysql-replication-listener
@@ -172,6 +182,8 @@ EOS
   end
 
   def run
+    return if instance_variable_defined? :@abort
+
     @context.table_meta.update
     Flydata::Mysql::TableDdl.migrate_tables(@context.tables, @db_opts,
                                             @context.sync_fm, @position_file,
@@ -252,6 +264,13 @@ EOS
   end
 
   def shutdown
+    return if instance_variable_defined? :@abort
+
+    if File.exists?(@lock_file) &&
+       Process.pid == File.open(@lock_file, "r") {|f| f.read}.to_i
+      File.delete(@lock_file)
+    end
+
     @idle_event_detector.stop
     if @thread and @thread.alive?
       $log.info "Requesting stop Kodama"

data/lib/flydata/sync_file_manager.rb

@@ -6,9 +6,12 @@ module Flydata
     DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
     BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
     TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
+    SYNC_TABLE_POSITIONS = 0
+
     def initialize(data_entry)
       @data_entry = data_entry
       @table_position_files = {} # File objects keyed by table name
+      @sync_table_positions_count = SYNC_TABLE_POSITIONS
     end
 
     def close
@@ -112,6 +115,13 @@ module Flydata
       end
     end
 
+    def load_sent_binlog(file_path = sent_binlog_path)
+      return nil unless File.exists?(file_path)
+      f, pos = IO.read(file_path).strip.split("\t")
+      return nil if f.nil? || f.empty? || pos.nil?
+      { binfile: f, pos: pos.to_i }
+    end
+
     def sent_binlog_path(master_binlog_path = binlog_path)
       unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
         raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
@@ -174,7 +184,7 @@ module Flydata
       file = File.join(table_positions_dir_path, table_name + ".pos")
       retry_count = 0
       begin
-        @table_position_files[table_name] ||= File.open(file, "r+")
+        @table_position_files[table_name] ||= (f = File.open(file, File::RDWR); f.sync = true; f)
       rescue Errno::ENOENT
         raise if retry_count > 0 # Already retried.  Must be a differentfile causing the error
         # File not exist.  Create one with initial value of '0'
@@ -184,22 +194,43 @@ module Flydata
       end
       f = @table_position_files[table_name]
       seq = f.read
+      prev_seq_len = seq.size
+      f.rewind
       seq = seq.to_i + 1
       seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
-      begin
-        yield(seq)
-      ensure
-        # when an error happened in yield, the sequence number should remain
-        # as is.  For the next call to read the value correctly, the position
-        # must be rewound.
-        f.rewind
+      new_seq_len = seq.to_s.size
+
+      seq_to_write =  seq.to_s
+      if new_seq_len < prev_seq_len
+        seq_to_write += " " * (prev_seq_len - new_seq_len)
       end
-      f.truncate(0)
-      f.write(seq)
-      f.flush
+      # logical transaction starts
+      yield(seq)
+      f.write(seq_to_write)
+      if @sync_table_positions_count > 1
+        @sync_table_positions_count -= 1
+      elsif @sync_table_positions_count == 1
+        f.fsync
+        @sync_table_positions_count = SYNC_TABLE_POSITIONS
+      end
+      # logical transaction ends
+      f.truncate(new_seq_len) if new_seq_len < prev_seq_len
       f.rewind
     end
 
+    def save_table_position(table_name, seq)
+      file = File.join(table_positions_dir_path, table_name + ".pos")
+
+      File.open(file, "w") {|f| f.write(seq)}
+    end
+
+    def get_table_position(table_name)
+      file = File.join(table_positions_dir_path, table_name + ".pos")
+      return nil unless File.exists?(file)
+
+      File.open(file) {|f| f.read}
+    end
+
     def sync_info_file
       File.join(dump_dir, "sync.info")
     end
@@ -217,8 +248,8 @@ module Flydata
         tables: items[1].split(" ") }
     end
 
-    def get_table_binlog_pos(table_name)
-      file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
+    def get_table_binlog_pos_init(table_name)
+      file = File.join(table_positions_dir_path, table_name + ".binlog.pos.init")
       return nil unless File.exists?(file)
       File.open(file, 'r').readline
     end
@@ -263,15 +294,28 @@ module Flydata
       end
     end
 
-    def save_table_binlog_pos(tables, binlog_pos)
+    def save_table_binlog_pos(tables, binlog_pos, options = {})
+      dest_dir = case options[:destination]
+                 when :positions; table_positions_dir_path
+                 when :dump; dump_dir
+                 else dump_dir
+                 end
+
+      tables = [ tables ] unless tables.kind_of?(Array)
       tables.each do |table_name|
-        file = File.join(dump_dir, table_name + ".binlog.pos")
+        file = File.join(dest_dir, table_name + ".binlog.pos")
         File.open(file, "w") do |f|
           f.write(binlog_content(binlog_pos))
         end
       end
     end
 
+    def get_table_binlog_pos(table_name)
+      file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
+      return nil unless File.exists?(file)
+      File.open(file, 'r').readline
+    end
+
     def install_table_binlog_files(tables)
       FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path)
       tables.each do |table_name|
@@ -331,7 +375,9 @@ module Flydata
     end
 
     def binlog_content(binlog_pos)
-      [binlog_pos[:binfile], binlog_pos[:pos]].join("\t")
+      binlog_pos.kind_of?(Hash) ?
+               [binlog_pos[:binfile], binlog_pos[:pos]].join("\t")
+             : binlog_pos
     end
 
     def load_mysql_table_marshal_dump

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-22 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -474,6 +474,7 @@ files:
 - flydata-core/Gemfile.lock
 - flydata-core/circle.yml
 - flydata-core/lib/flydata-core.rb
+- flydata-core/lib/flydata-core/agent.rb
 - flydata-core/lib/flydata-core/config/user_maintenance.rb
 - flydata-core/lib/flydata-core/core_ext.rb
 - flydata-core/lib/flydata-core/core_ext/module.rb
@@ -484,6 +485,7 @@ files:
 - flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb
 - flydata-core/lib/flydata-core/fluent/config_helper.rb
 - flydata-core/lib/flydata-core/logger.rb
+- flydata-core/lib/flydata-core/mysql/binlog_pos.rb
 - flydata-core/lib/flydata-core/mysql/command_generator.rb
 - flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
 - flydata-core/lib/flydata-core/mysql/config.rb
@@ -529,6 +531,7 @@ files:
 - flydata.gemspec
 - lib/fly_data_model.rb
 - lib/flydata.rb
+- lib/flydata/agent.rb
 - lib/flydata/api/agent.rb
 - lib/flydata/api/base.rb
 - lib/flydata/api/data_entry.rb
@@ -682,7 +685,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent