flydata 0.5.7 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 498996a0198e22b19c7417c3b40fe1be38f2826f
-  data.tar.gz: 1e069123fe4fead123563f63f43a4a5f0f663394
+  metadata.gz: 4c79a1c7705cdd2112d4fd3465f311bf31583471
+  data.tar.gz: b5a6e9d153bbf84dca5d582de5bc5df460d01a96
 SHA512:
-  metadata.gz: 395895f16250333b5f7f094ff063cfafacc404c4f1628f29f10f3585844852e452f96ac18effc39043238ac770b3afbaeeeabb78095b3e3b6eae2aff44be0409
-  data.tar.gz: f7e2c589cdc767b5d0edc1f134755e379bb6808903ec411532a6d575388b7e2a823c0cef4766d501806f1e05dd13f526fe44a4ac85af893423499240a0367d6a
+  metadata.gz: 30e4e6fee11a8ba1b76f0e7c14e05f5d723a90655fc0e4cc0e42724f6edcef4d73514cf5d970becf3178a224172a9aa33d4fbe1bbbf27e4a0c0b73419b15fe34
+  data.tar.gz: 9ae07ffd9eb356a7947f5c8c1331f48181cc1ac8f76392f5c2d92122f4cfff38ef6f2d94e4ac001914bbb1d537423ff6ba060650d9776154b8b865285601cb77

data/VERSION

@@ -1 +1 @@
-0.5.7
+0.5.8

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -1,15 +1,13 @@
 require 'flydata-core/errors'
+require 'flydata-core/mysql/config'
 require 'mysql2'
 
 module FlydataCore
   module Mysql
     class CompatibilityChecker
       def initialize(option = {})
-        if option && option.has_key?(:host) && option[:host] == 'localhost'
-          option = option.dup
-          option[:host] = '127.0.0.1' # avoid socket access which Agent never uses
-        end
-        @option = option || {}
+        option ||= {}
+        @option = option.merge FlydataCore::Mysql::Config.build_mysql_db_opts(option)
       end
 
       def do_check(option = @option, &block)

data/flydata-core/lib/flydata-core/mysql/config.rb

@@ -25,6 +25,9 @@ module FlydataCore
           db_opts[:sslcipher] = db_opts[:ssl_cipher]
         end
 
+        # to avoid socket access
+        db_opts[:host] = '127.0.0.1' if db_opts[:host] == 'localhost'
+
         db_opts
       end
     end

data/flydata-core/spec/mysql/config_spec.rb

@@ -19,7 +19,7 @@ module FlydataCore
 
         context 'with basic conf' do
           it { is_expected.to eq(
-            host: 'localhost',
+            host: '127.0.0.1',
             port: 1234,
             username: 'testuser',
             password: 'password',
@@ -33,7 +33,7 @@ module FlydataCore
           context 'when ssl_ca is not empty' do
             let(:conf) { base_conf.merge('ssl_ca' => 'test_path') }
             it { is_expected.to eq(
-              host: 'localhost',
+              host: '127.0.0.1',
               port: 1234,
               username: 'testuser',
               password: 'password',
@@ -47,7 +47,7 @@ module FlydataCore
           context 'when ssl_ca is empty' do
             let(:conf) { base_conf.merge('ssl_ca' => '') }
             it { is_expected.to eq(
-              host: 'localhost',
+              host: '127.0.0.1',
               port: 1234,
               username: 'testuser',
               password: 'password',
@@ -62,7 +62,7 @@ module FlydataCore
           context 'when ssl_cipher is not empty' do
             let(:conf) { base_conf.merge('ssl_cipher' => 'AAAA::BBBB') }
             it { is_expected.to eq(
-              host: 'localhost',
+              host: '127.0.0.1',
               port: 1234,
               username: 'testuser',
               password: 'password',
@@ -76,7 +76,7 @@ module FlydataCore
           context 'when ssl_cipher is empty' do
             let(:conf) { base_conf.merge('ssl_cipher' => '') }
             it { is_expected.to eq(
-              host: 'localhost',
+              host: '127.0.0.1',
               port: 1234,
               username: 'testuser',
               password: 'password',

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.7 ruby lib
+# stub: flydata 0.5.8 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.7"
+  s.version = "0.5.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-09-23"
+  s.date = "2015-09-25"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]

data/lib/flydata/command/sender.rb

@@ -23,8 +23,14 @@ module Flydata
         end
 
         if agent_locked?
-          $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
-          raise "Agent was not shutdown properly. Agent is stopped temporarily in order to avoid sync consistency issues.  Please contact support@flydata.com to solve the issue."
+          log_info_stdout("Agent was not shut down properly.  Agent will check the status and fix itself if necessary.")
+          repair_opts = Flydata::Command::Sync.slop_repair
+          repair_opts.parse!(["-y"])
+          sync = Flydata::Command::Sync.new(repair_opts)
+          sync.send(:_repair) # call internal method to bypass command lock
+          if agent_locked?
+            raise "Agent was not able to recover from the previous unexpected shutdown.  Please contact support@lyfdata.com to resolve the issue."
+          end
         end
 
         # Ends orphan_proceses if there is any

data/lib/flydata/command/sync.rb

@@ -310,7 +310,106 @@ EOS
       end
       run_exclusive :fix_binlogpos
 
+      # Command: flydata sync:repair
+      #  - Arguments
+      def self.slop_repair
+        Slop.new do
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+        end
+      end
+
       def repair
+        _repair
+      end
+      run_exclusive :repair
+
+      def check(options = {})
+        status, pos_mismatch_tables, gap_tables = _check(options)
+
+        if status.include? :OK
+          message = "\nNo errors are found.  Sync is clean.\n"
+        else
+          message = "\nFollowing errors are found.\n"
+
+          if status.include? :STUCK_AT_PROCESS
+            message += " - Data is stuck while processing\n"
+          end
+          if status.include? :STUCK_AT_UPLOAD
+            message += " - Data is stuck while uploading\n"
+          end
+          if status.include? :ABNORMAL_SHUTDOWN
+            message += " - Agent was not shut down correctly\n"
+          end
+          if gap_tables
+            message += " - Sync data is missing for the following table(s)\n"
+            gap_tables.each do |bt|
+              message += "     table:#{bt[:table]}\n"
+            end
+            message += "\n"
+          end
+          if pos_mismatch_tables
+            message += " - Incorrect table position(s)\n"
+            pos_mismatch_tables.each do |bt|
+              message += "     table:#{bt[:table]}, agent position:#{bt[:agent_seq] ? bt[:agent_seq] : '(missing)'}, server position:#{bt[:server_seq]}\n"
+            end
+            message += "\n"
+          end
+        end
+        log_info_stdout message
+      end
+      run_exclusive :check
+
+      private
+
+      def _check(options = {})
+        options[:stop_agent] ||= false
+
+        set_current_tables
+        sender = Flydata::Command::Sender.new
+        start_process = !options[:stop_agent] && sender.process_exist?
+        pos_mismatch_tables = nil
+        gap_tables = nil
+        data_stuck_at = nil
+        abnormal_shutdown = false
+        begin
+          begin
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 55)
+          rescue ServerDataProcessingTimeout => e
+            data_stuck_at = e.state
+          end
+
+          # Agent is stopped but locked.  There was an abnormal shutdown.
+          abnormal_shutdown = sender.agent_locked?
+          table_status_hash = get_table_status(@full_tables)
+          pos_mismatch_tables = check_position_files(table_status_hash)
+          gap_tables = check_gaps(table_status_hash)
+        ensure
+          Flydata::Command::Sender.new.start(quiet: true) if start_process
+        end
+
+        status = []
+        if data_stuck_at == :PROCESS
+          status << :STUCK_AT_PROCESS
+        end
+        if data_stuck_at == :UPLOAD
+          status << :STUCK_AT_UPLOAD
+        end
+        if gap_tables
+          status << :TABLE_GAPS
+        end
+        if pos_mismatch_tables
+          status << :TABLE_POS_MISMATCH
+        end
+        if abnormal_shutdown
+          status << :ABNORMAL_SHUTDOWN
+        end
+        if status.empty?
+          status << :OK
+        end
+        [status, pos_mismatch_tables, gap_tables, table_status_hash]
+      end
+
+      def _repair
         de = data_entry
         set_current_tables
         # Stop agent.  Check sync and make sure the state is :STUCK_AT_UPLOAD
@@ -325,12 +424,13 @@ EOS
         end
 
         if status.include? :OK
-          log_info_stdout
+          log_info_stdout ""
           log_info_stdout "Sync is in good condition.  Nothing to repair."
           return
         end
 
-        tables = []
+        gt = []
+        pt = []
         gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
         pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
         tables = gt | pt # position mismatch can be due to query queue items discarded by the copy handler so it also needs to be in the target table list.
@@ -422,94 +522,6 @@ EOS
 
         log_info_stdout "Repair is done.  Start Agent with `flydata start` command."
       end
-      run_exclusive :repair
-
-
-      def check(options = {})
-        status, pos_mismatch_tables, gap_tables = _check(options)
-
-        if status.include? :OK
-          message = "\nNo errors are found.  Sync is clean.\n"
-        else
-          message = "\nFollowing errors are found.\n"
-
-          if status.include? :STUCK_AT_PROCESS
-            message += " - Data is stuck while processing\n"
-          end
-          if status.include? :STUCK_AT_UPLOAD
-            message += " - Data is stuck while uploading\n"
-          end
-          if status.include? :ABNORMAL_SHUTDOWN
-            message += " - Agent was not shut down correctly\n"
-          end
-          if gap_tables
-            message += " - Sync data is missing for the following table(s)\n"
-            gap_tables.each do |bt|
-              message += "     table:#{bt[:table]}\n"
-            end
-            message += "\n"
-          end
-          if pos_mismatch_tables
-            message += " - Incorrect table position(s)\n"
-            pos_mismatch_tables.each do |bt|
-              message += "     table:#{bt[:table]}, agent position:#{bt[:agent_seq] ? bt[:agent_seq] : '(missing)'}, server position:#{bt[:server_seq]}\n"
-            end
-            message += "\n"
-          end
-        end
-        log_info_stdout message
-      end
-      run_exclusive :check
-
-      def _check(options = {})
-        options[:stop_agent] ||= false
-
-        set_current_tables
-        sender = Flydata::Command::Sender.new
-        start_process = !options[:stop_agent] && sender.process_exist?
-        pos_mismatch_tables = nil
-        gap_tables = nil
-        data_stuck_at = nil
-        abnormal_shutdown = false
-        begin
-          begin
-            flush_buffer_and_stop(@full_tables, force: false, timeout: 55)
-          rescue ServerDataProcessingTimeout => e
-            data_stuck_at = e.state
-          end
-
-          # Agent is stopped but locked.  There was an abnormal shutdown.
-          abnormal_shutdown = sender.agent_locked?
-          table_status_hash = get_table_status(@full_tables)
-          pos_mismatch_tables = check_position_files(table_status_hash)
-          gap_tables = check_gaps(table_status_hash)
-        ensure
-          Flydata::Command::Sender.new.start(quiet: true) if start_process
-        end
-
-        status = []
-        if data_stuck_at == :PROCESS
-          status << :STUCK_AT_PROCESS
-        end
-        if data_stuck_at == :UPLOAD
-          status << :STUCK_AT_UPLOAD
-        end
-        if gap_tables
-          status << :TABLE_GAPS
-        end
-        if pos_mismatch_tables
-          status << :TABLE_POS_MISMATCH
-        end
-        if abnormal_shutdown
-          status << :ABNORMAL_SHUTDOWN
-        end
-        if status.empty?
-          status << :OK
-        end
-        [status, pos_mismatch_tables, gap_tables, table_status_hash]
-      end
-
-      private
 
       # Initial sync
 

data/lib/flydata/helper/scheduler.rb

@@ -6,6 +6,7 @@ module Flydata
       include FlydataCore::Logger
 
       RUN_INTERVAL = 1.0  #second
+      RUN_GC_INTERVAL = 60.0 * 5  #seconds
 
       def initialize(helper_conf, server)
         @stop_flag = ServerEngine::BlockingFlag.new
@@ -96,9 +97,19 @@ module Flydata
           end
         end
 
+        run_gc_if_needed
         wait_until_next_turn(start_time)
       end
 
+      def run_gc_if_needed
+        now = Time.now
+        @last_gc_time ||= now
+        if (@last_gc_time + RUN_GC_INTERVAL) < now
+          GC.start
+          @last_gc_time = now
+        end
+      end
+
       def wait_until_next_turn(start_time)
         next_time = start_time + RUN_INTERVAL
         sleep_time = next_time - Time.now.to_f

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.8
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-23 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client