flydata 0.3.24 → 0.4.0

data/lib/flydata/helper/scheduler.rb

@@ -0,0 +1,114 @@
+require 'flydata-core/logger'
+
+module Flydata
+  module Helper
+    class Scheduler
+      include FlydataCore::Logger
+
+      RUN_INTERVAL = 1.0  #second
+
+      def initialize(helper_conf, server)
+        @stop_flag = ServerEngine::BlockingFlag.new
+        @server = server
+        self.logger = server.logger
+        reload(helper_conf)
+      end
+
+      attr_reader :server, :helper_conf
+
+      def start
+        log_info("start")
+        @thread = Thread.new(&method(:run))
+        self
+      end
+
+      def run
+        until @stop_flag.set?
+          run_once
+        end
+      rescue => e
+        log_error("unexpected error during running. error:#{e}\n" + e.backtrace.join("\n"))
+        retry unless @stop_flag.wait_for_set(5.0)
+      ensure
+        log_debug("finish running")
+      end
+
+      def wake
+        log_debug("wake")
+        @stop_flag.reset!
+      end
+
+      def stop
+        log_info("stop")
+        @stop_flag.set!
+      end
+
+      def reload(helper_conf)
+        log_info("reload")
+        @helper_conf = helper_conf
+        update_scheduled_actions
+      end
+
+      def shutdown
+        log_info("shutdown")
+        stop
+        join
+        self
+      end
+
+      def join
+        log_debug("join")
+        stop
+        if @thread
+          @thread.join
+          @thread = nil
+        end
+      end
+
+      # For debug
+
+      def running?
+        !!(@thread and @thread.alive?)
+      end
+
+      def scheduled_actions
+        @scheduled_actions.dup
+      end
+
+      private
+
+      def update_scheduled_actions
+        @scheduled_actions = helper_conf.scheduled_actions
+        @scheduled_actions.each {|k, v|
+          v[:name] = k
+          v[:last_request_time] = -1
+        }
+      end
+
+      def run_once
+        start_time = Time.now.to_f
+
+        # Check scheduled actions
+        @scheduled_actions.each do |name, action|
+          if (start_time - action[:last_request_time]) >= action[:check_interval]
+            ret = @server.action_ownership_channel.request_action(name)
+            action[:last_request_time] = Time.now.to_f
+          end
+        end
+
+        wait_until_next_turn(start_time)
+      end
+
+      def wait_until_next_turn(start_time)
+        next_time = start_time + RUN_INTERVAL
+        sleep_time = next_time - Time.now.to_f
+        @stop_flag.wait_for_set(sleep_time) if sleep_time > 0
+      end
+
+      def custom_log_items
+        super.merge(prefix: '[scheduler]')
+      end
+
+    end
+  end
+end

data/lib/flydata/helper/server.rb

@@ -0,0 +1,66 @@
+require 'serverengine'
+require 'flydata-core/logger'
+
+module Flydata
+  module Helper
+    module Server
+      include FlydataCore::Logger
+
+      OVERWRITE_PARAMETERS = {
+        :worker_type => 'thread',
+        :supervisor => true,
+      }
+
+      def self.run(opts={})
+        # See ServerEngine documents for details:
+        # https://github.com/frsyuki/serverengine
+        ServerEngine.create(Server, Worker) do
+          ConfigParser.parse_file(opts[:config_path]).
+            merge(opts).merge(OVERWRITE_PARAMETERS)
+        end.run
+      end
+
+      def initialize
+        super
+      end
+
+      attr_reader :action_ownership_channel
+
+      # ServerEngine hook point
+      def reload_config
+        super
+        setup_log_format
+        log_info "reload_config"
+        if @scheduler
+          @scheduler.reload(config[:helper])
+        end
+      end
+
+      # ServerEngine hook point
+      def before_run
+        log_debug "before_run"
+        helper_config = config[:helper]
+        @action_ownership_channel = ActionOwnershipChannel.new
+        @scheduler = Scheduler.new(helper_config, self)
+        @scheduler.start
+      end
+
+      # ServerEngine hook point
+      def after_run
+        log_debug "after_run"
+        @scheduler.shutdown if @scheduler
+      end
+
+      def custom_log_items
+        super.merge(prefix: '[server]')
+      end
+
+      def setup_log_format
+        logger.datetime_format = "%Y-%m-%d %H:%M:%S %z "
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime} helper.#{severity.to_s.downcase}: #{msg}\n"
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/worker.rb

@@ -0,0 +1,131 @@
+require 'flydata-core/logger'
+require 'flydata-core/errors'
+
+module Flydata
+  module Helper
+    module Worker
+      include FlydataCore::Logger
+
+      def initialize
+        @stop_flag = ServerEngine::BlockingFlag.new
+        super
+      end
+
+      # ServerEngine hook point
+      def run
+        add_log_context_items(object_id: self.object_id)
+        log_debug("run")
+        until @stop_flag.set?
+          run_once
+        end
+      rescue => e
+        log_error_with_backtrace("Unexpected error.", error: e)
+        @stop_flag.wait_for_set(5.0)
+        raise e
+      end
+
+      # ServerEngine hook point
+      def stop
+        log_debug("stop")
+        @stop_flag.set!
+      end
+
+      # ServerEngine hook point
+      def reload
+        log_debug("reload")
+        super
+        self.stop
+      end
+
+      # ServerEngine hook point
+      def after_start
+        log_debug("after_start")
+      end
+
+      private
+
+      def run_once
+        # Get action ownership
+        action = wait_action_ownership
+        return if action.nil?
+        action_ownership = action[:action_ownership]
+        action_info = action[:action_info]
+
+        # Process action
+        action_id = action_info && action_info[:id] ? "[#{action_info[:id]}]" : "" #scheduled actions (eg:- check_remote_action) don't have id
+        set_log_context_item(:prefix, "[worker][#{action_ownership.action_name}]#{action_id}")
+        begin
+          action_ownership.action_class.new(helper_conf).execute(action_info) do |new_action_name, new_action_info = nil|
+            # request action if the action requires
+            server.action_ownership_channel.request_action(new_action_name, new_action_info)
+          end
+          action_ownership.reset_retry_count
+        rescue => err
+          handle_error(action, err)
+        ensure
+          # Return action ownership to channel
+          server.action_ownership_channel.
+            return_action_ownership(action_ownership)
+        end
+      end
+
+      def handle_error(action, err)
+        action_ownership = action[:action_ownership]
+        action_info = action[:action_info]
+        action_name = action_ownership.action_name
+        r_cnt = action_ownership.increment_retry_count
+        retryable = err.kind_of?(FlydataCore::RetryableError)
+        log_level = :error
+
+        # logging
+        msg = ["Failed to handle action.", {
+          action: action_name,
+          retry_cnt: r_cnt,
+          retryable: retryable,
+          error: err,
+        }]
+
+        if retryable
+          if r_cnt > helper_conf.helper_retry_alert_limit
+            msg[1][:error] = err.original_exception
+          else
+            log_level = :warn
+          end
+        end
+
+        case log_level
+        when :warn
+          log_warn(*msg)
+        else
+          log_error_with_backtrace(*msg)
+        end
+
+        if r_cnt < helper_conf.helper_retry_limit
+          return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
+          server.action_ownership_channel.request_action(action_name, action_info)
+        else
+          log_error("Retry limit reached. Not retrying anymore.")
+          action_ownership.reset_retry_count
+        end
+      end
+
+      def helper_conf
+        config[:helper]
+      end
+
+      def wait_action_ownership
+        action = nil
+        loop do
+          action = server.action_ownership_channel.take_action_ownership(self)
+          break if action
+          return nil if @stop_flag.wait_for_set(0.5)
+        end
+        action
+      end
+
+      def custom_log_items
+        super.merge(prefix: '[worker]')
+      end
+    end
+  end
+end

data/lib/flydata/output/forwarder.rb

@@ -79,6 +79,8 @@ module Flydata
         end
         sock = nil
         retry_count = 0
+        byte_size = @buffer_size
+        record_count = @buffer_record_count
         begin
           sock = connect(pickup_server)
 
@@ -108,7 +110,7 @@ module Flydata
           end
         end
         reset
-        true
+        { byte_size: byte_size, record_count: record_count }
       end
 
       #TODO: Check server status

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,4 +1,5 @@
 require 'fiber'
+require 'io/wait'
 require 'flydata/mysql/mysql_util'
 
 module Flydata
@@ -51,15 +52,6 @@ module Flydata
       end
 
       class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
-        CHANGE_MASTER_TEMPLATE = <<EOS
---
--- Position to start replication or point-in-time recovery from
---
-
--- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
-
-EOS
-
         def dump(file_path = nil, &block)
           unless file_path || block
             raise ArgumentError.new("file_path or block must be given.")
@@ -84,7 +76,22 @@ EOS
               cmd_in.close_write
               cmd_out.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
 
-              first_line = cmd_out.gets # wait until first line comes
+              # wait until the command starts dumping.  Two different ways to
+              # check
+              if file_path
+                # mysqldump dumps to a file `file_path`.  Check if the file
+                # exists and not empty.
+                loop do
+                  if File.size? file_path
+                    break
+                  end
+                  sleep 1
+                end
+              else
+                # mysqldump dumps to an IO `cmd_out`.  Wait until it has
+                # readable contents.
+                cmd_out.wait_readable # wait until first line comes
+              end
               binfile, pos = table_locker.resume
               binlog_pos = {binfile: binfile, pos: pos}
 
@@ -93,7 +100,6 @@ EOS
               # filter dump stream and write data to pipe
               threads << Thread.new do
                 begin
-                  wr_io.print(first_line) # write a first line
                   filter_dump_stream(cmd_out, wr_io)
                 ensure
                   wr_io.close rescue nil
@@ -140,10 +146,6 @@ EOS
 
         private
 
-        def open_file_stream(file_path, &block)
-          File.open(file_path, "w", encoding: "utf-8") {|f| block.call(f)}
-        end
-
         # This query generates a query which flushes user tables with read lock
         FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
         USER_TABLES_QUERY = <<EOS
@@ -163,20 +165,33 @@ EOS
             # Lock tables
             client.query "FLUSH LOCAL TABLES;"
             q = flush_tables_with_read_lock_query(client)
-            puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
+            $log.debug "FLUSH TABLES query: #{q}"
             client.query q
+            $log.debug "lock acquired"
             begin
-              Fiber.yield  # Lock is done.  Let dump to start
+              Fiber.yield  # Lock is acquired.  Wait until it can be unlocked.
               # obtain binlog pos
+              $log.debug "obtaining the master binlog pos"
               result = client.query "SHOW MASTER STATUS;"
               row = result.first
               if row.nil?
                 raise "MySQL DB has no replication master status.  Check if the DB is set up as a replication master.  In case of RDS, make sure that Backup Retention Period is set to more than 0."
               end
+              $log.debug "master binlog pos obtained: #{row['File']}\t#{row['Position']}"
+            rescue => e
+              $log.error e.to_s
+              raise
             ensure
               # unlock tables
-              client.query "UNLOCK TABLES;"
-              client.close
+              begin
+                client.query "UNLOCK TABLES;"
+                $log.debug "lock released"
+                client.close
+              rescue
+                # table will be unlocked when the MySQL session is closed,
+                # which happens when `client` object becomes out of scope.
+                # So, we can ignore the error here.
+              end
             end
 
             [row["File"], row['Position']]

data/lib/flydata/sync_file_manager.rb

@@ -301,8 +301,29 @@ module Flydata
       BACKUP_DIR
     end
 
+    def stats_path
+      File.join(dump_dir, @data_entry['name']) + ".stats"
+    end
+
+    def save_record_count_stat(table, record_count)
+      stats = load_stats || Hash.new
+      stats[table] = stats[table] ? stats[table].to_i + record_count : record_count
+      save_stats(stats)
+    end
+
+    def load_stats
+      return nil unless File.exists?(stats_path)
+      Hash[*File.read(stats_path).split(/\t/)]
+    end
+
     private
 
+    def save_stats(stats)
+      File.open(stats_path, 'w') do |f|
+        f.write(stats.flatten.join("\t"))
+      end
+    end
+
     def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
       [status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
     end