flydata 0.3.24 → 0.4.0

data/lib/flydata/command/status.rb

@@ -1,13 +1,23 @@
 require 'flydata/command/base'
 require 'flydata/command/sender'
+require 'flydata/command/helper'
 
 module Flydata
   module Command
     class Status < Base
+      def self.slop
+        Slop.new do
+          on 'skip-helper', 'Do not include Helper status'
+        end
+      end
       def run
         show_purpose_name
         sender = Flydata::Command::Sender.new
         sender.status
+        unless opts.skip_helper?
+          helper = Flydata::Command::Helper.new
+          helper.status
+        end
       end
     end
   end

data/lib/flydata/command/stop.rb

@@ -1,12 +1,22 @@
 require 'flydata/command/base'
 require 'flydata/command/sender'
+require 'flydata/command/helper'
 
 module Flydata
   module Command
     class Stop < Base
+      def self.slop
+        Slop.new do
+          on 'f', 'full', 'Stop all the processes'
+        end
+      end
       def run
         sender = Flydata::Command::Sender.new
         sender.stop
+        if opts.full?
+          helper = Flydata::Command::Helper.new
+          helper.stop
+        end
       end
     end
   end

data/lib/flydata/command/sync.rb

@@ -181,6 +181,7 @@ EOS
           sync_fm.dump_pos_path,
           sync_fm.mysql_table_marshal_dump_path,
           sync_fm.sync_info_file,
+          sync_fm.stats_path,
           sync_fm.table_position_file_paths(*@input_tables),
           sync_fm.table_binlog_pos_paths(*@input_tables),
           sync_fm.table_binlog_pos_init_paths(*@input_tables),
@@ -546,14 +547,17 @@ EOM
                 {table_name: mysql_table_name, log: json}
               end
               ret = forwarder.emit(records)
+              sync_fm.save_record_count_stat(mysql_table_name, ret[:record_count]) if ret
               tmp_num_inserted_record += 1
               ret
             },
             # checkpoint
             Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
               # flush if buffer records exist
+              table_name = mysql_table.nil? ? '' : mysql_table.table_name
               if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
-                forwarder.flush # send buffer data to the server before checkpoint
+                ret = forwarder.flush # send buffer data to the server before checkpoint
+                sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
               end
 
               # show the current progress
@@ -564,7 +568,6 @@ EOM
               end
 
               # save check point
-              table_name = mysql_table.nil? ? '' : mysql_table.table_name
               sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
             }
           )
@@ -576,6 +579,8 @@ EOM
         end
         forwarder.close
         log_info_stdout("  -> Done")
+        #log_info_stdout("  -> Records sent to the server")
+        #log_info_stdout("  -> #{sync_fm.load_stats}")
         sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
 
         if ENV['FLYDATA_BENCHMARK']

data/lib/flydata/helper/action/agent_action.rb

@@ -0,0 +1,24 @@
+require 'flydata/helper/base_action'
+require 'flydata/util/shell'
+
+module Flydata
+  module Helper
+    module Action
+      class AgentAction <  BaseAction
+
+        FLYDATA_CMD = "flydata %{command}"
+        def execute(opts = {})
+          cmd = FLYDATA_CMD % { command: command }
+          log_debug("Executing #{cmd}")
+          o, e, s = Util::Shell.run_cmd(cmd)
+          log_error(e) if not e.to_s.empty?
+          log.error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
+        end
+
+        # Override this. Needs to be implemented by the subclass
+        #def command
+        #end
+      end
+    end
+  end
+end 

data/lib/flydata/helper/action/check_remote_actions.rb

@@ -0,0 +1,54 @@
+require 'flydata/util/file_util'
+require 'flydata/api_client'
+require 'flydata/helper/base_action'
+
+module Flydata
+  module Helper
+    module Action
+      class CheckRemoteActions < BaseAction
+        def initialize(config)
+          super
+          @api_client = ApiClient.instance
+        end
+
+        def execute(opts = {})
+          action_position = ActionPosition.new(config)
+
+          # Get actions from flydata web server
+          last_id = action_position.load
+          actions = @api_client.agent.actions(last_id)
+
+          actions['actions'].each do |action|
+            action_name = action['name']
+            action_id = action['id'].to_i
+            action_info = { id: action_id, config: action['config'] }
+            # Request action
+            yield action_name.to_sym, action_info
+            last_id = action_id if action_id > last_id
+          end
+
+          if last_id > 0
+            action_position.save(last_id)
+            true   # for debug
+          else
+            false  # for debug
+          end
+        end
+      end
+
+      class ActionPosition
+        include Util::FileUtil
+
+        def initialize(config)
+          @path = config.helper_action_position_path
+        end
+        def save(position)
+          write_line(@path, position.to_s)
+        end
+        def load
+          read_line(@path).to_i
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/action/restart_agent.rb

@@ -0,0 +1,13 @@
+require 'flydata/helper/action/agent_action'
+
+module Flydata
+  module Helper
+    module Action
+      class RestartAgent <  AgentAction
+        def command
+          "restart --skip-helper"
+        end
+      end
+    end
+  end
+end      

data/lib/flydata/helper/action/send_logs.rb

@@ -0,0 +1,33 @@
+require 'flydata/helper/base_action'
+require 'flydata/util/file_util'
+require 'flydata/api_client'
+
+module Flydata
+  module Helper
+    module Action
+      class SendLogs <  BaseAction
+        include Util::FileUtil
+
+        DEFAULT_NUM_OF_LINES = 100
+
+        def initialize(config)
+          super
+          @api_client = ApiClient.instance
+        end
+
+        def execute(opts = {})
+          log_debug("Sending logs")
+          num_of_lines = DEFAULT_NUM_OF_LINES
+          action_id = opts[:id]
+          begin
+            num_of_lines = JSON.parse(opts[:config])["num_of_lines"].to_i
+          rescue
+            # Use default number of lines if config is nil, mal-formed etc
+          end
+          tailed_lines = tail(FLYDATA_LOG, num_of_lines)
+          @api_client.agent.send_logs(action_id, tailed_lines)
+        end
+      end
+    end
+  end
+end    

data/lib/flydata/helper/action/stop_agent.rb

@@ -0,0 +1,13 @@
+require 'flydata/helper/action/agent_action'
+
+module Flydata
+  module Helper
+    module Action
+      class StopAgent <  AgentAction
+        def command
+          "stop"
+        end
+      end
+    end
+  end
+end      

data/lib/flydata/helper/action_ownership.rb

@@ -0,0 +1,56 @@
+module Flydata
+  module Helper
+    class ActionOwnership
+      attr_accessor :action_name
+
+      # Resource change flag
+      # true -> action must be taken exclusively
+      attr_accessor :resource_change
+
+      # action_class has a actual logic
+      attr_accessor :action_class
+
+      # Owner should be nil(channel) or worker
+      attr_accessor :owner
+
+      # last processeed time should be updated when worker return
+      # action ownership to channel
+      attr_accessor :last_processed_time
+
+      attr_accessor :retry_count
+
+      def initialize(action_name, resource_change = false, action_class = nil)
+        @action_name = action_name
+        @resource_change = resource_change
+        @action_class = action_class
+        @owner = nil
+        @last_processed_time = -1
+        @retry_count = 0
+      end
+
+      def processing?
+        !@owner.nil?
+      end
+
+      def reset_retry_count
+        @retry_count = 0
+      end
+
+      def increment_retry_count
+        @retry_count += 1
+      end
+
+      def self.action_ownership_map
+        [
+          self.new(:check_remote_actions, false, Action::CheckRemoteActions),
+          self.new(:send_logs, false, Action::SendLogs),
+          self.new(:stop_agent, true, Action::StopAgent),
+          self.new(:restart_agent, true, Action::RestartAgent)
+        ].inject({}) do |h, action|
+          h[action.action_name] = action
+          h
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/action_ownership_channel.rb

@@ -0,0 +1,93 @@
+require 'monitor'
+
+module Flydata
+  module Helper
+    # Take care of action ownership management
+    # - Receive a request from both scheduler and worker.
+    # - Give a action with ownership to workers
+    class ActionOwnershipChannel
+      include MonitorMixin
+
+      def initialize
+        @action_ownership_map = ActionOwnership.action_ownership_map
+        @queue = {}
+        super
+      end
+
+      # Called from schdeduler and worker
+      # action_name must be symbol
+      def request_action(action_name, action_info = {})
+        action_ownership = @action_ownership_map[action_name]
+        if action_ownership.nil?
+          raise "Received invalid action request:#{action_name}"
+        end
+        self.synchronize do
+          if @queue.has_key?(action_name)
+            false
+          else
+            @queue[action_name] =
+                      {
+                        action_ownership: action_ownership,
+                        action_info: action_info
+                      }
+            true
+          end
+        end
+      end
+
+      # Called from worker only
+      def take_action_ownership(new_owner)
+        self.synchronize do
+          return nil if @queue.empty?
+
+          # Wait until action will be processed
+          action = @queue.values.first
+          action_ownership = action[:action_ownership]
+          return nil if action_ownership.processing?
+
+          # Check resource change flag
+          if action_ownership.resource_change &&
+            @action_ownership_map.any? { |name, act_own|
+              (act_own.resource_change && act_own.processing?) }
+            return nil
+          end
+
+          @queue.shift  # delete last acion from queue
+          action_ownership.owner = new_owner
+          action
+        end
+      end
+
+      # Called from worker only
+      def return_action_ownership(action_ownership)
+        self.synchronize do
+          action_ownership.owner = nil
+          action_ownership.last_processed_time = Time.now.to_f
+        end
+      end
+
+      # For debug
+      def queue
+        @queue.dup
+      end
+
+      def map
+        @action_ownership_map.dup
+      end
+
+      def dump
+        $logger.debug "\n" + <<EOT
+========
+action_ownership_channel dump
+
+queue:
+#{@queue}
+
+action_ownership_map:
+#{@action_ownership_map}
+========
+EOT
+      end
+    end
+  end
+end

data/lib/flydata/helper/base_action.rb

@@ -0,0 +1,23 @@
+require 'flydata-core/logger'
+
+module Flydata
+  module Helper
+    class BaseAction
+      include FlydataCore::Logger
+
+      def initialize(config)
+        @config = config
+      end
+
+      attr_accessor :config
+
+      def get_service(service_class)
+        service_class.new(config)
+      end
+
+      # Override
+      #def execute(opts = {}, &block)
+      #end
+    end
+  end
+end

data/lib/flydata/helper/config_parser.rb

@@ -0,0 +1,197 @@
+require 'yaml'
+
+module Flydata
+  module Helper
+    class ConfigError < StandardError
+    end
+
+    # This module parses a config file and return a key symbolized hash.
+    # Additionally config for the helper is replaced with Config instance
+    #
+    # ex) helper.conf
+    # ============================================
+    # # For helper(serverengine)
+    # workers: 2
+    # # For helper
+    # helper:
+    #   scheduled_actions:
+    #     check_remote_actions:
+    #       check_interval: 10s
+    # ============================================
+    class Config < Hash
+      DEFAULT_INTERVAL = 30.0
+      DEFAULT_SCHEDULED_ACTIONS = {
+        check_remote_actions: {
+          check_interval: '30s',
+        },
+      }
+      DEFAULT_HELPER = {
+        scheduled_actions: DEFAULT_SCHEDULED_ACTIONS,
+      }
+      DEFAULT_CONFIG = {
+        helper: DEFAULT_HELPER
+      }
+
+      def self.create(hash)
+        self.new.merge(hash)
+      end
+
+      def self.default
+        create(DEFAULT_HELPER)
+      end
+
+      def fetch_scheduled_actions_conf(action_name, name, default = nil)
+        value = if self[:scheduled_actions] and
+                   self[:scheduled_actions][action_name.to_sym] and
+                   self[:scheduled_actions][action_name.to_sym][name.to_sym]
+                  self[:scheduled_actions][action_name.to_sym][name.to_sym]
+                else
+                  nil
+                end
+        value.nil? ? default : value
+      end
+
+      class << self
+        # snip from https://github.com/fluent/fluentd/blob/master/lib/fluent/config.rb#L119
+        def time_value(str)
+          case str.to_s
+          when /([0-9]+)s/
+            $~[1].to_i
+          when /([0-9]+)m/
+            $~[1].to_i * 60
+          when /([0-9]+)h/
+            $~[1].to_i * 60*60
+          when /([0-9]+)d/
+            $~[1].to_i * 24*60*60
+          else
+            str.to_f
+          end
+        end
+
+        def convert_format(format, value)
+          return nil if value.nil?
+          case format
+          when :time
+            self.time_value(value)
+          when :integer
+            value.to_i
+          when :float
+            value.to_f
+          when :string
+            value.to_s
+          when :bool
+            case value
+            when 'true'
+              true
+            when 'false'
+              false
+            else
+              !!(value)
+            end
+          else
+            raise "Invalid format:#{format}"
+          end
+        end
+
+        def config_param(name, format, option = {})
+          method_name = name.to_s
+          key = option[:key] || name
+          default_value = option[:default]
+
+          case option[:type]
+          when :scheduled_actions
+            define_method(method_name) do |action_name|
+              Config.convert_format(format, fetch_scheduled_actions_conf(
+                action_name, key, default_value))
+            end
+          else
+            define_method(method_name) do
+              def_val = if default_value.respond_to?(:call)
+                          default_value.call(self)
+                        else
+                          default_value
+                        end
+              Config.convert_format(format, self[key] || def_val)
+            end
+          end
+        end
+      end
+
+      config_param :helper_retry_alert_limit, :integer, default: 3
+      config_param :helper_retry_interval, :float, default: 10
+      config_param :helper_retry_limit, :integer, default: 15
+
+      # helper directories
+      config_param :helper_home, :string,
+        default: FLYDATA_HELPER_HOME
+      config_param :helper_pid_dir, :string,
+        default: lambda{|c| File.join(c.helper_home, 'pids') }
+      config_param :helper_position_dir, :string,
+        default: lambda{|c| File.join(c.helper_home, 'positions') }
+
+
+      # helper files
+      config_param :helper_action_position_path, :string,
+        default: lambda{|c| File.join(c.helper_position_dir, 'helper_action.pos') }
+      config_param :helper_log_path, :string,
+        default: FLYDATA_LOG
+      config_param :helper_pid_path, :string,
+        default: lambda{|c| File.join(c.helper_pid_dir, 'helper.pid') }
+
+      # Return deep copy
+      def scheduled_actions
+        actions = self[:scheduled_actions].dup
+        self[:scheduled_actions].each do |k, v|
+          actions[k] = v.dup
+          actions[k][:name] = k
+          actions[k][:check_interval] = Config.convert_format(:time, v)
+        end
+        actions
+      end
+    end
+
+    class ConfigParser
+      def self.parse_file(config_path = nil)
+        config_path.nil? ? { helper: Config.default } : parse(File.open(config_path).read)
+      end
+
+      def self.parse(config_content)
+        ConfigParser.new.parse(config_content)
+      end
+
+      def parse(config_content)
+        conf = YAML::load(config_content)
+        raise ConfigError.new("Invalid conf format.") unless conf and conf.kind_of?(Hash)
+        conf = symbolize_keys(conf)
+        conf[:helper] = Config.create(parse_helper(conf[:helper]))
+        conf
+      end
+
+      def parse_helper(conf)
+        return Config::DEFAULT_HELPER if conf.nil? or conf.empty?
+        conf[:scheduled_actions] = Config::DEFAULT_SCHEDULED_ACTIONS.
+          dup.merge(conf[:scheduled_actions] || {})
+        conf
+      end
+
+      def symbolize_keys(value)
+        case value
+        when Hash
+          value.inject({}){|result, (key, value)|
+            new_key = case key
+                      when String then key.to_sym
+                      else key
+                      end
+            new_value = symbolize_keys(value)
+            result[new_key] = new_value
+            result
+          }
+        when Array
+          value.collect {|e| symbolize_keys(e)}
+        else
+          value
+        end
+      end
+    end
+  end
+end