droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/catalog/version2.rb

@@ -24,7 +24,7 @@ require "droonga/catalog/version2_validator"
 module Droonga
   module Catalog
     class Version2 < Base
-      def initialize(data, path)
+      def initialize(raw, path)
         super
         validate
         prepare_data
@@ -44,16 +44,17 @@ module Droonga
             volume.slices.each do |slice|
               volume_address = slice.volume.address
               if volume_address.node == node
-                name = volume_address.name
+                local_name = volume_address.local_name
                 path = Path.databases(base_path) +
-                         device + name + "db"
-                migrate_database_location(path, device, name)
+                         device + local_name + "db"
+                migrate_database_location(path, device, local_name)
 
                 options = {
-                  :dataset => dataset_name,
-                  :database => path.to_s,
+                  :label     => volume_address.to_s,
+                  :dataset   => dataset_name,
+                  :database  => path.to_s,
                   :n_workers => n_workers,
-                  :plugins => plugins
+                  :plugins   => plugins
                 }
                 results[volume_address.to_s] = options
               end
@@ -69,13 +70,13 @@ module Droonga
 
       private
       def validate
-        validator = Version2Validator.new(@data, @path)
+        validator = Version2Validator.new(@raw, @path)
         validator.validate
       end
 
       def prepare_data
         @datasets = {}
-        @data["datasets"].each do |name, dataset|
+        @raw["datasets"].each do |name, dataset|
           @datasets[name] = Dataset.new(name, dataset)
         end
       end
@@ -83,7 +84,7 @@ module Droonga
       def collect_all_nodes
         nodes = []
         @datasets.each do |name, dataset|
-          nodes += dataset.all_nodes
+          nodes.concat(dataset.all_nodes)
         end
         nodes.sort.uniq
       end

data/lib/droonga/catalog/version2_validator.rb

@@ -18,8 +18,8 @@ require "droonga/catalog/errors"
 module Droonga
   module Catalog
     class Version2Validator
-      def initialize(data, path)
-        @data = data
+      def initialize(raw, path)
+        @raw = raw
         @path = path
       end
 
@@ -35,11 +35,11 @@ module Droonga
 
       private
       def validate_datasets
-        unless @data.key?("datasets")
+        unless @raw.key?("datasets")
           required_parameter_is_missing("datasets")
           return
         end
-        @data["datasets"].each do |name, dataset|
+        @raw["datasets"].each do |name, dataset|
           validate_dataset(name, dataset)
         end
       end

data/lib/droonga/catalog/volume.rb

@@ -14,17 +14,29 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/catalog/single_volume"
-require "droonga/catalog/collection_volume"
+require "droonga/catalog/slices_volume"
+require "droonga/catalog/replicas_volume"
 
 module Droonga
   module Catalog
     module Volume
+      class UnknownTypeVolume < ArgumentError
+        def initialize(raw)
+          super("volume must have one of 'address', 'slices' or 'replicas': " +
+                  "#{raw.inspect}")
+        end
+      end
+
       class << self
-        def create(dataset, raw_volume)
-          if raw_volume.key?("address")
-            SingleVolume.new(raw_volume)
+        def create(dataset, raw)
+          if raw.key?("address")
+            SingleVolume.new(raw)
+          elsif raw.key?("slices")
+            SlicesVolume.new(dataset, raw)
+          elsif raw.key?("replicas")
+            ReplicasVolume.new(dataset, raw)
           else
-            CollectionVolume.new(dataset, raw_volume)
+            raise UnknownTypeVolume.new(raw)
           end
         end
       end

data/lib/droonga/changable.rb

@@ -0,0 +1,25 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  module Changable
+    attr_writer :on_change
+
+    private
+    def on_change
+      @on_change.call if @on_change
+    end
+  end
+end

data/lib/droonga/cluster.rb

@@ -0,0 +1,237 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/loggable"
+require "droonga/changable"
+require "droonga/path"
+require "droonga/file_observer"
+require "droonga/engine_node"
+require "droonga/differ"
+
+module Droonga
+  class Cluster
+    include Loggable
+    include Changable
+
+    class NoCatalogLoaded < StandardError
+    end
+
+    class NotStartedYet < StandardError
+    end
+
+    class UnknownTarget < StandardError
+    end
+
+    class NoAcceptableReceiver < StandardError
+      def initialize(message)
+        super(message.inspect)
+      end
+    end
+
+    class << self
+      def load_state_file
+        path = Path.cluster_state
+
+        return default_state unless path.exist?
+
+        contents = path.read
+        return default_state if contents.empty?
+
+        begin
+          JSON.parse(contents)
+        rescue JSON::ParserError
+          default_state
+        end
+      end
+
+      def default_state
+        {}
+      end
+    end
+
+    attr_accessor :catalog
+
+    def initialize(loop, params)
+      @loop = loop
+
+      @params = params
+      @catalog = params[:catalog]
+      @state = nil
+
+      reload
+    end
+
+    def start_observe
+      return if @file_observer
+      logger.trace("start_observe: start")
+      @file_observer = FileObserver.new(@loop, Path.cluster_state)
+      @file_observer.on_change = lambda do
+        reload
+      end
+      @file_observer.start
+      logger.trace("start_observe: done")
+    end
+
+    def stop_observe
+      return unless @file_observer
+      logger.trace("stop_observe: start")
+      @file_observer.stop
+      @file_observer = nil
+      logger.trace("stop_observe: done")
+    end
+
+    def start
+      logger.trace("start: start")
+      engine_nodes.each(&:start)
+      start_observe
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      stop_observe
+      engine_nodes.each(&:shutdown)
+      logger.trace("shutdown: done")
+    end
+
+    def refresh_connection_for(name)
+      logger.trace("refresh_connection_for(#{name}): start")
+      engine_nodes.each do |node|
+        if node.name == name
+          node.refresh_connection
+        end
+      end
+      logger.trace("refresh_connection_for(#{name}): done")
+    end
+
+    def reload
+      logger.trace("reload: start")
+      if @state
+        old_state = @state.dup
+      else
+        old_state = nil
+      end
+      @state = self.class.load_state_file
+      if @state == old_state
+        logger.info("cluster state not changed")
+      else
+        logger.info("cluster state changed",
+                    :before => old_state,
+                    :after  => @state,
+                    :diff   => Differ.diff(old_state, @state))
+        clear_cache
+        engine_nodes.each(&:resume)
+        on_change
+      end
+      logger.trace("reload: done")
+    end
+
+    def engine_nodes
+      @engine_nodes ||= create_engine_nodes
+    end
+
+    def engine_nodes_status
+      nodes_status = {}
+      engine_nodes.each do |node|
+        nodes_status[node.name] = {
+          "status" => node.status,
+        }
+      end
+      sorted_nodes_status = {}
+      nodes_status.keys.sort.each do |key|
+        sorted_nodes_status[key] = nodes_status[key]
+      end
+      sorted_nodes_status
+    end
+
+    def forward(message, destination)
+      receiver = destination["to"]
+      receiver_node_name = receiver.match(/\A[^:]+:\d+\/[^.]+/).to_s
+      raise NotStartedYet.new unless @engine_nodes
+      @engine_nodes.each do |node|
+        if node.name == receiver_node_name
+          node.forward(message, destination)
+          return
+        end
+      end
+      raise UnknownTarget.new(receiver)
+    end
+
+    def bounce(message)
+      role = message["targetRole"].downcase
+      logger.info("bounce: trying to bounce message to another " +
+                    "node with the role: #{role}")
+      raise NotStartedYet.new unless @engine_nodes
+
+      acceptable_nodes = engine_nodes.select do |node|
+        node.role == role and
+          node.live?
+      end
+      receiver = acceptable_nodes.sample
+      if receiver
+        receiver.bounce(message)
+      else
+        logger.error("bounce: no available node with the role #{role}",
+                     :message => message)
+        # raise NoAcceptableReceiver.new(message)
+      end
+    end
+
+    def engine_node_names
+      @engine_node_names ||= engine_nodes.collect(&:name)
+    end
+
+    def readable_nodes
+      @readable_nodes ||= engine_nodes.select do |node|
+        node.readable?
+      end.collect(&:name)
+    end
+
+    def writable_nodes
+      @writable_nodes ||= engine_nodes.select do |node|
+        node.writable?
+      end.collect(&:name)
+    end
+
+    private
+    def clear_cache
+      @engine_nodes.each(&:shutdown) if @engine_nodes
+      @engine_nodes   = nil
+      @engine_node_names = nil
+      @readable_nodes = nil
+      @writable_nodes = nil
+    end
+
+    def all_node_names
+      raise NoCatalogLoaded.new unless @catalog
+      @catalog.all_nodes
+    end
+
+    def create_engine_nodes
+      all_node_names.collect do |name|
+        node_state = @state[name] || {}
+        EngineNode.new(@loop,
+                       name,
+                       node_state,
+                       :auto_close_timeout =>
+                         @params[:internal_connection_lifetime])
+      end
+    end
+
+    def log_tag
+      "cluster_state"
+    end
+  end
+end

data/lib/droonga/collector_runner.rb

@@ -44,6 +44,10 @@ module Droonga
       logger.trace("collector: done")
     end
 
+    def collectable?(message)
+      not find_collector_class(message).nil?
+    end
+
     private
     def find_collector_class(message)
       @collector_classes.find do |collector_class|

data/lib/droonga/collectors.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,3 +16,4 @@
 require "droonga/collectors/and"
 require "droonga/collectors/or"
 require "droonga/collectors/sum"
+require "droonga/collectors/recursive_sum"

data/lib/droonga/collectors/recursive_sum.rb

@@ -0,0 +1,26 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  module Collectors
+    class RecursiveSum
+      class << self
+        def operator
+          "recursive-sum"
+        end
+      end
+    end
+  end
+end

data/lib/droonga/command/droonga_engine.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -23,12 +23,16 @@ require "coolio"
 require "sigdump/setup"
 
 require "droonga/engine/version"
+require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/path"
-require "droonga/address"
+require "droonga/node_name"
+require "droonga/forwarder"
 require "droonga/serf"
-require "droonga/node_status"
+require "droonga/cluster"
 require "droonga/file_observer"
 require "droonga/process_supervisor"
+require "droonga/differ"
 
 module Droonga
   module Command
@@ -48,16 +52,13 @@ module Droonga
         parse_command_line_arguments!(command_line_arguments)
 
         setup_path
+        setup_log
 
         if @configuration.daemon?
           Process.daemon
         end
 
-        open_log_file do
-          write_pid_file do
-            run_main_loop
-          end
-        end
+        run_main_loop
       end
 
       private
@@ -80,104 +81,111 @@ module Droonga
         end
       end
 
+      def setup_log
+        ENV["DROONGA_LOG_LEVEL"] = @configuration.log_level
+      end
+
       def run_main_loop
         main_loop = MainLoop.new(@configuration)
         main_loop.run
       end
 
-      def open_log_file
-        if @configuration.log_file
-          File.open(@configuration.log_file, "a") do |file|
-            $stdout.reopen(file)
-            $stderr.reopen(file)
-            yield
-          end
-        else
-          yield
-        end
-      end
-
-      def write_pid_file
-        if @configuration.pid_file_path
-          @configuration.pid_file_path.open("w") do |file|
-            file.puts(Process.pid)
-          end
-          begin
-            yield
-          ensure
-            FileUtils.rm_f(@configuration.pid_file_path.to_s)
-          end
-        else
-          yield
-        end
-      end
-
       class Configuration
-        attr_reader :host, :port, :tag, :log_file, :pid_file_path
         attr_reader :ready_notify_fd
         def initialize
-          config = load_config
+          @config = nil
+
+          @host = nil
+          @port = nil
+          @tag  = nil
 
-          @host = config["host"] || Address::DEFAULT_HOST
-          @port = config["port"] || Address::DEFAULT_PORT
-          @tag  = config["tag"]  || Address::DEFAULT_TAG
+          @internal_connection_lifetime = nil
 
+          @log_level       = nil
           @log_file        = nil
-          @daemon          = false
+          @daemon          = nil
           @pid_file_path   = nil
           @ready_notify_fd = nil
 
-          if have_config_file?
-            self.pid_file_path = config["pid_file"] if config["pid_file"]
-            self.log_file  = config["log_file"] || Path.default_log_file
-            self.log_level = config["log_level"] if config.include?("log_level")
-          end
+          @listen_fd       = nil
+          @heartbeat_fd    = nil
+          @serf_agent_pid  = nil
+        end
+
+        def engine_name
+          "#{host}:#{port}/#{tag}"
         end
 
-        def have_config_file?
-          File.exist?(Path.config)
+        def address_family
+          ip_address = IPAddr.new(IPSocket.getaddress(host))
+          ip_address.family
         end
 
-        def load_config
-          if have_config_file?
-            YAML.load_file(Path.config)
-          else
-            {}
-          end
+        def host
+          @host || config["host"] || default_host
         end
 
-        def engine_name
-          "#{@host}:#{@port}/#{@tag}"
+        def port
+          @port || config["port"] || default_port
         end
 
-        def address_family
-          ip_address = IPAddr.new(IPSocket.getaddress(@host))
-          ip_address.family
+        def tag
+          @tag || config["tag"] || default_tag
         end
 
-        def log_level
-          ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default
+        def internal_connection_lifetime
+          @internal_connection_lifetime ||
+            config["internal_connection_lifetime"] ||
+            default_internal_connection_lifetime
         end
 
-        def log_level=(level)
-          ENV["DROONGA_LOG_LEVEL"] = level
+        def log_level
+          @log_level || config["log_level"] || default_log_level
         end
 
-        def log_file=(file)
-          @log_file = File.expand_path(file)
+        def log_file_path
+          @log_file_path || config["log_file"] || default_log_file_path
         end
 
-        def pid_file_path=(path)
-          @pid_file_path = Pathname.new(path).expand_path
+        def pid_file_path
+          @pid_file_path || config["pid_file"] || default_pid_file_path
         end
 
         def daemon?
-          @daemon
+          daemon = @daemon
+          daemon = config["daemon"] if daemon.nil?
+          daemon = false if daemon.nil?
+          daemon
         end
 
-        def to_command_line
+        def to_engine_command_line
+          command_line_options = [
+            "--host", host,
+            "--port", port.to_s,
+            "--tag", tag,
+            "--internal-connection-lifetime",
+              internal_connection_lifetime.to_s,
+            "--log-level", log_level,
+          ]
+          if log_file_path
+            command_line_options.concat(["--log-file", log_file_path.to_s])
+          end
+          if pid_file_path
+            command_line_options.concat(["--pid-file", pid_file_path.to_s])
+          end
+          if daemon?
+            command_line_options << "--daemon"
+          else
+            command_line_options << "--no-daemon"
+          end
+          command_line_options
+        end
+
+        def to_service_command_line
           command_line_options = [
             "--engine-name", engine_name,
+            "--internal-connection-lifetime",
+              internal_connection_lifetime.to_s,
           ]
           command_line_options
         end
@@ -188,35 +196,104 @@ module Droonga
           add_process_options(parser)
           add_path_options(parser)
           add_notification_options(parser)
+          add_internal_options(parser)
         end
 
         def listen_socket
-          @listen_socket ||= TCPServer.new(@host, @port)
+          @listen_socket ||= create_listen_socket
         end
 
         def heartbeat_socket
-          @heartbeat_socket ||= bind_heartbeat_socket
+          @heartbeat_socket ||= create_heartbeat_socket
+        end
+
+        def serf_agent_pid
+          @serf_agent_pid
         end
 
         private
+        def default_host
+          NodeName::DEFAULT_HOST
+        end
+
+        def default_port
+          NodeName::DEFAULT_PORT
+        end
+
+        def default_tag
+          NodeName::DEFAULT_TAG
+        end
+
+        def default_internal_connection_lifetime
+          Forwarder::DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS
+        end
+
+        def default_log_level
+          ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default
+        end
+
+        def default_log_file_path
+          nil
+        end
+
+        def default_pid_file_path
+          nil
+        end
+
+        def normalize_path(path)
+          if path == "-"
+            nil
+          else
+            Pathname.new(path).expand_path
+          end
+        end
+
+        def config
+          @config ||= load_config
+        end
+
+        def load_config
+          config_path = Path.config
+          return {} unless config_path.exist?
+
+          config = YAML.load_file(config_path)
+          path_keys = ["log_file", "pid_file"]
+          path_keys.each do |path_key|
+            path = config[path_key]
+            next if path.nil?
+
+            path = Pathname.new(path)
+            unless path.absolute?
+              path = (config_path.dirname + path).expand_path
+            end
+            config[path_key] = path
+          end
+          config
+        end
+
         def add_connection_options(parser)
           parser.separator("")
           parser.separator("Connection:")
           parser.on("--host=HOST",
                     "The host name of the Droonga engine",
-                    "(#{@host})") do |host|
+                    "(#{default_host})") do |host|
             @host = host
           end
           parser.on("--port=PORT", Integer,
                     "The port number of the Droonga engine",
-                    "(#{@port})") do |port|
+                    "(#{default_port})") do |port|
             @port = port
           end
           parser.on("--tag=TAG",
                     "The tag of the Droonga engine",
-                    "(#{@tag})") do |tag|
+                    "(#{default_tag})") do |tag|
             @tag = tag
           end
+          parser.on("--internal-connection-lifetime=SECONDS", Float,
+                    "The time to expire internal connections, in seconds",
+                    "(#{default_internal_connection_lifetime})") do |seconds|
+            @internal_connection_lifetime = seconds
+          end
         end
 
         def add_log_options(parser)
@@ -227,12 +304,13 @@ module Droonga
           parser.on("--log-level=LEVEL", levels,
                     "The log level of the Droonga engine",
                     "[#{levels_label}]",
-                    "(#{log_level})") do |level|
-            self.log_level = level
+                    "(#{default_log_level})") do |level|
+            @log_level = level
           end
           parser.on("--log-file=FILE",
-                    "Output logs to FILE") do |file|
-            self.log_file = file
+                    "Output logs to FILE",
+                    "(#{default_log_file_path})") do |path|
+            @log_file_path = normalize_path(path)
           end
         end
 
@@ -249,7 +327,7 @@ module Droonga
           end
           parser.on("--pid-file=PATH",
                     "Put PID to PATH") do |path|
-            self.pid_file_path = path
+            @pid_file_path = normalize_path(path)
           end
         end
 
@@ -260,6 +338,7 @@ module Droonga
                     "Use DIR as the base directory",
                     "(#{Path.base})") do |dir|
             Path.base = File.expand_path(dir)
+            @config = nil
           end
         end
 
@@ -272,34 +351,112 @@ module Droonga
           end
         end
 
-        def bind_heartbeat_socket
-          socket = UDPSocket.new(address_family)
-          socket.bind(@host, @port)
-          socket
+        def add_internal_options(parser)
+          parser.separator("")
+          parser.separator("Internal:")
+          parser.on("--listen-fd=FD", Integer,
+                    "FD of listen socket") do |fd|
+            @listen_fd = fd
+          end
+          parser.on("--heartbeat-fd=FD", Integer,
+                    "FD of heartbeat socket") do |fd|
+            @heartbeat_fd = fd
+          end
+          parser.on("--serf-agent-pid=PID", Integer,
+                    "PID of Serf agent") do |pid|
+            @serf_agent_pid = pid
+          end
+        end
+
+        def create_listen_socket
+          begin
+            TCPServer.new(host, port)
+          rescue Errno::EADDRINUSE
+            raise if @listen_fd.nil?
+            TCPServer.for_fd(@listen_fd)
+          end
+        end
+
+        def create_heartbeat_socket
+          begin
+            socket = UDPSocket.new(address_family)
+            socket.bind(host, port)
+            socket
+          rescue Errno::EADDRINUSE
+            raise if @heartbeat_fd.nil?
+            UDPSocket.for_fd(@heartbeat_fd)
+          end
         end
       end
 
       class MainLoop
+        include Loggable
+
         def initialize(configuration)
           @configuration = configuration
+          ENV["DROONGA_ENGINE_NAME"] = @configuration.engine_name
           @loop = Coolio::Loop.default
+          @log_file = nil
+          @pid_file_path = nil
         end
 
         def run
-          @serf = run_serf
-          @service_runner = run_service
-          setup_initial_on_ready
-          @catalog_observer = run_catalog_observer
-          @command_runner = run_command_runner
+          reopen_log_file
+          write_pid_file do
+            run_internal
+          end
+        end
+
+        private
+        def reopen_log_file
+          return if @configuration.log_file_path.nil?
+          @log_file = @configuration.log_file_path.open("a")
+          $stdout.reopen(@log_file)
+          $stderr.reopen(@log_file)
+        end
+
+        def write_pid_file
+          @pid_file_path = @configuration.pid_file_path
+          if @pid_file_path
+            @pid_file_path.open("w") do |file|
+              file.puts(Process.pid)
+            end
+            begin
+              yield
+            ensure
+              FileUtils.rm_f(@pid_file_path.to_s)
+            end
+          else
+            yield
+          end
+        end
+
+        def run_internal
+          logger.trace("run_internal: start")
+          start_serf
+          @serf_agent.on_ready = lambda do
+            logger.trace("run_internal: serf agent is ready")
+            @serf.initialize_tags
+            @serf.update_cluster_state
+            @service_runner = run_service
+            setup_initial_on_ready
+            @catalog_observer = run_catalog_observer
+            @cluster_state_observer = run_cluster_state_observer
+            @command_runner = run_command_runner
+          end
 
           trap_signals
           @loop.run
-          @serf.stop if @serf.running?
 
-          @service_runner.success?
+          while @service_runner.nil? do
+            sleep 1
+          end
+
+          succeeded = @service_runner.success?
+          logger.trace("run_internal: done")
+          succeeded
         end
 
-        private
         def setup_initial_on_ready
           return if @configuration.ready_notify_fd.nil?
           @service_runner.on_ready = lambda do
@@ -334,36 +491,74 @@ module Droonga
         end
 
         def stop_gracefully
-          @command_runner.stop
-          @serf.stop
-          @catalog_observer.stop
-          @service_runner.stop_gracefully
+          logger.trace("stop_gracefully: start")
+          logger.trace("stop_gracefully: stopping serf agent")
+          stop_serf do
+            logger.trace("stop_gracefully: stopping command runner")
+            @command_runner.stop
+            logger.trace("stop_gracefully: stopping cluster_state_observer")
+            @cluster_state_observer.stop
+            logger.trace("stop_gracefully: stopping catalog_observer")
+            @catalog_observer.stop
+            @service_runner.stop_gracefully
+            logger.trace("stop_gracefully: completely done")
+          end
+          logger.trace("stop_gracefully: done")
         end
 
         def stop_immediately
-          @command_runner.stop
-          @serf.stop
-          @catalog_observer.stop
-          @service_runner.stop_immediately
+          stop_serf do
+            @command_runner.stop
+            @cluster_state_observer.stop
+            @catalog_observer.stop
+            @service_runner.stop_immediately
+          end
         end
 
         def restart_graceful
+          return if @restarting
+          @restarting = true
+          logger.trace("restart_graceful: start")
           old_service_runner = @service_runner
+          reopen_log_file
           @service_runner = run_service
           @service_runner.on_ready = lambda do
+            logger.info("restart_graceful: new service runner is ready")
             @service_runner.on_failure = nil
+            @service_runner.refresh_self_reference
             old_service_runner.stop_gracefully
+            @restarting = false
+            logger.trace("restart_graceful: done")
           end
           @service_runner.on_failure = lambda do
+            logger.info("restart_graceful: failed to setup new service runner")
             @service_runner.on_failure = nil
             @service_runner = old_service_runner
+            @restarting = false
+            logger.trace("restart_graceful: failed")
           end
         end
 
         def restart_immediately
+          return if @restarting
+          @restarting = true
           old_service_runner = @service_runner
+          reopen_log_file
           @service_runner = run_service
           old_service_runner.stop_immediately
+          @restarting = false
+        end
+
+        def restart_self
+          logger.trace("restart_self: start")
+          old_pid_file_path = Pathname.new("#{@pid_file_path}.old")
+          FileUtils.mv(@pid_file_path.to_s, old_pid_file_path.to_s)
+          @pid_file_path = old_pid_file_path
+          stop_gracefully
+
+          engine_runner = EngineRunner.new(@configuration)
+          engine_runner.run
+          logger.trace("restart_self: done")
         end
 
         def run_service
@@ -372,22 +567,68 @@ module Droonga
           service_runner
         end
 
-        def run_serf
-          serf = Serf.new(@loop, @configuration.engine_name)
-          serf.start
-          serf
+        def start_serf
+          @serf = Serf.new(@configuration.engine_name)
+          @serf_agent = @serf.run_agent(@loop)
+        end
+
+        def stop_serf(&block)
+          logger.trace("stop_serf: start")
+          begin
+            @serf.leave
+          rescue Droonga::Serf::Command::Failure
+            logger.error("Failed to leave from Serf cluster: #{$!.message}")
+          end
+          @serf_agent.stop do
+            logger.trace("stop_serf: serf agent stopped")
+            yield
+          end
+          logger.trace("stop_serf: done")
         end
 
         def run_catalog_observer
           catalog_observer = FileObserver.new(@loop, Path.catalog)
           catalog_observer.on_change = lambda do
+            logger.info("restart by updated catalog.json")
             restart_graceful
-            @serf.update_cluster_id if @serf and @serf.running?
+            @serf.update_cluster_id
           end
           catalog_observer.start
           catalog_observer
         end
 
+        RESTART_TRIGGER_KEYS = [
+          "role",
+          "accept_messages_newer_than",
+        ]
+
+        def run_cluster_state_observer
+          previous_state = nil
+          cluster_state_observer = FileObserver.new(@loop, Path.cluster_state)
+          cluster_state_observer.on_change = lambda do
+            my_name   = @configuration.engine_name
+            new_state = Cluster.load_state_file
+            if new_state and previous_state
+              my_new_state = new_state[my_name].select do |key, _value|
+                RESTART_TRIGGER_KEYS.include?(key)
+              end
+              my_previous_state = previous_state[my_name].select do |key, _value|
+                RESTART_TRIGGER_KEYS.include?(key)
+              end
+              if my_new_state != my_previous_state
+                logger.info("restart by changes of myself in cluster-state.json",
+                            :previous => my_previous_state,
+                            :new      => my_new_state,
+                            :diff     => Differ.diff(my_previous_state, my_new_state))
+                restart_graceful
+              end
+            end
+            previous_state = new_state
+          end
+          cluster_state_observer.start
+          cluster_state_observer
+        end
+
         def run_command_runner
           command_runner = CommandRunner.new(@loop)
           command_runner.on_command = lambda do |command|
@@ -396,23 +637,45 @@ module Droonga
           command_runner.start
           command_runner
         end
+
+        def log_tag
+          "droonga-engine"
+        end
       end
 
-      class ServiceRunner
-        def initialize(raw_loop, configuration)
-          @raw_loop = raw_loop
+      class EngineRunner
+        def initialize(configuration)
           @configuration = configuration
-          @success = false
-          @on_ready = nil
-          @on_failure = nil
         end
 
-        def on_ready=(callback)
-          @on_ready = callback
+        def run
+          listen_fd = @configuration.listen_socket.fileno
+          heartbeat_fd = @configuration.heartbeat_socket.fileno
+          env = {}
+          command_line = [
+            RbConfig.ruby,
+            "-S",
+            "droonga-engine",
+            "--listen-fd", listen_fd.to_s,
+            "--heartbeat-fd", heartbeat_fd.to_s,
+            *@configuration.to_engine_command_line,
+          ]
+          options = {
+            listen_fd => listen_fd,
+            heartbeat_fd => heartbeat_fd,
+          }
+          spawn(env, *command_line, options)
         end
+      end
 
-        def on_failure=(callback)
-          @on_failure = callback
+      class ServiceRunner
+        include Loggable
+        include Deferrable
+
+        def initialize(raw_loop, configuration)
+          @raw_loop = raw_loop
+          @configuration = configuration
+          @success = false
         end
 
         def run
@@ -429,7 +692,7 @@ module Droonga
             "--heartbeat-fd", heartbeat_fd.to_s,
             "--control-read-fd", control_write_in.fileno.to_s,
             "--control-write-fd", control_read_out.fileno.to_s,
-            *@configuration.to_command_line,
+            *@configuration.to_service_command_line,
           ]
           options = {
             listen_fd => listen_fd,
@@ -446,17 +709,25 @@ module Droonga
         end
 
         def stop_gracefully
+          logger.trace("stop_gracefully: start")
           @supervisor.stop_gracefully
+          logger.trace("stop_gracefully: done")
         end
 
         def stop_immediately
+          logger.trace("stop_immediately: start")
           @supervisor.stop_immediately
+          logger.trace("stop_immediately: done")
         end
 
         def success?
           @success
         end
 
+        def refresh_self_reference
+          @supervisor.refresh_self_reference
+        end
+
         private
         def create_process_supervisor(input, output)
           supervisor = ProcessSupervisor.new(@raw_loop, input, output)
@@ -469,23 +740,21 @@ module Droonga
           supervisor
         end
 
-        def on_ready
-          @on_ready.call if @on_ready
-        end
-
-        def on_failure
-          @on_failure.call if @on_failure
-        end
-
         def on_finish
           _, status = Process.waitpid2(@pid)
           @success = status.success?
           @supervisor.stop
           on_failure unless success?
         end
+
+        def log_tag
+          "service_runner"
+        end
       end
 
       class CommandRunner
+        include Loggable
+
         attr_writer :on_command
         def initialize(loop)
           @loop = loop
@@ -494,8 +763,9 @@ module Droonga
         end
 
         def start
+          logger.trace("start: stert")
           @async_watcher = Coolio::AsyncWatcher.new
-          on_signal = lambda do
+          @async_watcher.on_signal do
             commands = @commands.uniq
             @commands.clear
             until commands.empty?
@@ -503,16 +773,19 @@ module Droonga
               @on_command.call(command) if @on_command
             end
           end
-          @async_watcher.on_signal do
-            on_signal.call
-          end
           @loop.attach(@async_watcher)
+          logger.trace("start: async watcher attached",
+                       :watcher => @async_watcher)
+          logger.trace("start: done")
         end
 
         def stop
           return if @async_watcher.nil?
+          logger.trace("stop: stert")
           @async_watcher.detach
+          # logger.trace("stop: watcher detached", :watcher => @async_watcher)
           @async_watcher = nil
+          logger.trace("stop: done")
         end
 
         def push_command(command)
@@ -521,6 +794,10 @@ module Droonga
           @commands << command
           @async_watcher.signal if first_command_p
         end
+
+        def log_tag
+          "command_runner"
+        end
       end
     end
   end