droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/command/droonga_engine_service.rb

@@ -37,6 +37,7 @@ module Droonga
 
       def initialize
         @engine_name = nil
+        @internal_connection_lifetime = nil
         @listen_fd = nil
         @heartbeat_fd = nil
         @contrtol_read_fd = nil
@@ -85,6 +86,10 @@ module Droonga
                   "Use NAME as the name of the engine") do |name|
           @engine_name = name
         end
+        parser.on("--internal-connection-lifetime=SECONDS", Float,
+                  "The time to expire internal connections, in seconds") do |seconds|
+          @internal_connection_lifetime = seconds
+        end
         parser.on("--listen-fd=FD", Integer,
                   "Use FD as the listen file descriptor") do |fd|
           @listen_fd = fd
@@ -129,18 +134,31 @@ module Droonga
 
       def create_internal_message_receiver
         InternalFluentMessageReceiver.new(@loop, host) do |tag, time, record|
+          logger.trace("InternalFluentMessageReceiver receive")
           on_message(tag, time, record)
         end
       end
 
-      def shutdown_internal_message_receiver
+      def shutdown_internal_message_receiver_gracefully
+        if @internal_message_receiver.nil?
+          yield
+          return
+        end
+        @internal_message_receiver, receiver = nil, @internal_message_receiver
+        receiver.shutdown_gracefully do
+          yield
+        end
+      end
+
+      def shutdown_internal_message_receiver_immediately
         return if @internal_message_receiver.nil?
         @internal_message_receiver, receiver = nil, @internal_message_receiver
-        receiver.shutdown
+        receiver.shutdown_immediately
       end
 
       def run_engine
-        @engine = Engine.new(@loop, @engine_name, @internal_engine_name)
+        @engine = Engine.new(@loop, @engine_name, @internal_engine_name,
+                             :internal_connection_lifetime => @internal_connection_lifetime)
         @engine.on_ready = lambda do
           @worker_process_agent.ready
         end
@@ -164,6 +182,9 @@ module Droonga
         @worker_process_agent.on_stop_immediately = lambda do
           stop_immediately
         end
+        @worker_process_agent.on_refresh_self_reference = lambda do
+          @engine.refresh_self_reference
+        end
         @worker_process_agent.start
       end
 
@@ -177,11 +198,14 @@ module Droonga
           :heartbeat_fd => @heartbeat_fd,
         }
         FluentMessageReceiver.new(@loop, options) do |tag, time, record|
+          logger.trace("FluentMessageReceiver receive")
           on_message(tag, time, record)
         end
       end
 
       def on_message(tag, time, record)
+        logger.trace("on_message: start", :record => record)
+
         prefix, type, *arguments = tag.split(/\./)
         if type.nil? or type.empty? or type == "message"
           message = record
@@ -201,6 +225,8 @@ module Droonga
         end
 
         @engine.process(message)
+
+        logger.trace("on_message: done")
       end
 
       def stop_gracefully
@@ -208,13 +234,23 @@ module Droonga
         logger.trace("stop_gracefully: start")
         @stopping = true
         @receiver.stop_gracefully
-        @engine.stop_gracefully do
-          shutdown_worker_process_agent
-          shutdown_internal_message_receiver
-          @receiver.shutdown_clients
-          logger.trace("stop_gracefully: done",
-                       :n_rest_watchers => @loop.watchers.size,
-                       :rest_watchers   => @loop.watchers)
+        #XXX To disconnect all clients to myself (old service),
+        #    we must refresh the connection via EngineNode
+        #    and Forwarder.
+        #    However, connections from workers can be still
+        #    there. Then we have to wait for their timeout.
+        @engine.refresh_self_reference
+        @receiver.ensure_no_client do
+          logger.trace("stop_gracefully: ready to stop service")
+          @engine.stop_gracefully do
+            logger.trace("stop_gracefully: ready to stop workers")
+            shutdown_worker_process_agent
+            shutdown_internal_message_receiver_gracefully do
+              logger.trace("stop_gracefully: done",
+                           :n_rest_watchers => @loop.watchers.size,
+                           :rest_watchers   => @loop.watchers)
+            end
+          end
         end
       end
 
@@ -222,7 +258,7 @@ module Droonga
       def stop_immediately
         shutdown_worker_process_agent
         @receiver.stop_immediately
-        shutdown_internal_message_receiver
+        shutdown_internal_message_receiver_immediately
         @engine.stop_immediately
         @loop.stop
       end

data/lib/droonga/command/droonga_engine_worker.rb

@@ -40,9 +40,11 @@ module Droonga
         @contrtol_read_fd = nil
         @contrtol_write_fd = nil
         @pid_file_path = nil
+        @label = nil
         @dataset = nil
         @database_path = nil
         @plugins = []
+        @internal_connection_lifetime = nil
         @worker_process_agent = nil
       end
 
@@ -90,6 +92,10 @@ module Droonga
                   "Put PID to PATH") do |path|
           @pid_file_path = Pathname.new(path)
         end
+        parser.on("--label=LABEL",
+                  "Use given value as the label") do |label|
+          @label = label
+        end
         parser.on("--dataset=DATASET",
                   "Process DATASET") do |dataset|
           @dataset = dataset
@@ -102,6 +108,10 @@ module Droonga
                   "Use PLUGINs") do |plugins|
           @plugins = plugins
         end
+        parser.on("--internal-connection-lifetime=SECONDS", Float,
+                  "The time to expire internal connections, in seconds") do |seconds|
+          @internal_connection_lifetime = seconds
+        end
       end
 
       def write_pid_file
@@ -159,8 +169,14 @@ module Droonga
         @loop.stop
       end
 
+      def refresh_node_reference
+        @forwarder.refresh_all_connections
+      end
+
       def start_forwarder
-        @forwarder = Forwarder.new(@loop)
+        @forwarder = Forwarder.new(@loop,
+                                   :auto_close_timeout =>
+                                     @internal_connection_lifetime)
         @forwarder.start
       end
 
@@ -170,6 +186,7 @@ module Droonga
 
       def start_handler_runner
         options = {
+          :label     => @label,
           :forwarder => @forwarder,
           :dataset   => @dataset,
           :database  => @database_path.to_s,
@@ -216,6 +233,9 @@ module Droonga
         @worker_process_agent.on_stop_immediately = lambda do
           stop_immediately
         end
+        @worker_process_agent.on_refresh_node_reference = lambda do
+          refresh_node_reference
+        end
         @worker_process_agent.start
         @worker_process_agent.ready
       end

data/lib/droonga/command/remote_command_base.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "slop"
+
+require "droonga/engine/version"
+require "droonga/node_name"
+require "droonga/serf"
+
+module Droonga
+  module Command
+    class RemoteCommandBase
+      private
+      def parse_options(&block)
+        options = Slop.parse(:help => true) do |option|
+          yield(option) if block_given?
+
+          option.separator("Connections:")
+          option.on(:host=,
+                    "Host name of the target node.",
+                    :required => true)
+          option.on(:port=,
+                    "Port number of the source cluster to be connected.",
+                    :as => Integer,
+                    :default => NodeName::DEFAULT_PORT)
+          option.on(:tag=,
+                    "Tag name of the soruce cluster to be connected.",
+                    :default => NodeName::DEFAULT_TAG)
+
+          option.separator("Miscellaneous:")
+          option.on(:verbose, "Output details for internal operations.",
+                    :default => false)
+        end
+        @options = options
+      rescue Slop::MissingOptionError => error
+        $stderr.puts(error)
+        exit(false)
+      end
+
+      def host
+        @options[:host]
+      end
+
+      def port
+        @options[:port]
+      end
+
+      def tag
+        @options[:tag]
+      end
+
+      def node
+        @node ||= NodeName.new(:host => host,
+                               :port => port,
+                               :tag  => tag)
+      end
+
+      def serf
+        @serf ||= Serf.new(node.to_s,
+                           :verbose => @options[:verbose])
+      end
+    end
+  end
+end

data/lib/droonga/command/serf_event_handler.rb

@@ -16,7 +16,7 @@
 require "json"
 require "fileutils"
 
-require "droonga/command/remote"
+require "droonga/serf/remote_command"
 
 module Droonga
   module Command
@@ -37,17 +37,28 @@ module Droonga
 
         serf_name = ENV["SERF_SELF_NAME"]
         command = command_class.new(serf_name, @payload)
-        command.process if command.should_process?
-        output_response(command.response)
+        begin
+          command.process if command.should_process?
+        rescue Exception => exception
+          command.log("Exception: #{exception.inspect}, #{exception.message}, #{exception.backtrace.join(", ")}")
+          raise exception
+        ensure
+          output_response(command.response)
+        end
         true
       rescue Exception => exception
         #XXX Any exception blocks following serf operations.
         #    To keep it working, I rescue any exception for now.
-        FileUtils.mkdir_p(Path.serf_event_handler_errors)
-        File.open(Path.serf_event_handler_error_file, "w") do |file|
-          file.write(exception.inspect)
-          file.write(exception.backtrace)
+        begin
+          FileUtils.mkdir_p(Path.serf_event_handler_errors)
+          File.open(Path.serf_event_handler_error_file, "w") do |file|
+            file.write(exception.inspect)
+            file.write(exception.backtrace)
+          end
+        rescue Errno::EACCES => permission_denied_exception
         end
+        puts exception.inspect
+        puts exception.backtrace
         true
       end
 
@@ -61,7 +72,7 @@ module Droonga
           @payload = JSON.parse($stdin.gets)
           detect_command_class_from_custom_event(ENV["SERF_QUERY_NAME"])
         when "member-join", "member-leave", "member-update", "member-reap"
-          Remote::UpdateLiveNodes
+          Serf::RemoteCommand::UpdateClusterState
         else
           nil
         end
@@ -70,23 +81,21 @@ module Droonga
       def detect_command_class_from_custom_event(event_name)
         case event_name
         when "change_role"
-          Remote::ChangeRole
-        when "report_status"
-          Remote::ReportStatus
-        when "set_status"
-          Remote::SetStatus
+          Serf::RemoteCommand::ChangeRole
+        when "report_last_message_timestamp"
+          Serf::RemoteCommand::ReportLastMessageTimestamp
+        when "accept_messages_newer_than"
+          Serf::RemoteCommand::AcceptMessagesNewerThan
         when "join"
-          Remote::Join
+          Serf::RemoteCommand::Join
         when "unjoin"
-          Remote::Unjoin
+          Serf::RemoteCommand::Unjoin
         when "set_replicas"
-          Remote::SetReplicas
+          Serf::RemoteCommand::SetReplicas
         when "add_replicas"
-          Remote::AddReplicas
+          Serf::RemoteCommand::AddReplicas
         when "remove_replicas"
-          Remote::RemoveReplicas
-        when "absorb_data"
-          Remote::AbsorbData
+          Serf::RemoteCommand::RemoveReplicas
         else
           nil
         end

data/lib/droonga/data_absorber_client.rb

@@ -0,0 +1,222 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "droonga/loggable"
+require "droonga/node_name"
+require "droonga/catalog/dataset"
+require "droonga/client"
+require "droonga/catalog/generator"
+require "droonga/catalog/fetcher"
+
+module Droonga
+  class DataAbsorberClient
+    include Loggable
+
+    class DestinationEqualsToSource < StandardError
+      def initialize(params)
+        super("The source and the destination are same", params)
+      end
+    end
+
+    class EmptyResponse < StandardError
+    end
+
+    class EmptyBody < StandardError
+    end
+
+    DEFAULT_MESSAGES_PER_SECOND = 100
+    DEFAULT_PROGRESS_INTERVAL_SECONDS = 3
+
+    DEFAULT_HOST    = NodeName::DEFAULT_HOST
+    DEFAULT_PORT    = NodeName::DEFAULT_PORT
+    DEFAULT_TAG     = NodeName::DEFAULT_TAG
+    DEFAULT_DATASET = Catalog::Dataset::DEFAULT_NAME
+
+    attr_reader :params
+    attr_reader :host, :port, :tag, :dataset
+    attr_reader :messages_per_second, :progress_interval_seconds
+    attr_reader :source_host, :source_port, :source_tag, :source_dataset
+    attr_reader :error_message
+
+    def initialize(params)
+      @params = params
+
+      @messages_per_second = @params[:messages_per_second] ||
+                               DEFAULT_MESSAGES_PER_SECOND
+      @progress_interval_seconds = @params[:progress_interval_seconds] ||
+                                     DEFAULT_PROGRESS_INTERVAL_SECONDS
+      @target_role = @params[:target_role]
+
+      @host    = @params[:host]    || DEFAULT_HOST
+      @port    = @params[:port]    || DEFAULT_PORT
+      @tag     = @params[:tag]     || DEFAULT_TAG
+      @dataset = @params[:dataset] || DEFAULT_DATASET
+
+      @source_host    = @params[:source_host]    || @host    || DEFAULT_HOST
+      @source_port    = @params[:source_port]    || @port    || DEFAULT_PORT
+      @source_tag     = @params[:source_tag]     || @tag     || DEFAULT_TAG
+      @source_dataset = @params[:source_dataset] || @dataset || DEFAULT_DATASET
+
+      @receiver_host = @params[:receiver_host] || @host
+      @receiver_port = @params[:receiver_port] || 0
+
+      @client_options = @params[:client_options] || {}
+
+      @error_message = nil
+
+      validate_params
+    end
+
+    def run
+      n_absorbers = 0
+
+      absorb_message = {
+        "type" => "system.absorb-data",
+        "dataset" => @dataset,
+        "body" => {
+          "host"    => @source_host,
+          "port"    => @source_port,
+          "tag"     => @source_tag,
+          "dataset" => @source_dataset,
+          "messagesPerSecond" => @messages_per_second,
+          "progressIntervalSeconds" => @progress_interval_seconds,
+          "targetRole" => @target_role,
+        },
+      }
+      client = Droonga::Client.new(destination_client_options)
+      client.subscribe(absorb_message) do |message|
+        case message
+        when Droonga::Client::Error
+          client.close
+          @error_message = message.to_s
+        else
+          case message["type"]
+          when "system.absorb-data.result", "system.absorb-data.error"
+            if message["statusCode"] != 200
+              client.close
+              error = message["body"]
+              @error_message = "#{error['name']}: #{error['message']}"
+            end
+          when "system.absorb-data.progress"
+            body = message["body"]
+            yield(:n_processed_messages => body["nProcessedMessages"],
+                  :percentage           => body["percentage"],
+                  :message              => body["message"])
+          when "system.absorb-data.start"
+            n_absorbers += 1
+          when "system.absorb-data.end"
+            n_absorbers -= 1
+            client.close if n_absorbers <= 0
+          end
+        end
+      end
+    end
+
+    def source_node_suspendable?
+      (source_replica_hosts - [@source_host]).size >= 1
+    end
+
+    def empty_destination?
+      table_names_in_destination_node.empty?
+    end
+
+    private
+    def validate_params
+      source_node_name = NodeName.new(:host => @source_host,
+                                      :port => @source_port,
+                                      :tag  => @source_tag)
+      destination_node_name = NodeName.new(:host => @host,
+                                           :port => @port,
+                                           :tag  => @tag)
+      if source_node_name == destination_node_name and
+           @source_dataset == @dataset
+        raise DestinationEqualsToSource.new(:host    => @host,
+                                            :port    => @port,
+                                            :tag     => @tag,
+                                            :dataset => @dataset)
+      end
+    end
+
+    def destination_client_options
+      {
+        :host          => @host,
+        :port          => @port,
+        :tag           => @tag,
+        :protocol      => :droonga,
+        :receiver_host => @receiver_host,
+        :receiver_port => @receiver_port,
+      }.merge(@client_options)
+    end
+
+    def synchronous_destination_client_options
+      destination_client_options.merge(:backend => :thread,
+                                       :loop => nil)
+    end
+
+    def table_names_in_destination_node
+      @table_names_in_destination_node ||= get_table_names_in_destination_node
+    end
+
+    def get_table_names_in_destination_node
+      response = nil
+      Droonga::Client.open(synchronous_destination_client_options) do |client|
+        response = client.request("dataset" => @source_dataset,
+                                  "type"    => "table_list")
+      end
+
+      unless response
+        raise EmptyResponse.new("table_list returns nil response")
+      end
+      unless response["body"]
+        raise EmptyBody.new("table_list returns nil result")
+      end
+
+      message_body = response["body"]
+      body = message_body[1]
+      tables = body[1..-1]
+      tables.collect do |table|
+        table[1]
+      end
+    end
+
+    def source_replica_hosts
+      @source_replica_hosts ||= get_source_replica_hosts
+    end
+
+    def get_source_replica_hosts
+      generator = Catalog::Generator.new
+      generator.load(source_catalog)
+      dataset = generator.dataset_for_host(@source_host)
+      return [] unless dataset
+      dataset.replicas.hosts
+    end
+
+    def source_catalog
+      @source_catalog ||= fetch_source_catalog
+    end
+
+    def fetch_source_catalog
+      fetcher = Catalog::Fetcher.new(:host          => @source_host,
+                                     :port          => @source_port,
+                                     :tag           => @source_tag,
+                                     :receiver_host => @receiver_host)
+      fetcher.fetch(:dataset => @source_dataset)
+    end
+
+    def log_tag
+      "data-absorber"
+    end
+  end
+end