droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/serf/agent.rb

@@ -0,0 +1,228 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "English"
+
+require "coolio"
+
+require "droonga/loggable"
+require "droonga/deferrable"
+
+module Droonga
+  class Serf
+    class Agent
+      # the port must be different from droonga-http-server's agent!
+      PORT = 7946
+
+      include Loggable
+      include Deferrable
+
+      MAX_N_READ_CHECKS = 10
+
+      def initialize(loop, serf, host, bind_port, rpc_port, *options)
+        @loop = loop
+        @serf = serf
+        @host = host
+        @bind_port = bind_port
+        @rpc_port = rpc_port
+        @options = options
+        @pid = nil
+        @n_ready_checks = 0
+      end
+
+      def start
+        capture_output do |output_write, error_write|
+          env = {}
+          spawn_options = {
+            :out => output_write,
+            :err => error_write,
+          }
+          @pid = spawn(env, @serf, "agent",
+                       "-bind", "#{@host}:#{@bind_port}",
+                       "-rpc-addr", "#{@host}:#{@rpc_port}",
+                       "-log-level", serf_log_level,
+                       *@options, spawn_options)
+        end
+        start_ready_check
+      end
+
+      def stop(&block)
+        if @pid.nil?
+          yield if block_given?
+          return
+        end
+        Process.waitpid(@pid)
+        @output_io.close
+        # logger.trace("stop: output_io watcher detached",
+        #              :watcher => @output_io)
+        @error_io.close
+        # logger.trace("stop: error_io watcher detached",
+        #              :watcher => @error_io)
+        @pid = nil
+        yield if block_given?
+      end
+
+      def running?
+        not @pid.nil?
+      end
+
+      private
+      def serf_log_level
+        level = Logger::Level.default
+        case level
+        when "trace", "debug", "info", "warn"
+          level
+        when "error", "fatal"
+          "err"
+        else
+          level # Or error?
+        end
+      end
+
+      def capture_output
+        result = nil
+        output_read, output_write = IO.pipe
+        error_read, error_write = IO.pipe
+
+        begin
+          result = yield(output_write, error_write)
+        rescue
+          output_read.close  unless output_read.closed?
+          output_write.close unless output_write.closed?
+          error_read.close   unless error_read.closed?
+          error_write.close  unless error_write.closed?
+          raise
+        end
+
+        output_line_buffer = LineBuffer.new
+        @output_io = Coolio::IO.new(output_read)
+        @output_io.on_read do |data|
+          on_standard_output(output_line_buffer, data)
+        end
+        @loop.attach(@output_io)
+        # logger.trace("capture_output: new output_io watcher attached",
+        #              :watcher => @output_io)
+
+        error_line_buffer = LineBuffer.new
+        @error_io = Coolio::IO.new(error_read)
+        @error_io.on_read do |data|
+          on_error_output(error_line_buffer, data)
+        end
+        @loop.attach(@error_io)
+        # logger.trace("capture_output: new error_io watcher attached",
+        #              :watcher => @error_io)
+
+        result
+      end
+
+      def on_standard_output(line_buffer, data)
+        line_buffer.feed(data) do |line|
+          line = line.chomp
+          case line
+          when /\A==> /
+            content = $POSTMATCH
+            logger.info(content)
+          when /\A    /
+            content = $POSTMATCH
+            case content
+            when /\A(\d{4})\/(\d{2})\/(\d{2}) (\d{2}):(\d{2}):(\d{2}) \[(\w+)\] /
+              # year, month, day = $1, $2, $3
+              # hour, minute, second = $4, $5, $6
+              level = $7
+              content = $POSTMATCH
+              return unless needed_log_message?(content)
+              level = normalize_level(level)
+              logger.send(level, content)
+            else
+              logger.info(content)
+            end
+          else
+            logger.info(line)
+          end
+        end
+      end
+
+      def needed_log_message?(content)
+        case content
+        when /\Amemberlist: Failed to receive remote state: EOF\z/
+          # See also: https://github.com/hashicorp/consul/issues/598#issuecomment-71576948
+          false
+        when /\Aagent: Script .*droonga-engine-serf-event-handler.* slow, execution exceeding/
+          # Droonga's serf event handler can be slow for absorbing or some operations.
+          false
+        else
+          true
+        end
+      end
+
+      def normalize_level(level)
+        level = level.downcase
+        case level
+        when "err"
+          "error"
+        else
+          level
+        end
+      end
+
+      def on_error_output(line_buffer, data)
+        line_buffer.feed(data) do |line|
+          line = line.chomp
+          logger.error(line.gsub(/\A==> /, ""))
+        end
+      end
+
+      def start_ready_check
+        @n_ready_checks += 1
+
+        checker = Coolio::TCPSocket.connect(@host, @bind_port)
+
+        checker.on_connect do
+          on_ready
+          checker.close
+          logger.trace("start_ready_check: socket detached",
+                       :watcher => checker)
+        end
+
+        checker.on_connect_failed do
+          if @n_ready_checks >= MAX_N_READ_CHECKS
+            on_failure
+          else
+            timer = Coolio::TimerWatcher.new(1)
+            timer.on_timer do
+              start_ready_check
+              timer.detach
+              logger.trace("start_ready_check: timer detached",
+                           :watcher => timer)
+            end
+            @loop.attach(timer)
+            logger.trace("start_ready_check: timer attached",
+                         :watcher => timer)
+          end
+        end
+
+        @loop.attach(checker)
+        logger.trace("start_ready_check: socket attached",
+                     :watcher => checker)
+      end
+
+      def log_tag
+        tag = "serf-agent"
+        tag << "[#{@pid}]" if @pid
+        tag
+      end
+    end
+  end
+end

data/lib/droonga/serf/command.rb

@@ -0,0 +1,94 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "open3"
+require "pp"
+
+require "droonga/loggable"
+
+module Droonga
+  class Serf
+    class Command
+      class Failure < Error
+        attr_reader :command_line, :exit_status, :output, :error
+        def initialize(command_line, exit_status, output, error)
+          @command_line = command_line
+          @exit_status = exit_status
+          @output = output
+          @error = error
+          message = "Failed to run serf: (#{@exit_status}): "
+          message << "#{@error.strip}[#{@output.strip}]: "
+          message << @command_line.join(" ")
+          super(message)
+        end
+      end
+
+      class ForbiddenCommandInEventHandler < Error
+        def initialize(command)
+          message = "#{command} is forbidden in an event handler script."
+          super(message)
+        end
+      end
+
+      DANGEROUS_COMMANDS_IN_EVENT_HANDLER = [
+        "event",
+        "query",
+      ]
+
+      include Loggable
+
+      attr_accessor :verbose
+
+      def initialize(serf, command, *options)
+        assert_safe_command(command)
+        @serf = serf
+        @command = command
+        @options = options
+        @verbose = false
+      end
+
+      def run
+        command_line = [@serf, @command] + @options
+        p command_line if @verbose
+        stdout, stderror, status = Open3.capture3(*command_line,
+                                                  :pgroup => true)
+        unless status.success?
+          raise Failure.new(command_line, status.to_i, stdout, stderror)
+        end
+        logger.error("run: #{stderror}") unless stderror.empty?
+        if @verbose
+          begin
+            pp JSON.parse(stdout)
+          rescue JSON::ParserError
+            p stdout
+          end
+        end
+        stdout
+      end
+
+      private
+      def assert_safe_command(command)
+        if ENV.key?("SERF_EVENT") and
+             DANGEROUS_COMMANDS_IN_EVENT_HANDLER.include?(command)
+          raise ForbiddenCommandInEventHandler.new(command)
+        end
+      end
+
+      def log_tag
+        "serf[#{@command}]"
+      end
+    end
+  end
+end

data/lib/droonga/serf/downloader.rb

@@ -0,0 +1,120 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "stringio"
+require "tmpdir"
+require "fileutils"
+
+require "faraday"
+require "faraday_middleware"
+require "archive/zip"
+
+require "droonga/loggable"
+
+module Droonga
+  class Serf
+    class Downloader
+      include Loggable
+
+      class DownloadFailed < StandardError
+      end
+
+      MAX_RETRY_COUNT = 5
+      RETRY_INTERVAL  = 10
+
+      TARGET_VERSION  = "0.6.3"
+
+      def initialize(output_path)
+        @output_path = output_path
+        @retry_count = 0
+      end
+
+      def download
+        detect_platform
+        url_base = "https://dl.bintray.com/mitchellh/serf"
+        base_name = "#{TARGET_VERSION}_#{@os}_#{@architecture}.zip"
+        connection = Faraday.new(url_base) do |builder|
+          builder.response(:follow_redirects)
+          builder.adapter(Faraday.default_adapter)
+        end
+        response = connection.get(base_name)
+        absolete_output_path = @output_path.expand_path
+        Dir.mktmpdir do |dir|
+          Archive::Zip.extract(StringIO.new(response.body),
+                               dir,
+                               :directories => false)
+          FileUtils.mv("#{dir}/serf", absolete_output_path.to_s)
+          FileUtils.chmod(0755, absolete_output_path.to_s)
+        end
+      rescue Archive::Zip::UnzipError => archive_error
+        logger.warn("Downloaded zip file is broken.",
+                    :detail => archive_error)
+        if @retry_count < MAX_RETRY_COUNT
+          @retry_count += 1
+          sleep(RETRY_INTERVAL * @retry_count)
+          download
+        else
+          raise DownloadFailed.new("Couldn't download serf executable. Try it later.")
+        end
+      rescue Faraday::ConnectionFailed => network_error
+        logger.warn("Connection failed.",
+                    :detail => network_error)
+        if @retry_count < MAX_RETRY_COUNT
+          @retry_count += 1
+          sleep(RETRY_INTERVAL * @retry_count)
+          download
+        else
+          raise DownloadFailed.new("Couldn't download serf executable. Try it later.")
+        end
+      end
+
+      private
+      def detect_platform
+        detect_os
+        detect_architecture
+      end
+
+      def detect_os
+        case RUBY_PLATFORM
+        when /linux/
+          @os = "linux"
+        when /freebsd/
+          @os = "freebsd"
+        when /darwin/
+          @os = "darwin"
+        when /mswin|mingw/
+          @os = "windows"
+        else
+          raise "Unsupported OS: #{RUBY_PLATFORM}"
+        end
+      end
+
+      def detect_architecture
+        case RUBY_PLATFORM
+        when /x86_64|x64/
+          @architecture = "amd64"
+        when /i\d86/
+          @architecture = "386"
+        else
+          raise "Unsupported architecture: #{RUBY_PLATFORM}"
+        end
+      end
+
+      def log_tag
+        "serf-downloader"
+      end
+    end
+  end
+end

data/lib/droonga/serf/remote_command.rb

@@ -0,0 +1,348 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "json"
+
+require "droonga/path"
+require "droonga/serf"
+require "droonga/node_name"
+require "droonga/catalog/generator"
+require "droonga/catalog/modifier"
+require "droonga/catalog/fetcher"
+require "droonga/safe_file_writer"
+require "droonga/timestamp"
+require "droonga/service_installation"
+
+module Droonga
+  class Serf
+    module RemoteCommand
+      class Base
+        attr_reader :response
+
+        def initialize(serf_name, params)
+          @serf_name = serf_name
+          @params    = params
+          @response  = {
+            "log" => []
+          }
+          @serf = Serf.new(@serf_name)
+
+          @service_installation = ServiceInstallation.new
+          @service_installation.ensure_using_service_base_directory
+
+          log("params = #{params}")
+        end
+
+        def process
+          # override me!
+        end
+
+        def should_process?
+          if @params.nil?
+            log("anonymous query (to be processed)")
+            return true
+          end
+          unless for_this_cluster?
+            log("query for different cluster (mine: #{cluster_id}, to be ignroed)")
+            return false
+          end
+
+          unless @params.include?("node")
+            log("anonymous node query (to be processed)")
+            return true
+          end
+          unless for_me?
+            log("query for different node (me: #{@serf_name}, to be ignored)")
+            return false
+          end
+
+          log("query for this node (to be processed)")
+          true
+        end
+
+        def log(message)
+          @response["log"] << message
+        end
+
+        private
+        def node
+          @node ||= NodeName.parse(@serf_name)
+        end
+
+        def host
+          node.host
+        end
+
+        def cluster_id
+          @serf.cluster_id
+        end
+
+        def target_cluster
+          return nil unless @params
+          @params["cluster_id"]
+        end
+
+        def target_node
+          return nil unless @params
+          @target_node ||= NodeName.parse(@params["node"] || "")
+        rescue ArgumentError
+          nil
+        end
+
+        def for_this_cluster?
+          target_cluster.nil? or target_cluster == cluster_id
+        end
+
+        def for_me?
+          target_node == node
+        end
+
+        def catalog
+          @catalog ||= JSON.parse(Path.catalog.read)
+        end
+      end
+
+      class ChangeRole < Base
+        def process
+          log("old role: #{@serf.role}")
+          @serf.role = @params["role"]
+          log("new role: #{@serf.role}")
+        end
+      end
+
+      class ReportLastMessageTimestamp < Base
+        def process
+          timestamp = Timestamp.last_message_timestamp
+          if timestamp
+            @response["timestamp"] = Timestamp.stringify(timestamp)
+          else
+            @response["timestamp"] = nil
+          end
+        end
+      end
+
+      class AcceptMessagesNewerThan < Base
+        def process
+          log("old timestamp: #{@serf.accept_messages_newer_than_timestamp}")
+          @serf.accept_messages_newer_than(@params["timestamp"])
+          log("new timestamp: #{@serf.accept_messages_newer_than_timestamp}")
+        end
+      end
+
+      class CrossNodeCommandBase < Base
+        private
+        def source_node
+          return nil unless @params
+          @source_node ||= NodeName.parse(@params["source"] || "")
+        rescue ArgumentError
+          nil
+        end
+
+        def dataset
+          @dataset ||= @params["dataset"]
+        end
+
+        def source_host
+          source_node.host
+        end
+
+        def port
+          @port ||= @params["port"] || source_node.port
+        end
+
+        def tag
+          @tag ||= @params["tag"] || source_node.tag
+        end
+      end
+
+      class Join < CrossNodeCommandBase
+        def process
+          log("type = #{type}")
+          case type
+          when "replica"
+            join_as_replica
+          end
+        end
+
+        private
+        def type
+          @params["type"]
+        end
+
+        def joining_node
+          target_node
+        end
+
+        def joining_host
+          joining_node.host
+        end
+
+        def valid_params?
+          not dataset.nil? and
+            not source_node.nil? and
+            not joining_node.nil?
+        end
+
+        def join_as_replica
+          return unless valid_params?
+
+          log("source_node = #{source_node}")
+
+          @catalog = fetch_catalog
+
+          @other_hosts = replica_hosts
+          log("other_hosts = #{@other_hosts}")
+          return if @other_hosts.empty?
+
+          join_to_cluster
+        end
+
+        def replica_hosts
+          generator = Catalog::Generator.new
+          generator.load(catalog)
+          dataset = generator.dataset_for_host(source_host) ||
+                      generator.dataset_for_host(host)
+          return [] unless dataset
+          dataset.replicas.hosts
+        end
+
+        def fetch_catalog
+          fetcher = Catalog::Fetcher.new(:host          => source_host,
+                                         :port          => port,
+                                         :tag           => tag,
+                                         :receiver_host => host)
+          fetcher.fetch(:dataset => dataset)
+        end
+
+        def join_to_cluster
+          log("joining to the cluster")
+          @serf.join(*@other_hosts)
+
+          log("update catalog.json from fetched catalog")
+          Catalog::Modifier.new(catalog).modify do |modifier, file|
+            modifier.datasets[dataset].replicas.hosts += [joining_host]
+            modifier.datasets[dataset].replicas.hosts.uniq!
+            @service_installation.ensure_correct_file_permission(file)
+          end
+          log("done")
+        end
+      end
+
+      class ModifyReplicasBase < Base
+        private
+        def dataset
+          @params["dataset"]
+        end
+
+        def hosts
+          @hosts ||= prepare_hosts
+        end
+
+        def prepare_hosts
+          hosts = @params["hosts"]
+          return nil unless hosts
+          hosts = [hosts] if hosts.is_a?(String)
+          hosts
+        end
+      end
+
+      class SetReplicas < ModifyReplicasBase
+        def process
+          return if dataset.nil? or hosts.nil?
+
+          log("new replicas: #{hosts.join(",")}")
+
+          log("joining to the cluster")
+          @serf.join(*hosts)
+
+          log("setting replicas to the cluster")
+          Catalog::Modifier.new(catalog).modify do |modifier, file|
+            modifier.datasets[dataset].replicas.hosts = hosts
+            @service_installation.ensure_correct_file_permission(file)
+          end
+          log("done")
+        end
+      end
+
+      class AddReplicas < ModifyReplicasBase
+        def process
+          return if dataset.nil? or hosts.nil?
+
+          added_hosts = hosts - [host]
+          log("adding replicas: #{added_hosts.join(",")}")
+          return if added_hosts.empty?
+
+          log("joining to the cluster")
+          @serf.join(*added_hosts)
+
+          log("adding replicas to the cluster")
+          Catalog::Modifier.new(catalog).modify do |modifier, file|
+            modifier.datasets[dataset].replicas.hosts += added_hosts
+            modifier.datasets[dataset].replicas.hosts.uniq!
+            @service_installation.ensure_correct_file_permission(file)
+          end
+          log("done")
+        end
+      end
+
+      class RemoveReplicas < ModifyReplicasBase
+        def process
+          return if dataset.nil? or hosts.nil?
+
+          log("removing replicas: #{hosts.join(",")}")
+
+          log("removing replicas from the cluster")
+          Catalog::Modifier.new(catalog).modify do |modifier, file|
+            modifier.datasets[dataset].replicas.hosts -= hosts
+            @service_installation.ensure_correct_file_permission(file)
+          end
+          log("done")
+        end
+      end
+
+      class Unjoin < ModifyReplicasBase
+        def process
+          return if dataset.nil? or hosts.nil?
+
+          log("unjoining replicas: #{hosts.join(",")}")
+
+          log("unjoining from the cluster")
+          Catalog::Modifier.new(catalog).modify do |modifier, file|
+            if unjoining_node?
+              modifier.datasets[dataset].replicas.hosts = hosts
+            else
+              modifier.datasets[dataset].replicas.hosts -= hosts
+            end
+            @service_installation.ensure_correct_file_permission(file)
+          end
+          log("done")
+        end
+
+        private
+        def unjoining_node?
+          hosts.include?(host)
+        end
+      end
+
+      class UpdateClusterState < Base
+        def process
+          log("updating cluster state")
+          @serf.update_cluster_state
+          log("done")
+        end
+      end
+    end
+  end
+end