droonga-engine 1.0.3 → 1.0.4

data/lib/droonga/safe_file_writer.rb

@@ -0,0 +1,39 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "pathname"
+require "fileutils"
+require "tempfile"
+
+module Droonga
+  class SafeFileWriter
+    class << self
+      def write(path, contents=nil)
+        # Don't output the file directly to prevent loading of incomplete file!
+        path = Pathname(path).expand_path
+        FileUtils.mkdir_p(path.dirname.to_s)
+        Tempfile.open(path.basename.to_s, path.dirname.to_s, "w") do |output|
+          if block_given?
+            yield(output)
+          else
+            output.write(contents)
+          end
+          output.flush
+          File.rename(output.path, path.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/serf.rb

@@ -13,25 +13,67 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "English"
+
+require "json"
+require "coolio"
+
 require "droonga/path"
 require "droonga/loggable"
 require "droonga/catalog_loader"
 require "droonga/serf_downloader"
+require "droonga/line_buffer"
 
 module Droonga
   class Serf
+    ROLE = {
+      :default => {
+        :port => 7946,
+      },
+      :source => {
+        :port => 7947,
+      },
+      :destination => {
+        :port => 7948,
+      },
+    }
+
     class << self
       def path
         Droonga::Path.base + "serf"
       end
+
+      def status_file
+        Droonga::Path.state + "status_file"
+      end
+
+      def load_status
+        if status_file.exist?
+          contents = status_file.read
+          unless contents.empty?
+            return JSON.parse(contents, :symbolize_names => true)
+          end
+        end
+        {}
+      end
+
+      def send_event(name, event, payload)
+        new(nil, name).send_event(event, payload)
+      end
+
+      def send_query(name, query, payload)
+        new(nil, name).send_query(query, payload)
+      end
     end
 
     include Loggable
 
     def initialize(loop, name)
+      # TODO: Don't allow nil for loop. It reduces nil checks and
+      # simplifies source code.
       @loop = loop
       @name = name
-      @pid = nil
+      @agent = nil
     end
 
     def start
@@ -43,29 +85,35 @@ module Droonga
       detect_other_hosts.each do |other_host|
         retry_joins.push("-retry-join", other_host)
       end
-      @pid = run("agent",
-                 "-node", @name,
-                 "-bind", extract_host(@name),
-                 "-event-handler", "droonga-engine-serf-event-handler",
-                 *retry_joins)
+      @agent = run("agent",
+                   "-node", @name,
+                   "-bind", "#{extract_host(@name)}:#{port}",
+                   "-event-handler", "droonga-engine-serf-event-handler",
+                   "-log-level", log_level,
+                   *retry_joins)
       logger.trace("start: done")
     end
 
     def running?
-      not @pid.nil?
+      @agent and @agent.running?
     end
 
     def shutdown
       logger.trace("shutdown: start")
-      Process.waitpid(run("leave"))
-      Process.waitpid(@pid)
-      @pid = nil
+      run("leave").shutdown
+      @agent.shutdown
+      @agent = nil
       logger.trace("shutdown: done")
     end
 
-    def restart
-      shutdown
-      start
+    def send_event(event, payload)
+      ensure_serf
+      run("event", event, JSON.generate(payload)).shutdown
+    end
+
+    def send_query(query, payload)
+      ensure_serf
+      run("query", query, JSON.generate(payload)).shutdown
     end
 
     private
@@ -90,17 +138,51 @@ module Droonga
     end
 
     def run(command, *options)
-      spawn(@serf, command, "-rpc-addr", rpc_address, *options)
+      process = SerfProcess.new(@loop, @serf, command,
+                                "-rpc-addr", rpc_address,
+                                *options)
+      process.start
+      process
     end
 
     def extract_host(node_name)
       node_name.split(":").first
     end
 
+    def log_level
+      level = Logger::Level.default
+      case level
+      when "trace", "debug", "info", "warn"
+        level
+      when "error", "fatal"
+        "err"
+      else
+        level # Or error?
+      end
+    end
+
     def rpc_address
       "#{extract_host(@name)}:7373"
     end
 
+    def status
+      @status ||= self.class.load_status
+    end
+
+    def role
+      if status[:role]
+        role = status[:role].to_sym
+        if self.class::ROLE.key?(role)
+          return role
+        end
+      end
+      :default
+    end
+
+    def port
+      self.class::ROLE[role][:port]
+    end
+
     def detect_other_hosts
       loader = CatalogLoader.new(Path.catalog.to_s)
       catalog = loader.load
@@ -115,5 +197,121 @@ module Droonga
     def log_tag
       "serf"
     end
+
+    class SerfProcess
+      include Loggable
+
+      def initialize(loop, serf, command, *options)
+        @loop = loop
+        @serf = serf
+        @command = command
+        @options = options
+        @pid = nil
+      end
+
+      def start
+        capture_output do |output_write, error_write|
+          env = {}
+          spawn_options = {
+            :out => output_write,
+            :err => error_write,
+          }
+          @pid = spawn(env, @serf, @command, *@options, spawn_options)
+        end
+      end
+
+      def shutdown
+        return if @pid.nil?
+        Process.waitpid(@pid)
+        @output_io.close
+        @error_io.close
+        @pid = nil
+      end
+
+      def running?
+        not @pid.nil?
+      end
+
+      private
+      def capture_output
+        result = nil
+        output_read, output_write = IO.pipe
+        error_read, error_write = IO.pipe
+
+        begin
+          result = yield(output_write, error_write)
+        rescue
+          output_read.close  unless output_read.closed?
+          output_write.close unless output_write.closed?
+          error_read.close   unless error_read.closed?
+          error_write.close  unless error_write.closed?
+          raise
+        end
+
+        output_line_buffer = LineBuffer.new
+        on_read_output = lambda do |data|
+          on_standard_output(output_line_buffer, data)
+        end
+        @output_io = Coolio::IO.new(output_read)
+        @output_io.on_read do |data|
+          on_read_output.call(data)
+        end
+        # TODO: Don't allow nil for loop. It reduces nil checks and
+        # simplifies source code.
+        @loop.attach(@output_io) if @loop
+
+        error_line_buffer = LineBuffer.new
+        on_read_error = lambda do |data|
+          on_error_output(error_line_buffer, data)
+        end
+        @error_io = Coolio::IO.new(error_read)
+        @error_io.on_read do |data|
+          on_read_error.call(data)
+        end
+        # TODO: Don't allow nil for loop. It reduces nil checks and
+        # simplifies source code.
+        @loop.attach(@error_io) if @loop
+
+        result
+      end
+
+      def on_standard_output(line_buffer, data)
+        line_buffer.feed(data) do |line|
+          line = line.chomp
+          case line
+          when /\A==> /
+            content = $POSTMATCH
+            logger.info(content)
+          when /\A    /
+            content = $POSTMATCH
+            case content
+            when /\A(\d{4})\/(\d{2})\/(\d{2}) (\d{2}):(\d{2}):(\d{2}) \[(\w+)\] /
+              year, month, day = $1, $2, $3
+              hour, minute, second = $4, $5, $6
+              level = $7
+              content = $POSTMATCH
+              logger.send(level.downcase, content)
+            else
+              logger.info(content)
+            end
+          else
+            logger.info(line)
+          end
+        end
+      end
+
+      def on_error_output(line_buffer, data)
+        line_buffer.feed(data) do |line|
+          line = line.chomp
+          logger.error(line.gsub(/\A==> /, ""))
+        end
+      end
+
+      def log_tag
+        tag = "serf"
+        tag << "[#{@pid}]" if @pid
+        tag
+      end
+    end
   end
 end

data/lib/droonga/test/stub_handler_messenger.rb

@@ -17,9 +17,12 @@ module Droonga
   module Test
     class StubHandlerMessenger
       attr_reader :values, :messages
+      attr_accessor :engine_state
+
       def initialize
         @values = []
         @messages = []
+        @engine_state = nil
       end
 
       def emit(value)

data/lib/droonga/worker.rb

@@ -22,8 +22,11 @@ module Droonga
     def initialize
       @raw_loop = Coolio::Loop.new
       @loop = EventLoop.new(@raw_loop)
+      @forwarder = Forwarder.new(@loop)
       @handler_runner = HandlerRunner.new(@loop,
-                                          config.merge(:dispatcher => nil))
+                                          config.merge(:dispatcher => nil,
+                                                       :engine_state => nil,
+                                                       :forwarder => @forwarder))
       receive_socket_path = config[:job_receive_socket_path]
       @job_receiver = JobReceiver.new(@loop, receive_socket_path) do |message|
         process(message)
@@ -32,10 +35,12 @@ module Droonga
 
     def run
       Droonga.logger.trace("#{log_tag}: run: start")
+      @forwarder.start
       @handler_runner.start
       @job_receiver.start
       @raw_loop.run
       @handler_runner.shutdown
+      @forwarder.shutdown
       Droonga.logger.trace("#{log_tag}: run: done")
     end
 

data/test/command/config/default/catalog.json

@@ -4,7 +4,7 @@
   "datasets": {
     "Default": {
       "nWorkers": 4,
-      "plugins": ["groonga", "crud", "search", "dump"],
+      "plugins": ["groonga", "crud", "search", "dump", "system"],
       "replicas": [
         {
           "dimension": "_key",

data/test/command/config/version1/catalog.json

@@ -11,7 +11,7 @@
   "datasets": {
     "Default": {
       "workers": 4,
-      "plugins": ["groonga", "crud", "search"],
+      "plugins": ["groonga", "crud", "search", "dump", "system"],
       "number_of_replicas": 2,
       "number_of_partitions": 3,
       "partition_key": "_key",
@@ -48,7 +48,7 @@
     },
     "Watch": {
       "workers": 4,
-      "plugins": ["groonga", "watch", "search", "crud"],
+      "plugins": ["groonga", "watch", "search", "crud", "dump", "system"],
       "number_of_replicas": 1,
       "number_of_partitions": 1,
       "partition_key": "_key",

data/test/command/suite/system/status.expected

@@ -0,0 +1,12 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "system.status.result",
+  "body": {
+    "nodes": {
+      "127.0.0.1:23003/droonga": {
+        "live": true
+      }
+    }
+  }
+}

data/test/command/suite/system/status.test

@@ -0,0 +1,5 @@
+{
+  "type": "system.status",
+  "dataset": "Default",
+  "body": {}
+}

data/test/unit/plugins/system/test_status.rb

@@ -0,0 +1,79 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugins/system"
+
+class SystemStatusHandlerTest < Test::Unit::TestCase
+  def setup
+    setup_handler
+  end
+
+  def teardown
+    teardown_handler
+  end
+
+  private
+  def setup_handler
+    @worker = StubWorker.new
+    @messenger = Droonga::Test::StubHandlerMessenger.new
+    @messenger.engine_state = StubEngineState.new
+    @loop = nil
+    @handler = Droonga::Plugins::System::StatusHandler.new("name",
+                                                           @worker.context,
+                                                           @messenger,
+                                                           @loop)
+  end
+
+  def teardown_handler
+    @handler = nil
+  end
+
+  def process(request)
+    message = Droonga::Test::StubHandlerMessage.new(request)
+    @handler.handle(message)
+  end
+
+  class StubEngineState
+    def all_nodes
+      [
+        "127.0.0.1:10031/droonga",
+        "127.0.0.1:10032/droonga",
+      ]
+    end
+
+    def live_nodes
+      [
+        "127.0.0.1:10031/droonga",
+      ]
+    end
+  end
+
+  public
+  def test_request
+    request = {}
+    response = process(request)
+    status = {
+      "nodes" => {
+        "127.0.0.1:10031/droonga" => {
+          "live" => true,
+        },
+        "127.0.0.1:10032/droonga" => {
+          "live" => false,
+        },
+      },
+    }
+    assert_equal(status, response)
+  end
+end

data/test/unit/test_catalog_generator.rb

@@ -55,7 +55,7 @@ class CatalogGeneratorTest < Test::Unit::TestCase
       @generator.add_dataset("Droonga", {})
       dataset = {
         "nWorkers" => 4,
-        "plugins" => ["groonga", "search", "crud", "dump"],
+        "plugins" => ["groonga", "search", "crud", "dump", "system"],
         "schema" => {},
         "replicas" => [
           {

data/test/unit/test_line_buffer.rb

@@ -0,0 +1,62 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/line_buffer"
+
+class LineBufferTest < Test::Unit::TestCase
+  def setup
+    @line_buffer = Droonga::LineBuffer.new
+  end
+
+  def feed(data)
+    lines = []
+    @line_buffer.feed(data) do |line|
+      lines << line
+    end
+    lines
+  end
+
+  class NoBufferTest < self
+    def test_no_new_line
+      assert_equal([], feed("a"))
+    end
+
+    def test_one_new_line
+      assert_equal(["a\n"], feed("a\n"))
+    end
+
+    def test_multiple_new_lines
+      assert_equal(["a\n", "b\n"], feed("a\nb\n"))
+    end
+  end
+
+  class BufferedTest < self
+    def test_one_line
+      assert_equal([], feed("a"))
+      assert_equal(["a\n"], feed("\n"))
+    end
+
+    def test_multiple_lines
+      assert_equal([], feed("a"))
+      assert_equal(["a\n", "b\n"], feed("\nb\n"))
+    end
+
+    def test_multiple_buffered
+      assert_equal([], feed("a"))
+      assert_equal(["a\n"], feed("\nb"))
+      assert_equal(["b\n"], feed("\n"))
+    end
+  end
+end