droonga-engine 1.0.1

data/lib/droonga/handler_message.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class HandlerMessage
+    attr_reader :raw
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def validate
+      unless task.is_a?(Hash)
+        raise "<task> value isn't object: <#{@raw.inspect}>"
+      end
+
+      unless step.is_a?(Hash)
+        raise "<task/step> value isn't object: <#{@raw.inspect}>"
+      end
+    end
+
+    def [](name)
+      @raw[name]
+    end
+
+    def body
+      @body ||= self["body"]
+    end
+
+    def task
+      @task ||= body["task"]
+    end
+
+    def step
+      @step ||= task["step"]
+    end
+
+    def request
+      @request ||= step["body"]
+    end
+
+    def id
+      @id ||= body["id"]
+    end
+
+    def descendants
+      @descendants ||= body["descendants"]
+    end
+  end
+end

data/lib/droonga/handler_messenger.rb

@@ -0,0 +1,119 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/replier"
+require "droonga/forwarder"
+
+module Droonga
+  class HandlerMessenger
+    attr_reader :database_name
+
+    def initialize(forwarder, message, options={})
+      @forwarder = forwarder
+      @message = message
+      @options = options
+      @replier = Replier.new(@forwarder)
+      @dispatcher = @options[:dispatcher]
+      @database_name = options[:database]
+    end
+
+    def emit(value)
+      descendants = @message.descendants
+      raw_message = @message.raw
+      if descendants.empty?
+        return if raw_message["replyTo"].nil?
+        @replier.reply(raw_message.merge("body" => value))
+      else
+        descendants.each do |name, dests|
+          body = {
+            "id"    => @message.id,
+            "input" => name,
+            "value" => value[name],
+          }
+          dests.each do |dest|
+            if @dispatcher
+              @dispatcher.dispatch(body, dest)
+            else
+              message = raw_message.merge("body" => body)
+              forward(message, "to" => dest, "type" => "dispatcher")
+            end
+          end
+        end
+      end
+    end
+
+    def error(status_code, body)
+      descendants = @message.descendants
+      raw_message = @message.raw
+      if descendants.empty?
+        return if raw_message["replyTo"].nil?
+        response = raw_message.merge("statusCode" => status_code,
+                                     "body" => body)
+        @replier.reply(response)
+      else
+        #XXX This is just a temporary solution. We should rewrite this,
+        #    to put errors outside of the message body. Otherwise
+        #    plugins cannot use the name "errors" for their message body.
+        body = {
+          "id"    => @message.id,
+          "input" => "errors",
+          "value" => {
+            database_name => {
+              "statusCode" => status_code,
+              "body" => body,
+            },
+          },
+        }
+        all_dests = []
+        descendants.each do |name, dests|
+          all_dests += dests
+        end
+        all_dests.each do |dest|
+          if @dispatcher
+            @dispatcher.dispatch(body, dest)
+          else
+            message = raw_message.merge("statusCode" => status_code,
+                                        "body" => body,)
+            forward(message, "to" => dest, "type" => "dispatcher")
+          end
+        end
+      end
+    end
+
+    # Forwards a Droonga message to other Droonga Engine.
+    #
+    # @param [Hash] droonga_message
+    #   The Droonga message to be forwarded.
+    # @param [Hash] destination
+    #   The destination of the Droonga message. See {Forwarder#forward} to
+    #   know about how to specify destination.
+    #
+    # @return [void]
+    #
+    # @see Forwarder#forward
+    def forward(droonga_message, destination)
+      @forwarder.forward(droonga_message, destination)
+    end
+
+    def inspect
+      "\#<#{self.class} id=#{object_id}>"
+    end
+
+    private
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] handler_messenger"
+    end
+  end
+end

data/lib/droonga/handler_runner.rb

@@ -0,0 +1,125 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga"
+
+require "droonga/loggable"
+require "droonga/forwarder"
+require "droonga/handler_message"
+require "droonga/handler_messenger"
+require "droonga/step_runner"
+
+module Droonga
+  class HandlerRunner
+    include Loggable
+
+    def initialize(loop, options={})
+      @loop = loop
+      @options = options
+      @name = options[:name]
+      @dataset_name = options[:dataset]
+      @database_name = options[:database]
+      prepare
+    end
+
+    def start
+      logger.trace("start: start")
+      @forwarder.start
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @forwarder.shutdown
+      if @database
+        @database.close
+        @context.close
+        @database = @context = nil
+      end
+      logger.trace("shutdown: done")
+    end
+
+    def prefer_synchronous?(type)
+      find_handler_class(type).action.synchronous?
+    end
+
+    def processable?(type)
+      not find_handler_class(type).nil?
+    end
+
+    def process(message)
+      logger.trace("process: start")
+      type = message["type"]
+      handler_class = find_handler_class(type)
+      if handler_class.nil?
+        logger.trace("process: done: no handler: <#{type}>")
+        return
+      end
+      process_type(handler_class, type, message)
+      logger.trace("process: done: <#{type}>",
+                   :handler => handler_class)
+    end
+
+    private
+    def prepare
+      if @database_name and !@database_name.empty?
+        @context = Groonga::Context.new
+        @database = @context.open_database(@database_name)
+      end
+      logger.debug("#{self.class.name}: activating plugins for the dataset \"#{@dataset_name}\": " +
+                     "#{@options[:plugins].join(", ")}")
+      @step_runner = StepRunner.new(nil, @options[:plugins] || [])
+      @forwarder = Forwarder.new(@loop)
+    end
+
+    def find_handler_class(type)
+      step_definition = @step_runner.find(type)
+      return nil if step_definition.nil?
+      step_definition.handler_class
+    end
+
+    def process_type(handler_class, type, raw_message)
+      handler_message = HandlerMessage.new(raw_message)
+      handler_message.validate
+
+      messenger = HandlerMessenger.new(@forwarder, handler_message, @options)
+      handler = handler_class.new(@name, @context, messenger, @loop)
+      begin
+        result = handler.handle(handler_message)
+        unless result.nil?
+          # XXX: It is just a workaround.
+          # Remove me when super step is introduced.
+          if handler.is_a?(Droonga::Plugins::Search::Handler)
+            messenger.emit(result)
+          else
+            messenger.emit("result" => result)
+          end
+        end
+      rescue ErrorMessage => error
+        messenger.error(error.status_code, error.response_body)
+      rescue => error
+        logger.exception("failed to handle message", error)
+        internal_server_error =
+          ErrorMessages::InternalServerError.new("Unknown internal error")
+        messenger.error(internal_server_error.status_code,
+                        internal_server_error.response_body)
+      end
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] handler"
+    end
+  end
+end

data/lib/droonga/input_message.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class InputMessage
+    def initialize(raw_message)
+      @raw_message = raw_message
+    end
+
+    def adapted_message
+      # TODO: We can create adapted message non-destructively.
+      # If it is not performance issue, it is better that we don't
+      # change message destructively. Consider about it later.
+      @raw_message
+    end
+
+    def body
+      @raw_message["body"]
+    end
+
+    def body=(body)
+      @raw_message["body"] = body
+    end
+
+    def type
+      @raw_message["type"]
+    end
+
+    def type=(type)
+      original_type = @raw_message["type"]
+      return if original_type == type
+      @raw_message["originalTypes"] ||= []
+      @raw_message["originalTypes"] << original_type
+      @raw_message["type"] = type
+    end
+  end
+end

data/lib/droonga/job_protocol.rb

@@ -0,0 +1,20 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module JobProtocol
+    READY_SIGNAL = "R"
+  end
+end

data/lib/droonga/job_pusher.rb

@@ -0,0 +1,179 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+require "droonga/logger"
+require "droonga/job_protocol"
+
+module Droonga
+  class JobPusher
+    include Loggable
+
+    attr_reader :socket_path
+    def initialize(loop, base_path)
+      @loop = loop
+      @socket_path = "#{base_path}.#{Process.pid}.#{object_id}.sock"
+      @job_queue = JobQueue.new(@loop)
+      @server = nil
+    end
+
+    def start
+      FileUtils.rm_f(@socket_path)
+      @server = Coolio::UNIXServer.new(@socket_path) do |connection|
+        @job_queue.add_worker(WorkerConnection.new(connection))
+      end
+      FileUtils.chmod(0600, @socket_path)
+      @loop.attach(@server)
+    end
+
+    def close
+      @server.close if @server
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @server.close if @server
+      @job_queue.close
+      FileUtils.rm_f(@socket_path)
+      logger.trace("shutdown: done")
+    end
+
+    def push(message)
+      logger.trace("push: start")
+      @job_queue.push(message)
+      logger.trace("push: done")
+    end
+
+    private
+    def log_tag
+      "job_pusher"
+    end
+
+    class JobQueue
+      include Loggable
+
+      def initialize(loop)
+        @loop = loop
+        @buffers = []
+        @ready_workers = []
+        @workers = []
+        @many_jobs_report_interval = 100
+        update_many_jobs_threshold
+      end
+
+      def close
+        @workers.each do |worker|
+          worker.close
+        end
+      end
+
+      def add_worker(worker)
+        @workers << worker
+        update_many_jobs_threshold
+        worker.on_ready = lambda do |ready_worker|
+          supply_job(ready_worker)
+        end
+      end
+
+      def push(message)
+        job = message.to_msgpack
+        if @ready_workers.empty?
+          @buffers << job
+          report_statistics_on_push
+        else
+          worker = @ready_workers.shift
+          if @buffers.empty?
+            worker.write(job)
+          else
+            @buffers << job
+            worker.write(@buffers.shift)
+          end
+        end
+      end
+
+      private
+      def supply_job(worker)
+        if @buffers.empty?
+          @ready_workers << worker
+        else
+          worker.write(@buffers.shift)
+          report_statistics_on_pull
+        end
+      end
+
+      def update_many_jobs_threshold
+        @many_jobs_threshold = @workers.size * 100
+      end
+
+      def report_statistics_on_push
+        if @buffers.size >= @many_jobs_threshold
+          if (@buffers.size % @many_jobs_report_interval).zero?
+            logger.warn("push: many jobs in queue: #{@buffers.size}")
+          end
+        end
+      end
+
+      def report_statistics_on_pull
+        if @buffers.size >= @many_jobs_threshold
+          if (@buffers.size % @many_jobs_report_interval).zero?
+            logger.info("pull: many jobs in queue: #{@buffers.size}")
+          end
+        elsif @buffers.size == (@many_jobs_threshold - 1)
+          logger.info("pull: reducing jobs in queue: #{@buffers.size}")
+        end
+      end
+
+      def log_tag
+        "job_queue"
+      end
+    end
+
+    class WorkerConnection
+      attr_writer :on_ready
+
+      def initialize(connection)
+        @connection = connection
+        @ready = false
+        @on_ready = nil
+        setup_connection
+      end
+
+      def ready?
+        @ready
+      end
+
+      def write(job)
+        @connection.write(job)
+        @ready = false
+      end
+
+      def close
+        @connection.close
+      end
+
+      private
+      def setup_connection
+        on_read = lambda do |data|
+          @ready = (data == JobProtocol::READY_SIGNAL)
+          @on_ready.call(self) if @on_ready
+        end
+        @connection.on_read do |data|
+          on_read.call(data)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/job_receiver.rb

@@ -0,0 +1,70 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "msgpack"
+
+require "droonga/loggable"
+require "droonga/job_protocol"
+
+module Droonga
+  class JobReceiver
+    include Loggable
+
+    def initialize(loop, socket_path, &callback)
+      @loop = loop
+      @socket_path = socket_path
+      @callback = callback
+    end
+
+    def start
+      logger.trace("start: start")
+      @receiver = Coolio::UNIXSocket.connect(@socket_path)
+      setup_receive_handler(@receiver)
+      @loop.attach(@receiver)
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      @receiver.close
+      logger.trace("shutdown: done")
+    end
+
+    private
+    def setup_receive_handler(connection)
+      unpacker = MessagePack::Unpacker.new
+      on_read = lambda do |data|
+        logger.trace("on_read: start")
+        unpacker.feed_each(data) do |message|
+          @callback.call(message)
+        end
+        logger.trace("on_read: done")
+        send_ready(connection)
+      end
+      connection.on_read do |data|
+        on_read.call(data)
+      end
+      send_ready(connection)
+    end
+
+    def send_ready(connection)
+      connection.write(JobProtocol::READY_SIGNAL)
+    end
+
+    def log_tag
+      "job_receiver"
+    end
+  end
+end

data/lib/droonga/loggable.rb

@@ -0,0 +1,29 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/logger"
+
+module Droonga
+  module Loggable
+    private
+    def logger
+      @logger ||= Logger.new(:tag => log_tag)
+    end
+
+    def log_tag
+      nil
+    end
+  end
+end