droonga-engine 1.0.2 → 1.0.3

data/lib/droonga/command/droonga_engine_service.rb

@@ -0,0 +1,273 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "optparse"
+
+require "coolio"
+
+require "droonga/service_control_protocol"
+require "droonga/engine"
+require "droonga/fluent_message_receiver"
+require "droonga/internal_fluent_message_receiver"
+require "droonga/plugin_loader"
+
+module Droonga
+  module Command
+    class DroongaEngineService
+      class << self
+        def run(command_line_arguments)
+          new.run(command_line_arguments)
+        end
+      end
+
+      include Loggable
+      include ServiceControlProtocol
+
+      def initialize
+        @engine_name = nil
+        @listen_fd = nil
+        @heartbeat_fd = nil
+        @contrtol_read_fd = nil
+        @contrtol_write_fd = nil
+        @contrtol_write_closed = false
+      end
+
+      def run(command_line_arguments)
+        create_new_process_group
+
+        parse_command_line_arguments!(command_line_arguments)
+        PluginLoader.load_all
+
+        control_write_io = IO.new(@control_write_fd)
+        success = true
+        begin
+          run_services
+        rescue
+          logger.exception("failed to run services", $!)
+          success = false
+        ensure
+          unless @control_write_closed
+            control_write_io.write(Messages::FINISH)
+            control_write_io.close
+          end
+        end
+
+        success
+      end
+
+      private
+      def create_new_process_group
+        begin
+          Process.setsid
+        rescue SystemCallError, NotImplementedError
+        end
+      end
+
+      def parse_command_line_arguments!(command_line_arguments)
+        parser = OptionParser.new
+        add_internal_options(parser)
+        parser.parse!(command_line_arguments)
+      end
+
+      def add_internal_options(parser)
+        parser.separator("")
+        parser.separator("Internal:")
+        parser.on("--engine-name=NAME",
+                  "Use NAME as the name of the engine") do |name|
+          @engine_name = name
+        end
+        parser.on("--listen-fd=FD", Integer,
+                  "Use FD as the listen file descriptor") do |fd|
+          @listen_fd = fd
+        end
+        parser.on("--heartbeat-fd=FD", Integer,
+                  "Use FD as the heartbeat file descriptor") do |fd|
+          @heartbeat_fd = fd
+        end
+        parser.on("--control-read-fd=FD", Integer,
+                  "Use FD to read control messages from the service") do |fd|
+          @control_read_fd = fd
+        end
+        parser.on("--control-write-fd=FD", Integer,
+                  "Use FD to write control messages from the service") do |fd|
+          @control_write_fd = fd
+        end
+      end
+
+      def host
+        @engine_name.split(":", 2).first
+      end
+
+      def run_services
+        @stopping = false
+        @engine = nil
+        @receiver = nil
+        @loop = Coolio::Loop.default
+
+        run_internal_message_receiver
+        run_engine
+        run_receiver
+        run_control_io
+        @loop.run
+      end
+
+      def run_internal_message_receiver
+        @internal_message_receiver = create_internal_message_receiver
+        host, port = @internal_message_receiver.start
+        tag = @engine_name.split("/", 2).last.split(".", 2).first
+        @internal_engine_name = "#{host}:#{port}/#{tag}"
+      end
+
+      def create_internal_message_receiver
+        InternalFluentMessageReceiver.new(@loop, host) do |tag, time, record|
+          on_message(tag, time, record)
+        end
+      end
+
+      def shutdown_internal_message_receiver
+        return if @internal_message_receiver.nil?
+        @internal_message_receiver, receiver = nil, @internal_message_receiver
+        receiver.shutdown
+      end
+
+      def run_engine
+        @engine = Engine.new(@loop, @engine_name, @internal_engine_name)
+        @engine.start
+      end
+
+      def run_receiver
+        @receiver = create_receiver
+        @receiver.start
+      end
+
+      def shutdown_receiver
+        return if @receiver.nil?
+        @receiver, receiver = nil, @receiver
+        receiver.shutdown
+      end
+
+      def run_control_io
+        @control_read = Coolio::IO.new(IO.new(@control_read_fd))
+        @control_read_fd = nil
+        on_read = lambda do |data|
+          # TODO: should buffer data to handle half line received case
+          data.each_line do |line|
+            case line
+            when Messages::STOP_GRACEFUL
+              stop_gracefully
+            when Messages::STOP_IMMEDIATELY
+              stop_immediately
+            end
+          end
+        end
+        @control_read.on_read do |data|
+          on_read.call(data)
+        end
+        read_on_close = lambda do
+          if @control_read
+            @control_read = nil
+            stop_immediately
+          end
+        end
+        @control_read.on_close do
+          read_on_close.call
+        end
+        @loop.attach(@control_read)
+
+        @control_write = Coolio::IO.new(IO.new(@control_write_fd))
+        @control_write_fd = nil
+        write_on_close = lambda do
+          if @control_write
+            @control_write = nil
+            stop_immediately
+          end
+          @control_write_closed = true
+        end
+        @control_write.on_close do
+          write_on_close.call
+        end
+        @loop.attach(@control_write)
+
+        @control_write.write(Messages::READY)
+      end
+
+      def shutdown_control_io
+        if @control_write
+          @control_write, control_write = nil, @control_write
+          control_write.detach
+        end
+        if @control_read
+          @control_read, control_read = nil, @control_read
+          control_read.close
+        end
+      end
+
+      def create_receiver
+        options = {
+          :listen_fd => @listen_fd,
+          :heartbeat_fd => @heartbeat_fd,
+        }
+        FluentMessageReceiver.new(@loop, options) do |tag, time, record|
+          on_message(tag, time, record)
+        end
+      end
+
+      def on_message(tag, time, record)
+        prefix, type, *arguments = tag.split(/\./)
+        if type.nil? or type.empty? or type == "message"
+          message = record
+        else
+          message = {
+            "type" => type,
+            "arguments" => arguments,
+            "body" => record
+          }
+        end
+        reply_to = message["replyTo"]
+        if reply_to.is_a? String
+          message["replyTo"] = {
+            "type" => "#{message["type"]}.result",
+            "to" => reply_to
+          }
+        end
+
+        @engine.process(message)
+      end
+
+      def stop_gracefully
+        return if @stopping
+        @stopping = true
+        shutdown_receiver
+        @engine.stop_gracefully do
+          shutdown_control_io
+          shutdown_internal_message_receiver
+        end
+      end
+
+      # It may be called after stop_gracefully.
+      def stop_immediately
+        shutdown_control_io
+        shutdown_receiver if @receiver
+        shutdown_internal_message_receiver
+        @engine.stop_immediately
+        @loop.stop
+      end
+
+      def log_tag
+        "droonga-engine-service"
+      end
+    end
+  end
+end

data/lib/droonga/command/serf_event_handler.rb

@@ -0,0 +1,85 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "optparse"
+require "pathname"
+require "json"
+require "fileutils"
+require "tempfile"
+
+require "droonga/path"
+require "droonga/serf"
+require "droonga/live_nodes_list_observer"
+
+module Droonga
+  module Command
+    class SerfEventHandler
+      class << self
+        def run
+          new.run
+        end
+      end
+
+      def initialize
+        @serf = ENV["SERF"] || Serf.path
+        @serf_rpc_address = ENV["SERF_RPC_ADDRESS"] || "127.0.0.1:7373"
+      end
+
+      def run
+        parse_event
+
+        output_live_nodes
+        true
+      end
+
+      private
+      def parse_event
+        @event_name = ENV["SERF_EVENT"]
+        case @event_name
+        when "user"
+          @event_name += ":#{ENV["SERF_USER_EVENT"]}"
+        when "query"
+          @event_name += ":#{ENV["SERF_USER_QUERY"]}"
+        end
+      end
+
+      def live_nodes
+        nodes = {}
+        members = `#{@serf} members -rpc-addr #{@serf_rpc_address}`
+        members.each_line do |member|
+          name, address, status, = member.strip.split(/\s+/)
+          if status == "alive"
+            nodes[name] = {
+              "serfAddress" => address,
+            }
+          end
+        end
+        nodes
+      end
+
+      def output_live_nodes
+        list_path = LiveNodesListObserver.path
+        nodes = live_nodes
+        file_contents = JSON.pretty_generate(nodes)
+        # Don't output the file directly to prevent loading of incomplete file!
+        Tempfile.open(list_path.basename.to_s, list_path.parent.to_s, "w") do |output|
+          output.write(file_contents)
+          output.flush
+          File.rename(output.path, list_path.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/dispatcher.rb

@@ -13,7 +13,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "English"
 require "tsort"
 
 require "droonga/loggable"
@@ -47,9 +46,12 @@ module Droonga
       end
     end
 
+    attr_accessor :live_nodes
+
     def initialize(engine_state, catalog)
       @engine_state = engine_state
       @catalog = catalog
+      @live_nodes = catalog.all_nodes
       @adapter_runners = create_adapter_runners
       @farm = Farm.new(@engine_state.name, @catalog, @engine_state.loop,
                        :dispatcher => self)
@@ -146,6 +148,8 @@ module Droonga
           session = session_planner.create_session(id, collector_runner)
           @engine_state.register_session(id, session)
         else
+          logger.error("no steps error: id=#{id}, message=#{message}")
+          return
           #todo: take cases receiving result before its query into account
         end
         session.start
@@ -167,9 +171,9 @@ module Droonga
       id = @engine_state.generate_id
       destinations = {}
       steps.each do |step|
-        dataset = step["dataset"]
+        dataset = @catalog.dataset(step["dataset"])
         if dataset
-          routes = @catalog.get_routes(dataset, step)
+          routes = dataset.get_routes(step, @live_nodes)
           step["routes"] = routes
         else
           step["routes"] ||= [id]
@@ -208,11 +212,7 @@ module Droonga
 
     private
     def farm_path(route)
-      if route =~ /\A.*:\d+\/[^\.]+/
-        $MATCH
-      else
-        route
-      end
+      @engine_state.farm_path(route)
     end
 
     def process_input_message(message)

data/lib/droonga/engine.rb

@@ -15,62 +15,126 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "time"
+require "fileutils"
 require "droonga/engine/version"
 require "droonga/loggable"
 require "droonga/engine_state"
-require "droonga/catalog_observer"
+require "droonga/catalog_loader"
 require "droonga/dispatcher"
+require "droonga/live_nodes_list_observer"
 
 module Droonga
   class Engine
     include Loggable
 
-    def initialize(loop, name)
-      @state = EngineState.new(loop, name)
-      @catalog_observer = Droonga::CatalogObserver.new(@state.loop)
-      @catalog_observer.on_reload = lambda do |catalog|
-        graceful_restart(catalog)
-        logger.info("restarted")
+    LAST_PROCESSED_TIMESTAMP = "last-processed.timestamp"
+    EFFECTIVE_MESSAGE_TIMESTAMP = "effective-message.timestamp"
+
+    def initialize(loop, name, internal_name)
+      @state = EngineState.new(loop, name, internal_name)
+      @catalog = load_catalog
+      @live_nodes = @catalog.all_nodes
+      @dispatcher = create_dispatcher
+      @live_nodes_list_observer = LiveNodesListObserver.new
+      @live_nodes_list_observer.on_update = lambda do |live_nodes|
+        @live_nodes = live_nodes
+        @dispatcher.live_nodes = live_nodes if @dispatcher
       end
     end
 
     def start
       logger.trace("start: start")
       @state.start
-      @catalog_observer.start
-      catalog = @catalog_observer.catalog
-      @dispatcher = create_dispatcher(catalog)
+      @live_nodes_list_observer.start
       @dispatcher.start
       logger.trace("start: done")
     end
 
-    def shutdown
-      logger.trace("shutdown: start")
-      @catalog_observer.stop
+    def stop_gracefully
+      logger.trace("stop_gracefully: start")
+      @live_nodes_list_observer.stop
+      on_finish = lambda do
+        output_last_processed_timestamp
+        @dispatcher.shutdown
+        @state.shutdown
+        yield
+      end
+      if @state.have_session?
+        @state.on_finish = on_finish
+      else
+        on_finish.call
+      end
+      logger.trace("stop_gracefully: done")
+    end
+
+    # It may be called after stop_gracefully.
+    def stop_immediately
+      logger.trace("stop_immediately: start")
+      output_last_processed_timestamp
+      @live_nodes_list_observer.stop
       @dispatcher.shutdown
       @state.shutdown
-      logger.trace("shutdown: done")
+      logger.trace("stop_immediately: done")
     end
 
     def process(message)
+      return unless effective_message?(message)
+      @last_processed_timestamp = message["date"]
       @dispatcher.process_message(message)
     end
 
     private
-    def create_dispatcher(catalog)
-      Dispatcher.new(@state, catalog)
+    def load_catalog
+      catalog_path = Path.catalog
+      loader = CatalogLoader.new(catalog_path.to_s)
+      catalog = loader.load
+      logger.info("catalog loaded",
+                  :path  => catalog_path,
+                  :mtime => catalog_path.mtime)
+      catalog
+    end
+
+    def create_dispatcher
+      dispatcher = Dispatcher.new(@state, @catalog)
+      dispatcher.live_nodes = @live_nodes
+      dispatcher
+    end
+
+    def output_last_processed_timestamp
+      File.open(last_processed_timestamp_file, "w") do |file|
+        file.write(@last_processed_timestamp)
+      end
+    end
+
+    def last_processed_timestamp_file
+      @last_processed_timestamp_file ||= File.join(Droonga::Path.state, LAST_PROCESSED_TIMESTAMP)
+    end
+
+    def effective_message?(message)
+      effective_timestamp = effective_message_timestamp
+      return true if effective_timestamp.nil?
+
+      message_timestamp = Time.parse(message["date"])
+      return false if effective_timestamp >= message_timestamp
+
+      FileUtils.rm(effective_message_timestamp_file)
+      true
+    end
+
+    def effective_message_timestamp
+      return nil unless File.exist?(effective_message_timestamp_file)
+
+      timestamp = File.read(effective_message_timestamp_file)
+      begin
+        Time.parse(timestamp)
+      rescue ArgumentError
+        nil
+      end
     end
 
-    def graceful_restart(catalog)
-      logger.trace("graceful_restart: start")
-      old_dispatcher = @dispatcher
-      logger.trace("graceful_restart: creating new dispatcher")
-      new_dispatcher = create_dispatcher(catalog)
-      new_dispatcher.start
-      @dispatcher = new_dispatcher
-      logger.trace("graceful_restart: shutdown old dispatcher")
-      old_dispatcher.shutdown
-      logger.trace("graceful_restart: done")
+    def effective_message_timestamp_file
+      @effective_message_timestamp_file ||= File.join(Droonga::Path.state, EFFECTIVE_MESSAGE_TIMESTAMP)
     end
 
     def log_tag