droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/distributed_command_planner.rb

@@ -55,33 +55,31 @@ module Droonga
       end
     end
 
-    def scatter(record, options={})
+    def scatter(options={})
       @processor = {
         "command" => @source_message["type"],
         "dataset" => @dataset.name,
         "body"    => options[:body] || @source_message["body"],
-        "record"  => record,
+        "record"  => options[:record],
         "type"    => "scatter",
         "outputs" => [],
-        "replica" => "all",
-        "post"    => true
+        "replica" => options[:replica] || "all",
+        "slice"   => options[:slice] || "all",
+        "post"    => options[:write] || false,
       }
     end
 
     def broadcast(options={})
-      processor = {
+      @processor = {
         "command" => @source_message["type"],
         "dataset" => @dataset.name,
         "body"    => options[:body] || @source_message["body"],
         "type"    => "broadcast",
         "outputs" => [],
-        "replica" => "random"
+        "replica" => options[:replica] || "random",
+        "slice"   => options[:slice] || "all",
+        "post"    => options[:write] || false,
       }
-      if options[:write]
-        processor["replica"] = "all"
-        processor["post"]    = true
-      end
-      @processor = processor
     end
 
     private

data/lib/droonga/engine.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,100 +15,123 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "time"
 require "fileutils"
+require "time"
+
 require "droonga/engine/version"
 require "droonga/loggable"
+require "droonga/deferrable"
 require "droonga/engine_state"
-require "droonga/catalog_loader"
+require "droonga/cluster"
+require "droonga/catalog/loader"
 require "droonga/dispatcher"
-require "droonga/file_observer"
-require "droonga/live_nodes_list_loader"
-require "droonga/node_status"
+require "droonga/serf"
+require "droonga/serf/tag"
+require "droonga/timestamp"
 
 module Droonga
   class Engine
     include Loggable
+    include Deferrable
 
-    attr_writer :on_ready
-    def initialize(loop, name, internal_name)
-      @state = EngineState.new(loop, name, internal_name)
+    attr_reader :cluster
+
+    def initialize(loop, name, internal_name, options={})
+      @name = name
+      @internal_name = internal_name
+      @loop = loop
       @catalog = load_catalog
-      @state.catalog = @catalog
+      @state = EngineState.new(loop, name,
+                               internal_name,
+                               :catalog  => @catalog,
+                               :internal_connection_lifetime =>
+                                 options[:internal_connection_lifetime])
+      @cluster = Cluster.new(loop,
+                             :catalog  => @catalog,
+                             :internal_connection_lifetime =>
+                               options[:internal_connection_lifetime])
+
       @dispatcher = create_dispatcher
-      @live_nodes_list_observer = FileObserver.new(loop, Path.live_nodes)
-      @live_nodes_list_observer.on_change = lambda do
-        @state.live_nodes = load_live_nodes
-      end
-      @node_status_observer = FileObserver.new(loop, Path.node_status)
-      @node_status_observer.on_change = lambda do
-        logger.trace("reloading node_status: start")
-        node_status.reload
-        logger.trace("reloading node_status: done")
+      @cluster.on_change = lambda do
+        @dispatcher.refresh_node_reference
       end
-      @on_ready = nil
     end
 
     def start
       logger.trace("start: start")
       @state.on_ready = lambda do
-        @on_ready.call if @on_ready
+        on_ready
+        serf = Serf.new(@name.to_s)
+        serf.set_tag(Serf::Tag.internal_node_name, @internal_name)
+      end
+      @state.on_failure = lambda do
+        on_failure
       end
       @state.start
-      @live_nodes_list_observer.start
-      @node_status_observer.start
+      @cluster.start
       @dispatcher.start
+      @last_message_timestamp_observer = run_last_message_timestamp_observer
       logger.trace("start: done")
     end
 
     def stop_gracefully
       logger.trace("stop_gracefully: start")
-      @live_nodes_list_observer.stop
-      @node_status_observer.stop
+      @last_message_timestamp_observer.stop
+      Timestamp.last_message_timestamp = nil # to avoid old timestamp is used
+      @cluster.shutdown
       on_finish = lambda do
-        logger.trace("stop_gracefully/on_finish: start")
-        save_last_processed_message_timestamp
+        logger.trace("stop_gracefully: middle")
         @dispatcher.stop_gracefully do
+          #XXX We must save last processed message timstamp
+          #    based on forwarded/dispatched messages while
+          #    "graceful stop" operations.
+          save_last_message_timestamp
           @state.shutdown
           yield
+          logger.trace("stop_gracefully: done")
         end
-        logger.trace("stop_gracefully/on_finish: done")
       end
       if @state.have_session?
-        logger.trace("stop_gracefully/having sessions")
+        logger.trace("stop_gracefully: having sessions")
         @state.on_finish = on_finish
       else
-        logger.trace("stop_gracefully/no session")
+        logger.trace("stop_gracefully: no session")
         on_finish.call
       end
-      logger.trace("stop_gracefully: done")
     end
 
     # It may be called after stop_gracefully.
     def stop_immediately
       logger.trace("stop_immediately: start")
-      save_last_processed_message_timestamp
-      @live_nodes_list_observer.stop
-      @node_status_observer.stop
+      @last_message_timestamp_observer.stop
+      Timestamp.last_message_timestamp = nil # to avoid old timestamp is used
       @dispatcher.stop_immediately
+      save_last_message_timestamp
+      @cluster.shutdown
       @state.shutdown
       logger.trace("stop_immediately: done")
     end
 
-    def process(message)
-      return unless effective_message?(message)
-      @last_processed_message_timestamp = message["date"]
-      @dispatcher.process_message(message)
+    def refresh_self_reference
+      @cluster.refresh_connection_for(@name)
+      @state.forwarder.refresh_connection_for(@name)
     end
 
-    def node_status
-      @node_status ||= NodeStatus.new
+    def process(message)
+      if message.include?("date")
+        date = Time.parse(message["date"])
+        if @last_message_timestamp.nil? or
+             @last_message_timestamp < date
+          @last_message_timestamp = date
+        end
+      end
+      @dispatcher.process_message(message)
     end
 
     private
     def load_catalog
       catalog_path = Path.catalog
-      loader = CatalogLoader.new(catalog_path.to_s)
+      loader = Catalog::Loader.new(catalog_path.to_s)
       catalog = loader.load
       logger.info("catalog loaded",
                   :path  => catalog_path.to_s,
@@ -116,48 +139,30 @@ module Droonga
       catalog
     end
 
-    def load_live_nodes
-      path = Path.live_nodes
-      loader = LiveNodesListLoader.new(path)
-      live_nodes = loader.load
-      logger.info("live-nodes loaded",
-                  :path  => path.to_s,
-                  :mtime => path.mtime)
-      live_nodes
-    end
-
     def create_dispatcher
-      Dispatcher.new(@state, @catalog)
-    end
-
-    def save_last_processed_message_timestamp
-      logger.trace("output_last_processed_message_timestamp: start")
-      node_status.set(:last_processed_message_timestamp, @last_processed_message_timestamp.to_s)
-      logger.trace("output_last_processed_message_timestamp: done")
+      Dispatcher.new(@state, @cluster, @catalog)
     end
 
-    def effective_message?(message)
-      effective_timestamp = effective_message_timestamp
-      return true if effective_timestamp.nil?
-
-      message_timestamp = Time.parse(message["date"])
-      logger.trace("checking effective_message_timestamp (#{effective_timestamp}) vs message_timestamp(message_timestamp)")
-      return false if effective_timestamp >= message_timestamp
-
-      logger.trace("deleting obsolete effective_message_timestamp: start")
-      node_status.delete(:effective_message_timestamp)
-      logger.trace("deleting obsolete effective_message_timestamp: done")
-      true
+    def save_last_message_timestamp
+      logger.trace("save_last_message_timestamp: start",
+                   :current => @last_message_timestamp)
+      Timestamp.last_message_timestamp = @last_message_timestamp
+      logger.trace("save_last_message_timestamp: done")
     end
 
-    def effective_message_timestamp
-      timestamp = node_status.get(:effective_message_timestamp)
-      return nil unless timestamp
-
-      begin
-        Time.parse(timestamp)
-      rescue ArgumentError
-        nil
+    def run_last_message_timestamp_observer
+      Timestamp.run_last_message_timestamp_observer(@loop) do |timestamp|
+        logger.trace("last message timestamp file is modified",
+                     :loaded  => timestamp,
+                     :current => @last_message_timestamp)
+        if timestamp
+          if @last_message_timestamp.nil? or
+               timestamp > @last_message_timestamp
+            @last_message_timestamp = timestamp
+          elsif timestamp < @last_message_timestamp
+            Timestamp.last_message_timestamp = @last_message_timestamp
+          end
+        end
       end
     end
 

data/lib/droonga/engine/version.rb

@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.0.9"
+    VERSION = "1.1.0"
   end
 end

data/lib/droonga/engine_node.rb

@@ -0,0 +1,301 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "time"
+require "coolio"
+
+require "droonga/loggable"
+require "droonga/forward_buffer"
+require "droonga/fluent_message_sender"
+require "droonga/node_name"
+require "droonga/node_role"
+
+module Droonga
+  class EngineNode
+    include Loggable
+
+    DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS = 60
+
+    attr_reader :name
+
+    def initialize(loop, name, state, options={})
+      @loop = loop
+      @name  = name
+      @state = state
+      logger.trace("initialize: start")
+
+      @buffer = ForwardBuffer.new(name)
+      boundary_timestamp = accept_messages_newer_than_timestamp
+      @buffer.process_messages_newer_than(boundary_timestamp)
+      @buffer.on_forward = lambda do |message, destination|
+        output(message, destination)
+      end
+
+      @node_name = NodeName.parse(@name)
+
+      @sender = nil
+      @auto_close_timer = nil
+      @auto_close_timeout = options[:auto_close_timeout] ||
+                              DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS
+
+      logger.trace("initialize: done")
+    end
+
+    def start
+      logger.trace("start: start")
+      resume
+      logger.trace("start: done")
+    end
+
+    def shutdown
+      logger.trace("shutdown: start")
+      if @sender
+        @sender.shutdown
+        @sender = nil
+      end
+      if @auto_close_timer
+        @auto_close_timer.detach
+        @auto_close_timer = nil
+      end
+      logger.trace("shutdown: done")
+    end
+
+    def refresh_connection
+      logger.trace("refresh_connection: start")
+      shutdown
+      sender # instantiate new sender
+      logger.trace("refresh_connection: done")
+    end
+
+    def forward(message, destination)
+      if read_message?(message)
+        # A node can receive read messages for other nodes,
+        # while changing its role. They must not be buffered.
+        output(message, destination)
+        return
+      end
+
+      unless really_writable?
+        # The target node is not ready. We should send the message later.
+        @buffer.add(message, destination)
+        return
+      end
+
+      # The target node is ready.
+      if @buffer.empty?
+        output(message, destination)
+      else
+        @buffer.add(message, destination)
+        @buffer.start_forward
+      end
+    end
+
+    def bounce(message)
+      destination = {
+        "to"   => name,
+        "type" => message["type"],
+      }
+      output(message, destination)
+    end
+
+    def role
+      if @state
+        @state["role"]
+      else
+        NodeRole::SERVICE_PROVIDER
+      end
+    end
+
+    def live?
+      @state.nil? or @state["live"]
+    end
+
+    def forwardable?
+      return false unless live?
+      role == NodeRole.mine
+    end
+
+    def readable?
+      forwardable? and @buffer.empty? and
+        (complete_service_provider? or not service_provider?)
+    end
+
+    def writable?
+      case NodeRole.mine
+      when NodeRole::SERVICE_PROVIDER
+        true
+      when NodeRole::ABSORB_SOURCE
+        absorb_source?
+      when NodeRole::ABSORB_DESTINATION
+        absorb_destination?
+      else
+        false
+      end
+    end
+
+    def status
+      if readable?
+        "active"
+      elsif forwardable?
+        "inactive"
+      elsif dead?
+        "dead"
+      else
+        "inactive"
+      end
+    end
+
+    def to_json
+      {
+        "name"   => name,
+        "role"   => role,
+        "live"   => live?,
+        "status" => status,
+      }
+    end
+
+    def resume
+      logger.trace("resume: start")
+      sender.resume
+      unless @buffer.empty?
+        if really_writable?
+          logger.info("Target becomes writable. Start to forwarding.")
+          @buffer.start_forward
+        else
+          logger.info("Target is still unwritable.")
+        end
+      end
+      logger.trace("resume: done")
+    end
+
+    private
+    def parse_node_name(name)
+      unless name =~ /\A(.*):(\d+)\/([^.]+)\z/
+        raise "name format: hostname:port/tag"
+      end
+      {
+        :host => $1,
+        :port => $2,
+        :tag  => $3,
+      }
+    end
+
+    def have_unprocessed_messages?
+      @state and @state["have_unprocessed_messages"]
+    end
+
+    def accept_messages_newer_than_timestamp
+      @accept_messages_newer_than_timestamp ||= parse_accept_messages_newer_than_timestamp
+    end
+
+    def parse_accept_messages_newer_than_timestamp
+      return nil if @state.nil? or @state["accept_messages_newer_than"].nil?
+      Time.parse(@state["accept_messages_newer_than"])
+    end
+
+    def dead?
+      not live?
+    end
+
+    def service_provider?
+      role == NodeRole::SERVICE_PROVIDER
+    end
+
+    def absorb_source?
+      role == NodeRole::ABSORB_SOURCE
+    end
+
+    def absorb_destination?
+      role == NodeRole::ABSORB_DESTINATION
+    end
+
+    def complete_service_provider?
+      service_provider? and not have_unprocessed_messages?
+    end
+
+    def really_writable?
+      return false unless writable?
+      case NodeRole.mine
+      when NodeRole::SERVICE_PROVIDER
+        service_provider?
+      when NodeRole::ABSORB_SOURCE
+        not absorb_destination?
+      else
+        true
+      end
+    end
+
+    def read_message?(message)
+      steps = message["body"]["steps"]
+      return false unless steps
+      steps.all? do |step|
+        not step["write"]
+      end
+    end
+
+    def output(message, destination)
+      command = destination["type"]
+      receiver = destination["to"]
+      arguments = destination["arguments"]
+      parsed_receiver = parse_node_name(receiver)
+
+      override_message = {
+        "type" => command,
+      }
+      override_message["arguments"] = arguments if arguments
+      message = message.merge(override_message)
+      output_tag = "#{parsed_receiver[:tag]}.message"
+      log_info = "<#{receiver}>:<#{output_tag}>"
+      logger.trace("forward: start: #{log_info}")
+      sender.send(output_tag, message)
+      set_auto_close_timer
+      logger.trace("forward: end")
+    end
+
+    def sender
+      @sender ||= create_sender
+    end
+
+    def create_sender
+      sender = FluentMessageSender.new(@loop,
+                                       @node_name.host,
+                                       @node_name.port,
+                                       :buffering => true)
+      sender.start
+      sender
+    end
+
+    def set_auto_close_timer
+      previous_timer = @auto_close_timer
+      previous_timer.detach if previous_timer
+
+      @auto_close_timer = Coolio::TimerWatcher.new(@auto_close_timeout)
+      @auto_close_timer.on_timer do
+        @auto_close_timer.detach
+        @auto_close_timer = nil
+        if @sender
+          logger.info("sender for #{name} is automatically closed by timeout.")
+          @sender.shutdown
+          @sender = nil
+        end
+      end
+      @loop.attach(@auto_close_timer)
+    end
+
+    def log_tag
+      "[#{Process.ppid}] engine-node: #{@name}"
+    end
+  end
+end