fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/collector_plugin.rb

@@ -22,9 +22,8 @@ module Droonga
     extend PluginRegisterable
 
     attr_reader :task, :input_name, :component, :output_values, :body, :output_names
-    def initialize(dispatcher)
+    def initialize
       super()
-      @dispatcher = dispatcher
     end
 
     def process(command, message)
@@ -39,44 +38,13 @@ module Droonga
       @id = message["id"]
       @value = message["value"]
       @input_name = message["name"]
-      @descendants = message["descendants"]
       super(command, @value)
-      output if @descendants
       true
     end
 
     # TODO: consider better name
-    def emit(value, name=nil)
-      unless name
-        if @output_names
-          name = @output_names.first
-        else
-          @output_values = @task["values"] = value
-          return
-        end
-      end
+    def emit(name, value)
       @output_values[name] = value
     end
-
-    def post(message, destination=nil)
-      @dispatcher.post(message, destination)
-    end
-
-    def output
-      result = @task["values"]
-      post(result, @component["post"]) if @component["post"]
-      @descendants.each do |name, dests|
-        message = {
-          "id" => @id,
-          "input" => name,
-          "value" => result[name]
-        }
-        dests.each do |routes|
-          routes.each do |route|
-            post(message, "to"=>route, "type"=>"dispatcher")
-          end
-        end
-      end
-    end
   end
 end

data/lib/droonga/dispatcher.rb

@@ -15,149 +15,185 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require 'tsort'
-require "droonga/adapter"
+require "English"
+require "tsort"
+
+require "droonga/input_adapter"
+require "droonga/output_adapter"
 require "droonga/distributor"
 require "droonga/catalog"
 require "droonga/collector"
 require "droonga/farm"
-require "droonga/input_message"
+require "droonga/session"
+require "droonga/replier"
+require "droonga/message_processing_error"
 
 module Droonga
   class Dispatcher
-    attr_reader :name, :envelope, :collectors
+    attr_reader :name
+
+    class MissingDatasetParameter < BadRequest
+      def initialize
+        super("\"dataset\" must be specified.")
+      end
+    end
+
+    class UnknownCommand < BadRequest
+      def initialize(command, dataset)
+        super("The command #{command.inspect} is not available " +
+                "for the dataset #{dataset.inspect}.")
+      end
+    end
 
     def initialize(options)
       @options = options
       @name = @options[:name]
-      @farm = Farm.new(name)
-      @collectors = {}
+      @sessions = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
-      @adapter = Adapter.new(self,
-                             :adapters => Droonga.catalog.option("plugins"))
-      @forwarder = Forwarder.new
+      @input_adapter =
+        InputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
+      @output_adapter =
+        OutputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
+      @loop = EventLoop.new
+      @farm = Farm.new(name, @loop, :dispatcher => self)
+      @forwarder = Forwarder.new(@loop)
+      @replier = Replier.new(@forwarder)
       @distributor = Distributor.new(self, @options)
+      @collector = Collector.new
     end
 
     def start
+      @forwarder.start
       @farm.start
+      @loop_thread = Thread.new do
+        @loop.run
+      end
     end
 
     def shutdown
       @forwarder.shutdown
       @distributor.shutdown
-      @adapter.shutdown
+      @collector.shutdown
+      @input_adapter.shutdown
+      @output_adapter.shutdown
       @farm.shutdown
+      @loop.stop
+      @loop_thread.join
     end
 
-    def add_route(route)
-      envelope["via"].push(route)
-    end
-
-    def handle_envelope(envelope)
-      @envelope = envelope
-      if envelope["type"] == "dispatcher"
-        handle(envelope["body"], envelope["arguments"])
+    def process_message(message)
+      @message = message
+      if message["type"] == "dispatcher"
+        process_internal_message(message["body"])
       else
-        process_input_message(envelope)
-      end
-    end
-
-    def post(body, destination=nil)
-      $log.trace("#{log_tag}: post: start")
-      route = nil
-      unless is_route?(destination)
-        route = envelope["via"].pop
-        destination = route
-      end
-      unless is_route?(destination)
-        destination = envelope["replyTo"]
-      end
-      command = nil
-      receiver = nil
-      arguments = nil
-      synchronous = nil
-      case destination
-      when String
-        command = destination
-      when Hash
-        command = destination["type"]
-        receiver = destination["to"]
-        arguments = destination["arguments"]
-        synchronous = destination["synchronous"]
-      end
-      if receiver
-        @forwarder.forward(envelope, body,
-                           "type" => command,
-                           "to" => receiver,
-                           "arguments" => arguments)
-      else
-        if command == "dispatcher"
-          handle(body, arguments)
-        elsif @adapter.processable?(command)
-          @adapter.process(command, body, *arguments)
-        else
-          @distributor.distribute(envelope.merge("type" => command,
-                                                 "body" => body))
+        begin
+          assert_valid_message
+          process_input_message(message)
+        rescue MessageProcessingError => error
+          reply("statusCode" => error.status_code,
+                "body"       => error.response_body)
         end
       end
-      add_route(route) if route
-      $log.trace("#{log_tag}: post: done")
     end
 
-    def handle(message, arguments)
-      case message
-      when Array
-        handle_incoming_message(message)
-      when Hash
-        handle_internal_message(message)
-      end
+    def forward(message, destination)
+      $log.trace("#{log_tag}: forward start")
+      @forwarder.forward(message, destination)
+      $log.trace("#{log_tag}: forward done")
     end
 
-    def handle_incoming_message(message)
-      id = generate_id
-      planner = Planner.new(self, message)
-      destinations = planner.resolve(id)
-      components = planner.components
-      message = { "id" => id, "components" => components }
-      destinations.each do |destination, frequency|
-        dispatch(message, destination)
-      end
+    # Replies response to replyTo.
+    #
+    # @param [Hash] message
+    #   The message to be replied. See {Replier#reply} for available keys.
+    #
+    #   The key-value pairs in request message are used as the default
+    #   key-value pairs. For example, if the passed message doesn't
+    #   include `id` key, `id` key's value is used in request message.
+    #
+    # @return [void]
+    #
+    # @see Replier#reply
+    def reply(message)
+      adapted_message = @output_adapter.adapt(@message.merge(message))
+      return if adapted_message["replyTo"].nil?
+      @replier.reply(adapted_message)
     end
 
-    def handle_internal_message(message)
+    def process_internal_message(message)
       id = message["id"]
-      collector = @collectors[id]
-      unless collector
+      session = @sessions[id]
+      if session
+        session.receive(message["input"], message["value"])
+      else
         components = message["components"]
         if components
           planner = Planner.new(self, components)
-          collector = planner.get_collector(id)
+          session = planner.create_session(id, @collector)
+          @sessions[id] = session
         else
           #todo: take cases receiving result before its query into account
         end
+        session.start
       end
-      collector.handle(message["input"], message["value"])
+      @sessions.delete(id) if session.done?
     end
 
     def dispatch(message, destination)
       if local?(destination)
-        handle_internal_message(message)
+        process_internal_message(message)
       else
-        post(message, "to"=>farm_path(destination), "type"=>"dispatcher")
+        @forwarder.forward(@message.merge("body" => message),
+                           "type" => "dispatcher",
+                           "to"   => destination)
       end
     end
 
-    def deliver(id, route, message, type, synchronous)
-      if id == route
-        post(message, "type" => type, "synchronous"=> synchronous)
-      else
-        envelope = @envelope.merge("body" => message, "type" => type)
-        @farm.process(route, envelope, synchronous)
+    def dispatch_components(components)
+      id = generate_id
+      destinations = {}
+      components.each do |component|
+        dataset = component["dataset"]
+        if dataset
+          routes = Droonga.catalog.get_routes(dataset, component)
+          component["routes"] = routes
+        else
+          component["routes"] ||= [id]
+        end
+        routes = component["routes"]
+        routes.each do |route|
+          destinations[farm_path(route)] = true
+        end
+      end
+      dispatch_message = { "id" => id, "components" => components }
+      destinations.each_key do |destination|
+        dispatch(dispatch_message, destination)
       end
     end
 
+    def process_local_message(local_message)
+      task = local_message["task"]
+      partition_name = task["route"]
+      component = task["component"]
+      command = component["command"]
+      descendants = {}
+      component["descendants"].each do |name, routes|
+        descendants[name] = routes.collect do |route|
+          farm_path(route)
+        end
+      end
+      local_message["descendants"] = descendants
+      farm_message = @message.merge("body" => local_message,
+                                    "type" => command)
+      @farm.process(partition_name, farm_message)
+    end
+
+    def local?(route)
+      route =~ @local
+    end
+
+    private
     def generate_id
       id = @current_id
       @current_id = id.succ
@@ -166,38 +202,21 @@ module Droonga
 
     def farm_path(route)
       if route =~ /\A.*:\d+\/[^\.]+/
-        $&
+        $MATCH
       else
         route
       end
     end
 
-    def local?(route)
-      route =~ @local
-    end
-
-    private
-    def is_route?(route)
-      route.is_a?(String) || route.is_a?(Hash)
+    def process_input_message(message)
+      adapted_message = @input_adapter.adapt(message)
+      @distributor.process(adapted_message["type"], adapted_message)
+    rescue Droonga::Pluggable::UnknownPlugin => error
+      raise UnknownCommand.new(error.command, message["dataset"])
     end
 
-    def apply_input_adapters(envelope)
-      adapted_envelope = envelope
-      loop do
-        input_message = InputMessage.new(adapted_envelope)
-        command = input_message.command
-        break unless @adapter.processable?(command)
-        @adapter.process(command, input_message)
-        new_command = input_message.command
-        adapted_envelope = input_message.adapted_envelope
-        break if command == new_command
-      end
-      adapted_envelope
-    end
-
-    def process_input_message(envelope)
-      adapted_envelope = apply_input_adapters(envelope)
-      @distributor.distribute(adapted_envelope)
+    def assert_valid_message
+      raise MissingDatasetParameter.new unless @message.include?("dataset")
     end
 
     def log_tag
@@ -206,65 +225,13 @@ module Droonga
 
     class Planner
       attr_reader :components
-      class UndefinedInputError < StandardError
-        attr_reader :input
-        def initialize(input)
-          @input = input
-          super("undefined input assigned: <#{input}>")
-        end
-      end
 
-      class CyclicComponentsError < StandardError
-        attr_reader :components
-        def initialize(components)
-          @components = components
-          super("cyclic components found: <#{components}>")
-        end
-      end
-
-      include TSort
       def initialize(dispatcher, components)
         @dispatcher = dispatcher
         @components = components
       end
 
-      def resolve(id)
-        @dependency = {}
-        @components.each do |component|
-          @dependency[component] = component["inputs"]
-          next unless component["outputs"]
-          component["outputs"].each do |output|
-            @dependency[output] = [component]
-          end
-        end
-        @components = []
-        each_strongly_connected_component do |cs|
-          raise CyclicComponentsError.new(cs) if cs.size > 1
-          @components.concat(cs) unless cs.first.is_a? String
-        end
-        resolve_routes(id)
-      end
-
-      def resolve_routes(id)
-        local = [id]
-        destinations = Hash.new(0)
-        @components.each do |component|
-          dataset = component["dataset"]
-          routes =
-            if dataset
-              Droonga.catalog.get_routes(dataset, component)
-            else
-              local
-            end
-          routes.each do |route|
-            destinations[@dispatcher.farm_path(route)] += 1
-          end
-          component["routes"] = routes
-        end
-        return destinations
-      end
-
-      def get_collector(id)
+      def create_session(id, collector)
         resolve_descendants
         tasks = []
         inputs = {}
@@ -284,9 +251,7 @@ module Droonga
             end
           end
         end
-        collector = Collector.new(id, @dispatcher, @components, tasks, inputs)
-        @dispatcher.collectors[id] = collector
-        return collector
+        Session.new(id, @dispatcher, collector, tasks, inputs)
       end
 
       def resolve_descendants
@@ -300,35 +265,15 @@ module Droonga
           component["n_of_expects"] = 0
         end
         @components.each do |component|
-          descendants = get_descendants(component)
-          component["descendants"] = descendants
-          descendants.each do |key, indices|
-            indices.each do |index|
+          descendants = {}
+          (component["outputs"] || []).each do |output|
+            descendants[output] = []
+            @descendants[output].each do |index|
               @components[index]["n_of_expects"] += component["routes"].size
+              descendants[output].concat(@components[index]["routes"])
             end
           end
-        end
-      end
-
-      def get_descendants(component)
-        return {} unless component["outputs"]
-        descendants = {}
-        component["outputs"].each do |output|
-          descendants[output] = @descendants[output]
-        end
-        descendants
-      end
-
-      def tsort_each_node(&block)
-        @dependency.each_key(&block)
-      end
-
-      def tsort_each_child(node, &block)
-        if node.is_a? String and @dependency[node].nil?
-          raise UndefinedInputError.new(node)
-        end
-        if @dependency[node]
-          @dependency[node].each(&block)
+          component["descendants"] = descendants
         end
       end
     end