fluent-plugin-droonga 0.0.2 → 0.7.0

data/lib/droonga/catalog.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -63,7 +63,7 @@ module Droonga
       @options[name]
     end
 
-    def get_engines(name)
+    def get_partitions(name)
       device = @catalog["farms"][name]["device"]
       pattern = Regexp.new("^#{name}\.")
       results = {}

data/lib/droonga/collector.rb

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/collector_plugin"
+
+module Droonga
+  class Collector
+    include Pluggable
+
+    def initialize(id, dispatcher, components, tasks, inputs)
+      @id = id
+      @dispatcher = dispatcher
+      @components = components
+      @tasks = tasks
+      @n_dones = 0
+      @inputs = inputs
+      load_plugins(["basic"]) # TODO: make customizable
+    end
+
+    def handle(name, value)
+      tasks = @inputs[name]
+      unless tasks
+        #TODO: result arrived before its query
+        return
+      end
+      tasks.each do |task|
+        task["n_of_inputs"] += 1 if name
+        component = task["component"]
+        type = component["type"]
+        command = component["command"] || ("collector_" + type)
+        n_of_expects = component["n_of_expects"]
+        synchronous = nil
+        if command
+          synchronous = true unless n_of_expects.zero?
+          # TODO: check if asynchronous execution is available.
+          message = {
+            "task"=>task,
+            "name"=>name,
+            "value"=>value
+          }
+          unless synchronous
+            descendants = {}
+            component["descendants"].each do |name, indices|
+              descendants[name] = indices.collect do |index|
+                @components[index]["routes"].map do |route|
+                  @dispatcher.farm_path(route)
+                end
+              end
+            end
+            message["descendants"] = descendants
+            message["id"] = @id
+          end
+          if @id == task["route"]
+            process(command, message)
+          else
+            @dispatcher.deliver(@id, task["route"], message, command, synchronous)
+          end
+        end
+        return if task["n_of_inputs"] < n_of_expects
+        #the task is done
+        if synchronous
+          result = task["values"]
+          post = component["post"]
+          @dispatcher.post(result, post) if post
+          component["descendants"].each do |name, indices|
+            message = {
+              "id" => @id,
+              "input" => name,
+              "value" => result[name]
+            }
+            indices.each do |index|
+              @components[index]["routes"].each do |route|
+                @dispatcher.dispatch(message, route)
+              end
+            end
+          end
+        end
+        @n_dones += 1
+        @dispatcher.collectors.delete(@id) if @n_dones == @tasks.size
+      end
+    end
+
+    private
+    def instantiate_plugin(name)
+      CollectorPlugin.repository.instantiate(name, @dispatcher)
+    end
+
+    def log_tag
+      "collector"
+    end
+  end
+end

data/lib/droonga/collector_plugin.rb

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+
+module Droonga
+  class CollectorPlugin < Plugin
+    extend PluginRegisterable
+
+    attr_reader :task, :input_name, :component, :output_values, :body, :output_names
+    def initialize(dispatcher)
+      super()
+      @dispatcher = dispatcher
+    end
+
+    def process(command, message)
+      return false unless message.is_a? Hash
+      @task = message["task"]
+      return false unless @task.is_a? Hash
+      @component = @task["component"]
+      return false unless @component.is_a? Hash
+      @output_values = @task["values"]
+      @body = @component["body"]
+      @output_names = @component["outputs"]
+      @id = message["id"]
+      @value = message["value"]
+      @input_name = message["name"]
+      @descendants = message["descendants"]
+      super(command, @value)
+      output if @descendants
+      true
+    end
+
+    # TODO: consider better name
+    def emit(value, name=nil)
+      unless name
+        if @output_names
+          name = @output_names.first
+        else
+          @output_values = @task["values"] = value
+          return
+        end
+      end
+      @output_values[name] = value
+    end
+
+    def post(message, destination=nil)
+      @dispatcher.post(message, destination)
+    end
+
+    def output
+      result = @task["values"]
+      post(result, @component["post"]) if @component["post"]
+      @descendants.each do |name, dests|
+        message = {
+          "id" => @id,
+          "input" => name,
+          "value" => result[name]
+        }
+        dests.each do |routes|
+          routes.each do |route|
+            post(message, "to"=>route, "type"=>"dispatcher")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/command_mapper.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/lib/droonga/{proxy.rb → dispatcher.rb}

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,36 +16,94 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require 'tsort'
-require "droonga/handler"
 require "droonga/adapter"
+require "droonga/distributor"
 require "droonga/catalog"
+require "droonga/collector"
+require "droonga/farm"
+require "droonga/input_message"
 
 module Droonga
-  class Proxy
-    attr_reader :collectors
-    def initialize(worker, name)
-      @engines = {}
-      Droonga::catalog.get_engines(name).each do |name, options|
-        engine = Droonga::Engine.new(options.merge(:proxy => false,
-                                                   :with_server => false))
-        engine.start
-        @engines[name] = engine
-      end
-      @worker = worker
-      @name = name
+  class Dispatcher
+    attr_reader :name, :envelope, :collectors
+
+    def initialize(options)
+      @options = options
+      @name = @options[:name]
+      @farm = Farm.new(name)
       @collectors = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
-      plugins = ["proxy"] + (Droonga::catalog.option("plugins")||[]) + ["adapter"]
-      plugins.each do |plugin|
-        @worker.add_handler(plugin)
-      end
+      @adapter = Adapter.new(self,
+                             :adapters => Droonga.catalog.option("plugins"))
+      @forwarder = Forwarder.new
+      @distributor = Distributor.new(self, @options)
+    end
+
+    def start
+      @farm.start
     end
 
     def shutdown
-      @engines.each do |name, engine|
-        engine.shutdown
+      @forwarder.shutdown
+      @distributor.shutdown
+      @adapter.shutdown
+      @farm.shutdown
+    end
+
+    def add_route(route)
+      envelope["via"].push(route)
+    end
+
+    def handle_envelope(envelope)
+      @envelope = envelope
+      if envelope["type"] == "dispatcher"
+        handle(envelope["body"], envelope["arguments"])
+      else
+        process_input_message(envelope)
+      end
+    end
+
+    def post(body, destination=nil)
+      $log.trace("#{log_tag}: post: start")
+      route = nil
+      unless is_route?(destination)
+        route = envelope["via"].pop
+        destination = route
+      end
+      unless is_route?(destination)
+        destination = envelope["replyTo"]
+      end
+      command = nil
+      receiver = nil
+      arguments = nil
+      synchronous = nil
+      case destination
+      when String
+        command = destination
+      when Hash
+        command = destination["type"]
+        receiver = destination["to"]
+        arguments = destination["arguments"]
+        synchronous = destination["synchronous"]
+      end
+      if receiver
+        @forwarder.forward(envelope, body,
+                           "type" => command,
+                           "to" => receiver,
+                           "arguments" => arguments)
+      else
+        if command == "dispatcher"
+          handle(body, arguments)
+        elsif @adapter.processable?(command)
+          @adapter.process(command, body, *arguments)
+        else
+          @distributor.distribute(envelope.merge("type" => command,
+                                                 "body" => body))
+        end
       end
+      add_route(route) if route
+      $log.trace("#{log_tag}: post: done")
     end
 
     def handle(message, arguments)
@@ -87,7 +145,7 @@ module Droonga
       if local?(destination)
         handle_internal_message(message)
       else
-        post(message, "to"=>farm_path(destination), "type"=>"proxy")
+        post(message, "to"=>farm_path(destination), "type"=>"dispatcher")
       end
     end
 
@@ -95,15 +153,11 @@ module Droonga
       if id == route
         post(message, "type" => type, "synchronous"=> synchronous)
       else
-        envelope = @worker.envelope.merge("body" => message, "type" => type)
-        @engines[route].emit('', Time.now.to_f, envelope, synchronous)
+        envelope = @envelope.merge("body" => message, "type" => type)
+        @farm.process(route, envelope, synchronous)
       end
     end
 
-    def post(message, destination)
-      @worker.post(message, destination)
-    end
-
     def generate_id
       id = @current_id
       @current_id = id.succ
@@ -122,6 +176,34 @@ module Droonga
       route =~ @local
     end
 
+    private
+    def is_route?(route)
+      route.is_a?(String) || route.is_a?(Hash)
+    end
+
+    def apply_input_adapters(envelope)
+      adapted_envelope = envelope
+      loop do
+        input_message = InputMessage.new(adapted_envelope)
+        command = input_message.command
+        break unless @adapter.processable?(command)
+        @adapter.process(command, input_message)
+        new_command = input_message.command
+        adapted_envelope = input_message.adapted_envelope
+        break if command == new_command
+      end
+      adapted_envelope
+    end
+
+    def process_input_message(envelope)
+      adapted_envelope = apply_input_adapters(envelope)
+      @distributor.distribute(adapted_envelope)
+    end
+
+    def log_tag
+      "[#{Process.ppid}][#{Process.pid}] dispatcher"
+    end
+
     class Planner
       attr_reader :components
       class UndefinedInputError < StandardError
@@ -141,8 +223,8 @@ module Droonga
       end
 
       include TSort
-      def initialize(proxy, components)
-        @proxy = proxy
+      def initialize(dispatcher, components)
+        @dispatcher = dispatcher
         @components = components
       end
 
@@ -170,12 +252,12 @@ module Droonga
           dataset = component["dataset"]
           routes =
             if dataset
-              Droonga::catalog.get_routes(dataset, component)
+              Droonga.catalog.get_routes(dataset, component)
             else
               local
             end
           routes.each do |route|
-            destinations[@proxy.farm_path(route)] += 1
+            destinations[@dispatcher.farm_path(route)] += 1
           end
           component["routes"] = routes
         end
@@ -188,7 +270,7 @@ module Droonga
         inputs = {}
         @components.each do |component|
           component["routes"].each do |route|
-            next unless @proxy.local?(route)
+            next unless @dispatcher.local?(route)
             task = {
               "route" => route,
               "component" => component,
@@ -202,8 +284,8 @@ module Droonga
             end
           end
         end
-        collector = Collector.new(id, @proxy, @components, tasks, inputs)
-        @proxy.collectors[id] = collector
+        collector = Collector.new(id, @dispatcher, @components, tasks, inputs)
+        @dispatcher.collectors[id] = collector
         return collector
       end
 
@@ -250,122 +332,5 @@ module Droonga
         end
       end
     end
-
-    class Collector
-      def initialize(id, proxy, components, tasks, inputs)
-        @id = id
-        @proxy = proxy
-        @components = components
-        @tasks = tasks
-        @n_dones = 0
-        @inputs = inputs
-      end
-
-      def handle(name, value)
-        tasks = @inputs[name]
-        unless tasks
-          #TODO: result arrived before its query
-          return
-        end
-        tasks.each do |task|
-          task["n_of_inputs"] += 1 if name
-          component = task["component"]
-          type = component["type"]
-          command = component["command"] || ("proxy_" + type)
-          n_of_expects = component["n_of_expects"]
-          synchronous = nil
-          if command
-            # TODO: should be controllable for each command respectively.
-            synchronous = !n_of_expects.zero?
-            # TODO: check if asynchronous execution is available.
-            message = {
-              "task"=>task,
-              "name"=>name,
-              "value"=>value
-            }
-            unless synchronous
-              descendants = {}
-              component["descendants"].each do |name, indices|
-                descendants[name] = indices.collect do |index|
-                  @components[index]["routes"].map do |route|
-                    @proxy.farm_path(route)
-                  end
-                end
-              end
-              message["descendants"] = descendants
-              message["id"] = @id
-            end
-            @proxy.deliver(@id, task["route"], message, command, synchronous)
-          end
-          return if task["n_of_inputs"] < n_of_expects
-          #the task is done
-          if synchronous
-            result = task["values"]
-            post = component["post"]
-            @proxy.post(result, post) if post
-            component["descendants"].each do |name, indices|
-              message = {
-                "id" => @id,
-                "input" => name,
-                "value" => result[name]
-              }
-              indices.each do |index|
-                @components[index]["routes"].each do |route|
-                  @proxy.dispatch(message, route)
-                end
-              end
-            end
-          end
-          @n_dones += 1
-          @proxy.collectors.delete(@id) if @n_dones == @tasks.size
-        end
-      end
-    end
-  end
-
-  class ProxyMessageHandler < Droonga::Handler
-    Droonga::HandlerPlugin.register("proxy_message", self)
-    def initialize(*arguments)
-      super
-      @proxy = Droonga::Proxy.new(@worker, @worker.name)
-    end
-
-    def shutdown
-      @proxy.shutdown
-    end
-
-    command :proxy
-    def proxy(request, *arguments)
-      @proxy.handle(request, arguments)
-    end
-
-    def prefer_synchronous?(command)
-      return true
-    end
-  end
-
-  class ProxyHandler < Droonga::Handler
-    attr_reader :task, :input_name, :component, :output_values, :body, :output_names
-    def handle(command, request, *arguments)
-      return false unless request.is_a? Hash
-      @task = request["task"]
-      return false unless @task.is_a? Hash
-      @component = @task["component"]
-      return false unless @component.is_a? Hash
-      @output_values = @task["values"]
-      @body = @component["body"]
-      @output_names = @component["outputs"]
-      @id = request["id"]
-      @value = request["value"]
-      @input_name = request["name"]
-      @descendants = request["descendants"]
-      invoke(command, @value, *arguments)
-      output if @descendants
-      true
-    end
-
-    def prefer_synchronous?(command)
-      return true
-    end
   end
 end