fluent-plugin-droonga 0.9.0 → 0.9.9

data/lib/droonga/dispatcher.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,9 +16,8 @@
 require "English"
 require "tsort"
 
-require "droonga/input_adapter"
-require "droonga/output_adapter"
-require "droonga/distributor"
+require "droonga/adapter_runner"
+require "droonga/planner"
 require "droonga/catalog"
 require "droonga/collector"
 require "droonga/farm"
@@ -28,6 +25,7 @@ require "droonga/session"
 require "droonga/replier"
 require "droonga/message_processing_error"
 require "droonga/catalog_observer"
+require "droonga/distributor"
 
 module Droonga
   class Dispatcher
@@ -39,6 +37,12 @@ module Droonga
       end
     end
 
+    class UnknownDataset < NotFound
+      def initialize(dataset)
+        super("The dataset #{dataset.inspect} does not exist.")
+      end
+    end
+
     class UnknownCommand < BadRequest
       def initialize(command, dataset)
         super("The command #{command.inspect} is not available " +
@@ -54,15 +58,14 @@ module Droonga
       @sessions = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
-      @input_adapter =
-        InputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
-      @output_adapter =
-        OutputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
+      @adapter_runners = create_adapter_runners
       @farm = Farm.new(name, @loop, :dispatcher => self)
       @forwarder = Forwarder.new(@loop)
       @replier = Replier.new(@forwarder)
-      @distributor = Distributor.new(self, @options)
-      @collector = Collector.new
+      # TODO: make customizable
+      @planner = Planner.new(self, ["search", "crud", "groonga", "watch"])
+      # TODO: make customizable
+      @collector = Collector.new(["basic", "search"])
     end
 
     def start
@@ -75,10 +78,11 @@ module Droonga
 
     def shutdown
       @forwarder.shutdown
-      @distributor.shutdown
+      @planner.shutdown
       @collector.shutdown
-      @input_adapter.shutdown
-      @output_adapter.shutdown
+      @adapter_runners.each_value do |adapter_runner|
+        adapter_runner.shutdown
+      end
       @farm.shutdown
       @loop.stop
       @loop_thread.join
@@ -90,11 +94,17 @@ module Droonga
         process_internal_message(message["body"])
       else
         begin
-          assert_valid_message
+          assert_valid_message(message)
           process_input_message(message)
         rescue MessageProcessingError => error
           reply("statusCode" => error.status_code,
                 "body"       => error.response_body)
+        rescue => error
+          Logger.error("failed to process input message", error)
+          formatted_error = MessageProcessingError.new("Unknown internal error")
+          reply("statusCode" => formatted_error.status_code,
+                "body"       => formatted_error.response_body)
+          raise error
         end
       end
     end
@@ -118,7 +128,11 @@ module Droonga
     #
     # @see Replier#reply
     def reply(message)
-      adapted_message = @output_adapter.adapt(@message.merge(message))
+      adapted_message = @message.merge(message)
+      adapter_runner = @adapter_runners[adapted_message["dataset"]]
+      if adapter_runner
+        adapted_message = adapter_runner.adapt_output(adapted_message)
+      end
       return if adapted_message["replyTo"].nil?
       @replier.reply(adapted_message)
     end
@@ -129,10 +143,10 @@ module Droonga
       if session
         session.receive(message["input"], message["value"])
       else
-        components = message["components"]
-        if components
-          planner = Planner.new(self, components)
-          session = planner.create_session(id, @collector)
+        steps = message["steps"]
+        if steps
+          session_planner = SessionPlanner.new(self, steps)
+          session = session_planner.create_session(id, @collector)
           @sessions[id] = session
         else
           #todo: take cases receiving result before its query into account
@@ -152,23 +166,23 @@ module Droonga
       end
     end
 
-    def dispatch_components(components)
+    def dispatch_steps(steps)
       id = generate_id
       destinations = {}
-      components.each do |component|
-        dataset = component["dataset"]
+      steps.each do |step|
+        dataset = step["dataset"]
         if dataset
-          routes = Droonga.catalog.get_routes(dataset, component)
-          component["routes"] = routes
+          routes = Droonga.catalog.get_routes(dataset, step)
+          step["routes"] = routes
         else
-          component["routes"] ||= [id]
+          step["routes"] ||= [id]
         end
-        routes = component["routes"]
+        routes = step["routes"]
         routes.each do |route|
           destinations[farm_path(route)] = true
         end
       end
-      dispatch_message = { "id" => id, "components" => components }
+      dispatch_message = { "id" => id, "steps" => steps }
       destinations.each_key do |destination|
         dispatch(dispatch_message, destination)
       end
@@ -177,10 +191,10 @@ module Droonga
     def process_local_message(local_message)
       task = local_message["task"]
       partition_name = task["route"]
-      component = task["component"]
-      command = component["command"]
+      step = task["step"]
+      command = step["command"]
       descendants = {}
-      component["descendants"].each do |name, routes|
+      step["descendants"].each do |name, routes|
         descendants[name] = routes.collect do |route|
           farm_path(route)
         end
@@ -211,43 +225,61 @@ module Droonga
     end
 
     def process_input_message(message)
-      adapted_message = @input_adapter.adapt(message)
-      @distributor.process(adapted_message["type"], adapted_message)
-    rescue Droonga::Pluggable::UnknownPlugin => error
+      dataset = message["dataset"]
+      adapter_runner = @adapter_runners[dataset]
+      adapted_message = adapter_runner.adapt_input(message)
+      plan = @planner.process(adapted_message["type"], adapted_message)
+      distributor = Distributor.new(self)
+      distributor.distribute(plan)
+    rescue Droonga::LegacyPluggable::UnknownPlugin => error
       raise UnknownCommand.new(error.command, message["dataset"])
     end
 
-    def assert_valid_message
-      raise MissingDatasetParameter.new unless @message.include?("dataset")
+    def assert_valid_message(message)
+      unless message.key?("dataset")
+        raise MissingDatasetParameter.new
+      end
+      dataset = message["dataset"]
+      unless Droonga.catalog.have_dataset?(dataset)
+        raise UnknownDataset.new(dataset)
+      end
+    end
+
+    def create_adapter_runners
+      runners = {}
+      Droonga.catalog.datasets.each do |name, configuration|
+        runners[name] = AdapterRunner.new(self, configuration["plugins"] || [])
+      end
+      runners
     end
 
     def log_tag
       "[#{Process.ppid}][#{Process.pid}] dispatcher"
     end
 
-    class Planner
-      attr_reader :components
+    class SessionPlanner
+      attr_reader :steps
 
-      def initialize(dispatcher, components)
+      def initialize(dispatcher, steps)
         @dispatcher = dispatcher
-        @components = components
+        @steps = steps
       end
 
       def create_session(id, collector)
         resolve_descendants
         tasks = []
         inputs = {}
-        @components.each do |component|
-          component["routes"].each do |route|
+        @steps.each do |step|
+          step["routes"].each do |route|
             next unless @dispatcher.local?(route)
             task = {
               "route" => route,
-              "component" => component,
+              "step" => step,
               "n_of_inputs" => 0,
               "values" => {}
             }
             tasks << task
-            (component["inputs"] || [nil]).each do |input|
+            (step["inputs"] || [nil]).each do |input|
               inputs[input] ||= []
               inputs[input] << task
             end
@@ -258,24 +290,24 @@ module Droonga
 
       def resolve_descendants
         @descendants = {}
-        @components.size.times do |index|
-          component = @components[index]
-          (component["inputs"] || []).each do |input|
+        @steps.size.times do |index|
+          step = @steps[index]
+          (step["inputs"] || []).each do |input|
             @descendants[input] ||= []
             @descendants[input] << index
           end
-          component["n_of_expects"] = 0
+          step["n_of_expects"] = 0
         end
-        @components.each do |component|
+        @steps.each do |step|
           descendants = {}
-          (component["outputs"] || []).each do |output|
+          (step["outputs"] || []).each do |output|
             descendants[output] = []
             @descendants[output].each do |index|
-              @components[index]["n_of_expects"] += component["routes"].size
-              descendants[output].concat(@components[index]["routes"])
+              @steps[index]["n_of_expects"] += step["routes"].size
+              descendants[output].concat(@steps[index]["routes"])
             end
           end
-          component["descendants"] = descendants
+          step["descendants"] = descendants
         end
       end
     end

data/lib/droonga/distributed_command_planner.rb

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class DistributedCommandPlanner
+    attr_accessor :key, :dataset
+
+    REDUCE_SUM = "sum"
+
+    DEFAULT_LIMIT = -1
+
+    def initialize(source_message)
+      @source_message = source_message
+
+      @key = nil
+      @dataset = nil
+      @outputs = []
+
+      @reducers = []
+      @gatherers = []
+      @processor = nil
+
+      plan_errors_handling
+    end
+
+    def plan
+      unified_reducers + unified_gatherers + [fixed_processor]
+    end
+
+    def reduce(params=nil)
+      return unless params
+      params.each do |name, reducer|
+        gatherer = nil
+        if reducer.is_a?(Hash) && reducer[:gather]
+          gatherer = reducer[:gather]
+          reducer = reducer[:reduce]
+        end
+        @reducers << reducer_message(reduce_command, name, reducer)
+        @gatherers << gatherer_message(gather_command, name, gatherer)
+        @outputs << name
+      end
+    end
+
+    def scatter(options={})
+      @processor = {
+        "command" => @source_message["type"],
+        "dataset" => @dataset || @source_message["dataset"],
+        "body"    => options[:body] || @source_message["body"],
+        "key"     => nil,
+        "type"    => "scatter",
+        "outputs" => [],
+        "replica" => "all",
+        "post"    => true
+      }
+    end
+
+    def broadcast(options={})
+      processor = {
+        "command" => @source_message["type"],
+        "dataset" => @dataset || @source_message["dataset"],
+        "body"    => options[:body] || @source_message["body"],
+        "type"    => "broadcast",
+        "outputs" => [],
+        "replica" => "random"
+      }
+      if options[:write]
+        processor["replica"] = "all"
+        processor["post"]    = true
+      end
+      @processor = processor
+    end
+
+    private
+    def reduce_command
+      "reduce"
+    end
+
+    def gather_command
+      "gather"
+    end
+
+    def unified_reducers
+      unified_reducers = {}
+      @reducers.each do |reducer|
+        type = reducer["type"]
+        unified = unified_reducers[type]
+        if unified
+          unified["body"] = unified["body"].merge(reducer["body"])
+          unified["inputs"] = unified["inputs"] + reducer["inputs"]
+          unified["outputs"] = unified["outputs"] + reducer["outputs"]
+        else
+          unified_reducers[type] = Marshal.load(Marshal.dump(reducer))
+        end
+      end
+      unified_reducers.values
+    end
+
+    def unified_gatherers
+      unified_gatherers = {}
+      @gatherers.each do |gatherer|
+        type = gatherer["type"]
+        unified = unified_gatherers[type]
+        if unified
+          unified["body"] = unified["body"].merge(gatherer["body"])
+          unified["inputs"] = unified["inputs"] + gatherer["inputs"]
+        else
+          unified_gatherers[type] = Marshal.load(Marshal.dump(gatherer))
+        end
+      end
+      unified_gatherers.values
+    end
+
+    def fixed_processor
+      @processor["outputs"] = @outputs
+      if @processor["type"] == "scatter"
+        raise MessageProcessingError.new("missing key") unless @key
+        @processor["key"] = @key
+      end
+      @processor
+    end
+
+    def reducer_message(command, name, reducer)
+      if reducer.is_a?(String)
+        reducer = {
+          "type" => reducer,
+        }
+        if reducer["type"] == REDUCE_SUM
+          reducer["limit"] = DEFAULT_LIMIT
+        end
+      end
+      {
+        "type"    => command,
+        "body"    => {
+          name => {
+            output_name(name) => reducer,
+          },
+        },
+        "inputs"  => [name],
+        "outputs" => [output_name(name)],
+      }
+    end
+
+    def gatherer_message(command, name, gatherer=nil)
+      gatherer ||= {}
+      {
+        "type"   => command,
+        "body"   => {
+          output_name(name) => {
+            "output" => name,
+          }.merge(gatherer),
+        },
+        "inputs" => [output_name(name)],
+        "post"   => true,
+      }
+    end
+
+    def output_name(name)
+      "#{name}_reduced"
+    end
+
+    #XXX Now, we include definitions to merge errors in the body.
+    #    However, this makes the term "errors" reserved, so plugins
+    #    cannot use their custom "errors" in the body.
+    #    This must be rewritten. 
+    def plan_errors_handling
+      reduce("errors"=> REDUCE_SUM)
+    end
+  end
+end