fluent-plugin-droonga 0.9.9 → 1.0.0

data/lib/droonga/plugin_loader.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,39 +17,47 @@
 
 require "pathname"
 
+require "droonga/loggable"
+
 module Droonga
   class PluginLoader
+    include Loggable
+
     class << self
       def load_all
-        $LOAD_PATH.each do |load_path|
-          Dir.glob("#{load_path}/droonga/plugin/*") do |type_path|
-            next unless File.directory?(type_path)
-            type = File.basename(type_path)
-            Dir.glob("#{type_path}/*.rb") do |path|
-              name = File.basename(path, ".rb")
-              loader = new(type, name)
-              loader.load
-            end
-          end
-
-          Pathname.glob("#{load_path}/droonga/plugins/*.rb") do |plugin_path|
-            relative_plugin_path =
-              plugin_path.relative_path_from(Pathname(load_path))
-            require_path = relative_plugin_path.to_s.gsub(/\.rb\z/, "")
-            require require_path
-          end
-        end
+        loader = new
+        loader.load_all
+      end
+    end
+
+    def initialize
+    end
+
+    def load(name)
+      logger.debug("loading...: <#{name}>")
+      path = "droonga/plugins/#{name}"
+      begin
+        require path
+      rescue StandardError, SyntaxError => error
+        logger.exception("failed to load: <#{path}>", error)
+        raise
       end
     end
 
-    def initialize(type, name)
-      @type = type
-      @name = name
+    def load_all
+      $LOAD_PATH.each do |load_path|
+        search_pattern = "#{load_path}/droonga/plugins/*.rb"
+        logger.debug("searching...: <#{search_pattern}>")
+        Pathname.glob(search_pattern) do |plugin_path|
+          name = Pathname(plugin_path).basename(".rb").to_s
+          load(name)
+        end
+      end
     end
 
-    def load
-      return if @type == "metadata"
-      require "droonga/plugin/#{@type}/#{@name}"
+    private
+    def log_tag
+      "plugin-loader"
     end
   end
 end

data/lib/droonga/plugin_registry.rb

@@ -17,6 +17,12 @@ module Droonga
   class PluginRegistry
     include Enumerable
 
+    class UnknownPlugin < Error
+      def initialize(name, klass)
+        super("[#{klass}] Plugin not found: <#{name}>")
+      end
+    end
+
     def initialize
       @plugins = {}
     end
@@ -39,7 +45,9 @@ module Droonga
 
     def find_sub_classes(name, klass)
       plugin_module = self[name]
-      return [] if plugin_module.nil?
+      if plugin_module.nil?
+        raise UnknownPlugin.new(name, klass)
+      end
       sub_classes = []
       collect_sub_classes_recursive(plugin_module, klass, sub_classes)
       sub_classes

data/lib/droonga/plugins/basic.rb

@@ -0,0 +1,54 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+require "droonga/reducer"
+
+module Droonga
+  module Plugins
+    module Basic
+      extend Plugin
+      register("basic")
+
+      class GatherCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "gather"]
+
+        def collect(message)
+          output = message.input || message.name
+          if output.is_a?(Hash)
+            output_name = output["output"]
+          else
+            output_name = output
+          end
+          message.values[output_name] = message.value
+        end
+      end
+
+      class ReduceCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "reduce"]
+
+        def collect(message)
+          message.input.each do |output_name, deal|
+            left_value = message.values[output_name]
+            right_value = message.value
+            reducer = Reducer.new(deal)
+            value = reducer.reduce(left_value, right_value)
+            message.values[output_name] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/plugins/crud.rb

@@ -16,28 +16,40 @@
 require "groonga"
 
 require "droonga/plugin"
-require "droonga/message_processing_error"
+require "droonga/error_messages"
 
 module Droonga
   module Plugins
     module CRUD
-      Plugin.registry.register("crud", self)
+      extend Plugin
+      register("crud")
 
       class Adapter < Droonga::Adapter
-        message.input_pattern  = ["type", :equal, "add"]
-        message.output_pattern = ["body.success", :exist?]
+        input_message.pattern  = ["type", :equal, "add"]
+        output_message.pattern = ["body.success", :exist]
+
+        def adapt_input(input_message)
+          request = input_message.body
+          key = request["key"] || rand.to_s
+          values = request["values"] || {}
+          request["filter"] = values.merge("key" => key)
+        end
 
         def adapt_output(output_message)
-          success = output_message.body["success"]
-          unless success.nil?
-            output_message.body = output_message.body["success"]
+          if output_message.errors
+            detail = output_message.body["detail"]
+            return if detail.nil?
+            detail.delete("filter")
+            output_message.errors.each do |path, error|
+              error["body"]["detail"].delete("filter")
+            end
+          else
+            output_message.body.delete("filter")
           end
         end
       end
 
       class Handler < Droonga::Handler
-        message.type = "add"
-
         class MissingTableParameter < BadRequest
           def initialize
             super("\"table\" must be specified.")
@@ -71,12 +83,8 @@ module Droonga
           end
         end
 
-        def handle(message, messenger)
-          succeeded = process_add(message.request)
-          outputs = {
-            "success" => succeeded,
-          }
-          messenger.emit(outputs)
+        def handle(message)
+          process_add(message.request)
         end
 
         private
@@ -119,6 +127,19 @@ module Droonga
           end
         end
       end
+
+      define_single_step do |step|
+        step.name = "add"
+        step.inputs = {
+          "table" => {
+            :type => :table,
+            :filter => "filter",
+          },
+        }
+        step.write = true
+        step.handler = Handler
+        step.collector = Collectors::And
+      end
     end
   end
 end

data/lib/droonga/plugins/error.rb

@@ -18,14 +18,15 @@ require "droonga/plugin"
 module Droonga
   module Plugins
     module Error
-      Plugin.registry.register("error", self)
+      extend Plugin
+      register("error")
 
       class Adapter < Droonga::Adapter
-        message.output_pattern = ["body.errors", :exist?]
+        output_message.pattern = ["body.errors", :exist]
 
         def adapt_output(output_message)
           errors = output_message.body["errors"]
-          if errors && !errors.empty?
+          if errors and !errors.empty?
             output_message.errors = errors
 
             status_codes = []
@@ -36,7 +37,7 @@ module Droonga
             if status_codes.size == 1
               output_message.status_code = status_codes.first
             else
-              output_message.status_code = MessageProcessingError::STATUS_CODE
+              output_message.status_code = ErrorMessages::InternalServerError::STATUS_CODE
             end
 
             output_message.body = errors.values.first["body"]

data/lib/droonga/plugins/groonga.rb

@@ -14,16 +14,19 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/plugin"
-require "droonga/plugins/groonga/generic_response"
-require "droonga/plugins/groonga/select"
-require "droonga/plugins/groonga/table_create"
-require "droonga/plugins/groonga/table_remove"
-require "droonga/plugins/groonga/column_create"
 
 module Droonga
   module Plugins
     module Groonga
-      Plugin.registry.register("groonga", self)
+      extend Plugin
+      register("groonga")
     end
   end
 end
+
+require "droonga/plugins/groonga/generic_response"
+require "droonga/plugins/groonga/select"
+require "droonga/plugins/groonga/table_create"
+require "droonga/plugins/groonga/table_remove"
+require "droonga/plugins/groonga/column_create"
+

data/lib/droonga/plugins/groonga/column_create.rb

@@ -28,7 +28,7 @@ module Droonga
             @command = command_class.new("column_create", request)
 
             table_name = @command["table"]
-            if table_name.nil? || @context[table_name].nil?
+            if table_name.nil? or @context[table_name].nil?
               message = "table doesn't exist: <#{table_name.to_s}>"
               raise CommandError.new(:status => Status::INVALID_ARGUMENT,
                                      :message => message,
@@ -108,15 +108,20 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          message.type = "column_create"
           action.synchronous = true
 
-          def handle(message, messenger)
+          def handle(message)
             command = Command.new(@context)
-            outputs = command.execute(message.request)
-            messenger.emit(outputs)
+            command.execute(message.request)
           end
         end
+
+        Groonga.define_single_step do |step|
+          step.name = "column_create"
+          step.write = true
+          step.handler = Handler
+          step.collector = Collectors::Add
+        end
       end
     end
   end

data/lib/droonga/plugins/groonga/generic_command.rb

@@ -41,9 +41,9 @@ module Droonga
         def execute(request)
           @start_time = Time.now.to_f
           result = process_request(request)
-          format(header(Status::SUCCESS), result)
+          [header(Status::SUCCESS), result]
         rescue CommandError => error
-          format(header(error.status, error.message), error.result)
+          [header(error.status, error.message), error.result]
         end
 
         private
@@ -53,12 +53,6 @@ module Droonga
           header.push(error_message) unless error_message.empty?
           header
         end
-
-        def format(header, body)
-          {
-            "result" => [header, body],
-          }
-        end
       end
     end
   end

data/lib/droonga/plugins/groonga/generic_response.rb

@@ -25,8 +25,8 @@ module Droonga
             "table_remove",
             "column_create",
           ]
-          message.input_pattern  = ["type", :in, groonga_commands]
-          message.output_pattern = ["body.result", :exist?]
+          input_message.pattern  = ["type", :in, groonga_commands]
+          output_message.pattern = ["body.result", :exist]
 
           def adapt_output(output_message)
             output_message.body = output_message.body["result"]

data/lib/droonga/plugins/groonga/select.rb

@@ -101,13 +101,13 @@ module Droonga
         end
 
         class Adapter < Droonga::Adapter
-          message.input_pattern = ["type", :equal, "select"]
+          input_message.pattern = ["type", :equal, "select"]
 
           def adapt_input(input_message)
             converter = RequestConverter.new
             select_request = input_message.body
             search_request = converter.convert(select_request)
-            input_message.command = "search"
+            input_message.type = "search"
             input_message.body = search_request
           end
 

data/lib/droonga/plugins/groonga/table_create.rb

@@ -91,15 +91,20 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          message.type = "table_create"
           action.synchronous = true
 
-          def handle(message, messenger)
+          def handle(message)
             command = Command.new(@context)
-            outputs = command.execute(message.request)
-            messenger.emit(outputs)
+            command.execute(message.request)
           end
         end
+
+        Groonga.define_single_step do |step|
+          step.name = "table_create"
+          step.write = true
+          step.handler = Handler
+          step.collector = Collectors::Add
+        end
       end
     end
   end

data/lib/droonga/plugins/groonga/table_remove.rb

@@ -28,7 +28,7 @@ module Droonga
             @command = command_class.new("table_remove", request)
 
             name = @command["name"]
-            if name.nil? || @context[name].nil?
+            if name.nil? or @context[name].nil?
               raise CommandError.new(:status => Status::INVALID_ARGUMENT,
                                      :message => "table not found",
                                      :result => false)
@@ -42,15 +42,20 @@ module Droonga
         end
 
         class Handler < Droonga::Handler
-          message.type = "table_remove"
           action.synchronous = true
 
-          def handle(message, messenger)
+          def handle(message)
             command = Command.new(@context)
-            outputs = command.execute(message.request)
-            messenger.emit(outputs)
+            command.execute(message.request)
           end
         end
+
+        Groonga.define_single_step do |step|
+          step.name = "table_remove"
+          step.write = true
+          step.handler = Handler
+          step.collector = Collectors::Add
+        end
       end
     end
   end

data/lib/droonga/plugins/search.rb

@@ -15,16 +15,23 @@
 
 require "droonga/plugin"
 require "droonga/searcher"
+require "droonga/plugins/search/distributed_search_planner"
 
 module Droonga
   module Plugins
     module Search
-      Plugin.registry.register("search", self)
+      extend Plugin
+      register("search")
 
-      class Handler < Droonga::Handler
-        message.type = "search"
+      class Planner < Droonga::Planner
+        def plan(message)
+          planner = DistributedSearchPlanner.new(message)
+          planner.plan
+        end
+      end
 
-        def handle(message, messenger)
+      class Handler < Droonga::Handler
+        def handle(message)
           searcher = Droonga::Searcher.new(@context)
           values = {}
           request = message.request
@@ -32,9 +39,103 @@ module Droonga
           searcher.search(request["queries"]).each do |output, value|
             values[output] = value
           end
-          messenger.emit(values)
+          values
+        end
+      end
+
+      class GatherCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "search_gather"]
+
+        def collect(message)
+          output = message.input || message.name
+          if output.is_a?(Hash)
+            elements = output["elements"]
+            if elements and elements.is_a?(Hash)
+              # because "count" mapper requires all records,
+              # I have to apply it at first, before "limit" and "offset" are applied.
+              body = message.body
+              value = message.value
+              count_mapper = elements["count"]
+              if count_mapper
+                if count_mapper["no_output"]
+                  value.delete("count")
+                else
+                  value["count"] = value[count_mapper["target"]].size
+                end
+              end
+
+              records_mapper = elements["records"]
+              if records_mapper and value["records"]
+                if records_mapper["no_output"]
+                  value.delete("records")
+                else
+                  value["records"] = Reducer.apply_range(value["records"],
+                                                         records_mapper)
+                  value["records"] = apply_output_attributes_and_format(value["records"], records_mapper)
+                end
+              end
+            end
+            output_name = output["output"]
+          else
+            output_name = output
+          end
+          message.values[output_name] = message.value
+        end
+
+        private
+        def apply_output_attributes_and_format(items, output)
+          attributes = output["attributes"] || []
+          if output["format"] == "complex"
+            items.collect do |item|
+              complex_item = {}
+              attributes.each_with_index do |label, index|
+                complex_item[label] = item[index]
+              end
+              complex_item
+            end
+          else
+            items.collect do |item|
+              item[0...attributes.size]
+            end
+          end
         end
       end
+
+      class ReduceCollector < Droonga::Collector
+        message.pattern = ["task.step.type", :equal, "search_reduce"]
+
+        def collect(message)
+          #XXX This is just a workaround. Errors should be handled by the framework itself.
+          if message.name == "errors"
+            basic_reduce_collector = Basic::ReduceCollector.new
+            return basic_reduce_collector.collect(message)
+          end
+
+          message.input.each do |output_name, elements|
+            old_value = message.values[output_name]
+            if old_value
+              value = reduce_elements(elements, old_value, message.value)
+            else
+              value = message.value
+            end
+            message.values[output_name] = value
+          end
+        end
+
+        def reduce_elements(elements, left_values, right_values)
+          result = {}
+          elements.each do |key, deal|
+            reducer = Reducer.new(deal)
+            result[key] = reducer.reduce(left_values[key], right_values[key])
+          end
+          result
+        end
+      end
+
+      define_single_step do |step|
+        step.name = "search"
+        step.handler = Handler
+      end
     end
   end
 end