fluent-plugin-droonga 0.9.9 → 1.0.0

data/lib/droonga/collectors.rb

@@ -0,0 +1,18 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/collectors/add"
+require "droonga/collectors/and"
+require "droonga/collectors/sum"

data/lib/droonga/{catalog.rb → collectors/add.rb}

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,10 +13,14 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/catalog_loader"
-
 module Droonga
-  class << self
-    attr_accessor :catalog
+  module Collectors
+    class Add
+      class << self
+        def operator
+          "or"
+        end
+      end
+    end
   end
 end

data/lib/droonga/{command_repository.rb → collectors/and.rb}

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,19 +14,12 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 module Droonga
-  class CommandRepository
-    attr_reader :commands
-    def initialize
-      @commands = []
-    end
-
-    def register(command)
-      @commands << command
-    end
-
-    def find(message)
-      @commands.find do |command|
-        command.match?(message)
+  module Collectors
+    class And
+      class << self
+        def operator
+          "and"
+        end
       end
     end
   end

data/lib/droonga/collectors/sum.rb

@@ -0,0 +1,26 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Collectors
+    class Sum
+      class << self
+        def operator
+          "sum"
+        end
+      end
+    end
+  end
+end

data/lib/droonga/dispatcher.rb

@@ -16,56 +16,54 @@
 require "English"
 require "tsort"
 
+require "droonga/loggable"
 require "droonga/adapter_runner"
-require "droonga/planner"
-require "droonga/catalog"
-require "droonga/collector"
+require "droonga/collector_runner"
+require "droonga/step_runner"
 require "droonga/farm"
 require "droonga/session"
 require "droonga/replier"
-require "droonga/message_processing_error"
-require "droonga/catalog_observer"
+require "droonga/error_messages"
 require "droonga/distributor"
 
 module Droonga
   class Dispatcher
+    include Loggable
+
     attr_reader :name
 
-    class MissingDatasetParameter < BadRequest
+    class MissingDatasetParameter < ErrorMessages::BadRequest
       def initialize
-        super("\"dataset\" must be specified.")
+        super("Missing required parameter: <dataset>")
       end
     end
 
-    class UnknownDataset < NotFound
+    class UnknownDataset < ErrorMessages::NotFound
       def initialize(dataset)
-        super("The dataset #{dataset.inspect} does not exist.")
+        super("Unknown dataset: <#{dataset}>")
       end
     end
 
-    class UnknownCommand < BadRequest
-      def initialize(command, dataset)
-        super("The command #{command.inspect} is not available " +
-                "for the dataset #{dataset.inspect}.")
+    class UnknownType < ErrorMessages::BadRequest
+      def initialize(type, dataset)
+        super("[#{dataset}] Handler not found for the type: <#{type}>")
       end
     end
 
-    def initialize(options)
+    def initialize(catalog, options)
+      @catalog = catalog
       @options = options
       @name = @options[:name]
       @loop = EventLoop.new
-      @catalog_observer = CatalogObserver.new(@loop)
       @sessions = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
       @adapter_runners = create_adapter_runners
-      @farm = Farm.new(name, @loop, :dispatcher => self)
+      @farm = Farm.new(name, @catalog, @loop, :dispatcher => self)
       @forwarder = Forwarder.new(@loop)
       @replier = Replier.new(@forwarder)
-      # TODO: make customizable
-      @planner = Planner.new(self, ["search", "crud", "groonga", "watch"])
-      # TODO: make customizable
-      @collector = Collector.new(["basic", "search"])
+      @collector_runners = create_collector_runners
+      @step_runners = create_step_runners
     end
 
     def start
@@ -74,12 +72,15 @@ module Droonga
       @loop_thread = Thread.new do
         @loop.run
       end
+
+      ensure_schema
     end
 
     def shutdown
       @forwarder.shutdown
-      @planner.shutdown
-      @collector.shutdown
+      @collector_runners.each_value do |collector_runner|
+        collector_runner.shutdown
+      end
       @adapter_runners.each_value do |adapter_runner|
         adapter_runner.shutdown
       end
@@ -96,12 +97,12 @@ module Droonga
         begin
           assert_valid_message(message)
           process_input_message(message)
-        rescue MessageProcessingError => error
+        rescue ErrorMessage => error
           reply("statusCode" => error.status_code,
                 "body"       => error.response_body)
-        rescue => error
-          Logger.error("failed to process input message", error)
-          formatted_error = MessageProcessingError.new("Unknown internal error")
+        rescue StandardError, LoadError, SyntaxError => error
+          logger.exception("failed to process input message", error)
+          formatted_error = ErrorMessages::InternalServerError.new("Unknown internal error")
           reply("statusCode" => formatted_error.status_code,
                 "body"       => formatted_error.response_body)
           raise error
@@ -110,9 +111,9 @@ module Droonga
     end
 
     def forward(message, destination)
-      $log.trace("#{log_tag}: forward start")
+      logger.trace("forward start")
       @forwarder.forward(message, destination)
-      $log.trace("#{log_tag}: forward done")
+      logger.trace("forward done")
     end
 
     # Replies response to replyTo.
@@ -146,7 +147,9 @@ module Droonga
         steps = message["steps"]
         if steps
           session_planner = SessionPlanner.new(self, steps)
-          session = session_planner.create_session(id, @collector)
+          dataset = message["dataset"] || @message["dataset"]
+          collector_runner = @collector_runners[dataset]
+          session = session_planner.create_session(id, collector_runner)
           @sessions[id] = session
         else
           #todo: take cases receiving result before its query into account
@@ -172,7 +175,7 @@ module Droonga
       steps.each do |step|
         dataset = step["dataset"]
         if dataset
-          routes = Droonga.catalog.get_routes(dataset, step)
+          routes = @catalog.get_routes(dataset, step)
           step["routes"] = routes
         else
           step["routes"] ||= [id]
@@ -190,7 +193,7 @@ module Droonga
 
     def process_local_message(local_message)
       task = local_message["task"]
-      partition_name = task["route"]
+      slice_name = task["route"]
       step = task["step"]
       command = step["command"]
       descendants = {}
@@ -202,7 +205,7 @@ module Droonga
       local_message["descendants"] = descendants
       farm_message = @message.merge("body" => local_message,
                                     "type" => command)
-      @farm.process(partition_name, farm_message)
+      @farm.process(slice_name, farm_message)
     end
 
     def local?(route)
@@ -228,11 +231,13 @@ module Droonga
       dataset = message["dataset"]
       adapter_runner = @adapter_runners[dataset]
       adapted_message = adapter_runner.adapt_input(message)
-      plan = @planner.process(adapted_message["type"], adapted_message)
+      step_runner = @step_runners[dataset]
+      plan = step_runner.plan(message)
       distributor = Distributor.new(self)
       distributor.distribute(plan)
-    rescue Droonga::LegacyPluggable::UnknownPlugin => error
-      raise UnknownCommand.new(error.command, message["dataset"])
+    rescue Droonga::UnsupportedMessageError => error
+      target_message = error.message
+      raise UnknownType.new(target_message["type"], target_message["dataset"])
     end
 
     def assert_valid_message(message)
@@ -240,19 +245,47 @@ module Droonga
         raise MissingDatasetParameter.new
       end
       dataset = message["dataset"]
-      unless Droonga.catalog.have_dataset?(dataset)
+      unless @catalog.have_dataset?(dataset)
         raise UnknownDataset.new(dataset)
       end
     end
 
-    def create_adapter_runners
+    def create_runners
       runners = {}
-      Droonga.catalog.datasets.each do |name, configuration|
-        runners[name] = AdapterRunner.new(self, configuration["plugins"] || [])
+      @catalog.datasets.each do |name, configuration|
+        runners[name] = yield(configuration)
       end
       runners
     end
 
+    def create_adapter_runners
+      create_runners do |configuration|
+        AdapterRunner.new(self, configuration["plugins"] || [])
+      end
+    end
+
+    def create_collector_runners
+      create_runners do |configuration|
+        CollectorRunner.new(configuration["plugins"] || [])
+      end
+    end
+
+    def create_step_runners
+      create_runners do |configuration|
+        StepRunner.new(configuration["plugins"] || [])
+      end
+    end
+
+    def ensure_schema
+      @catalog.datasets.each do |name, dataset|
+        schema = dataset.schema
+        messages = schema.to_messages
+        messages.each do |message|
+          process_message(message)
+        end
+      end
+    end
+
     def log_tag
       "[#{Process.ppid}][#{Process.pid}] dispatcher"
     end
@@ -265,7 +298,7 @@ module Droonga
         @steps = steps
       end
 
-      def create_session(id, collector)
+      def create_session(id, collector_runner)
         resolve_descendants
         tasks = []
         inputs = {}
@@ -285,7 +318,7 @@ module Droonga
             end
           end
         end
-        Session.new(id, @dispatcher, collector, tasks, inputs)
+        Session.new(id, @dispatcher, collector_runner, tasks, inputs)
       end
 
       def resolve_descendants

data/lib/droonga/distributed_command_planner.rb

@@ -45,7 +45,7 @@ module Droonga
       return unless params
       params.each do |name, reducer|
         gatherer = nil
-        if reducer.is_a?(Hash) && reducer[:gather]
+        if reducer.is_a?(Hash) and reducer[:gather]
           gatherer = reducer[:gather]
           reducer = reducer[:reduce]
         end
@@ -127,7 +127,7 @@ module Droonga
     def fixed_processor
       @processor["outputs"] = @outputs
       if @processor["type"] == "scatter"
-        raise MessageProcessingError.new("missing key") unless @key
+        raise ErrorMessages::InternalServerError.new("missing key") unless @key
         @processor["key"] = @key
       end
       @processor

data/lib/droonga/engine.rb

@@ -15,14 +15,17 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/logger"
+require "droonga/loggable"
 require "droonga/dispatcher"
 
 module Droonga
   class Engine
-    def initialize(options={})
+    include Loggable
+
+    def initialize(catalog, options={})
+      @catalog = catalog
       @options = options
-      @dispatcher = Dispatcher.new(@options)
+      @dispatcher = Dispatcher.new(@catalog, @options)
     end
 
     def start
@@ -30,13 +33,18 @@ module Droonga
     end
 
     def shutdown
-      $log.trace("engine: shutdown: start")
+      logger.trace("shutdown: start")
       @dispatcher.shutdown
-      $log.trace("engine: shutdown: done")
+      logger.trace("shutdown: done")
     end
 
     def process(message)
       @dispatcher.process_message(message)
     end
+
+    private
+    def log_tag
+      "engine"
+    end
   end
 end

data/lib/droonga/{message_processing_error.rb → error.rb}

@@ -1,4 +1,6 @@
-# Copyright (C) 2013 Droonga Project
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,14 +16,31 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 module Droonga
-  class MessageProcessingError < StandardError
-    STATUS_CODE = 500
+  class Error < StandardError
+  end
+
+  class MultiplexError < Error
+    attr_reader :errors
+
+    def initialize(errors=[])
+      @errors = errors
+      error_messages = @errors.collect do |error|
+        error.message
+      end
+      message = error_messages.sort.join("\n-----------------------\n")
+      super(message)
+    end
+  end
 
-    attr_reader :message, :detail
+  # the base class for any error which can be described as a Droonga message
+  class ErrorMessage < Error
+    STATUS_CODE = nil
+
+    attr_reader :detail
 
     def initialize(message, detail=nil)
-      @message = message
       @detail = detail
+      super(message)
     end
 
     def name
@@ -35,18 +54,20 @@ module Droonga
     def response_body
       body = {
         "name"    => name,
-        "message" => @message,
+        "message" => message,
       }
       body["detail"] = @detail unless @detail.nil?
       body
     end
   end
 
-  class BadRequest < MessageProcessingError
-    STATUS_CODE = 400
-  end
-
-  class NotFound < MessageProcessingError
-    STATUS_CODE = 404
+  # TODO: Move to common file for runners
+  class UnsupportedMessageError < Error
+    attr_reader :phase, :message
+    def initialize(phase, message)
+      @phase = phase
+      @message = message
+      super("[#{@phase}] Unsupported message: #{@message.inspect}")
+    end
   end
 end