fluent-plugin-droonga 1.0.0 → 1.0.1

data/lib/droonga/catalog/version2_validator.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/errors"
+
+module Droonga
+  module Catalog
+    class Version2Validator
+      def initialize(data, path)
+        @data = data
+        @path = path
+      end
+
+      def validate
+        @details = []
+
+        validate_datasets
+
+        unless @details.empty?
+          raise ValidationError.new(@path, @details)
+        end
+      end
+
+      private
+      def validate_datasets
+        unless @data.key?("datasets")
+          required_parameter_is_missing("datasets")
+          return
+        end
+        @data["datasets"].each do |name, dataset|
+          validate_dataset(name, dataset)
+        end
+      end
+
+      def validate_dataset(name, dataset)
+        unless dataset.key?("replicas")
+          required_parameter_is_missing("datasets.#{name}.replicas")
+          return
+        end
+      end
+
+      def add_detail(value_path, message)
+        @details << ValidationError::Detail.new(value_path, message)
+      end
+
+      def required_parameter_is_missing(value_path)
+        add_detail(value_path, "required parameter is missing")
+      end
+    end
+  end
+end

data/lib/droonga/catalog/volume.rb

@@ -0,0 +1,33 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/single_volume"
+require "droonga/catalog/collection_volume"
+
+module Droonga
+  module Catalog
+    module Volume
+      class << self
+        def create(dataset, raw_volume)
+          if raw_volume.key?("address")
+            SingleVolume.new(raw_volume)
+          else
+            CollectionVolume.new(dataset, raw_volume)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/volume_collection.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Catalog
+    class VolumeCollection
+      include Enumerable
+
+      def initialize(volumes)
+        @volumes = volumes
+      end
+
+      def each(&block)
+        @volumes.each(&block)
+      end
+
+      def ==(other)
+        other.is_a?(self.class) and
+          to_a == other.to_a
+      end
+
+      def eql?(other)
+        self == other
+      end
+
+      def hash
+        to_a.hash
+      end
+
+      def select(how=nil)
+        case how
+        when :top
+          [@volumes.first]
+        when :random
+          [@volumes.sample]
+        when :all
+          @volumes
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog_observer.rb

@@ -34,29 +34,17 @@ module Droonga
     end
 
     def start
-      @loop = Cool.io::Loop.new
-      attach(@loop)
-      @thread = Thread.new do
-        begin
-          @loop.run
-        rescue => error
-          logger.exception("error in catalog observing thread", error)
-        end
+      @watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
+      observer = self
+      @watcher.on_timer do
+        observer.ensure_latest_catalog_loaded
       end
+      loop = Coolio::Loop.default
+      @watcher.attach(loop)
     end
 
     def stop
-      @loop.stop
-      @thread.join
-    end
-
-    def attach(loop)
-      watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
-      observer = self
-      watcher.on_timer do
-        observer.ensure_latest_catalog_loaded
-      end
-      loop.attach(watcher)
+      @watcher.detach
     end
 
     def ensure_latest_catalog_loaded

data/lib/droonga/collectors.rb

@@ -13,6 +13,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/collectors/add"
 require "droonga/collectors/and"
+require "droonga/collectors/or"
 require "droonga/collectors/sum"

data/lib/droonga/collectors/{add.rb → or.rb}

@@ -15,7 +15,7 @@
 
 module Droonga
   module Collectors
-    class Add
+    class Or
       class << self
         def operator
           "or"

data/lib/droonga/dispatcher.rb

@@ -54,7 +54,7 @@ module Droonga
       @catalog = catalog
       @options = options
       @name = @options[:name]
-      @loop = EventLoop.new
+      @loop = EventLoop.new(Coolio::Loop.default)
       @sessions = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
@@ -69,9 +69,6 @@ module Droonga
     def start
       @forwarder.start
       @farm.start
-      @loop_thread = Thread.new do
-        @loop.run
-      end
 
       ensure_schema
     end
@@ -85,8 +82,6 @@ module Droonga
         adapter_runner.shutdown
       end
       @farm.shutdown
-      @loop.stop
-      @loop_thread.join
     end
 
     def process_message(message)
@@ -134,8 +129,19 @@ module Droonga
       if adapter_runner
         adapted_message = adapter_runner.adapt_output(adapted_message)
       end
-      return if adapted_message["replyTo"].nil?
-      @replier.reply(adapted_message)
+      if adapted_message["replyTo"].nil?
+        status_code = adapted_message["statusCode"] || 200
+        if status_code != 200
+          dataset = adapted_message["dataset"]
+          body = adapted_message["body"] || {}
+          name = body["name"] || "Unknown"
+          message = body["message"] || "unknown error"
+          logger.error("orphan error: " +
+                         "<#{dataset}>[#{name}](#{status_code}): #{message}")
+        end
+      else
+        @replier.reply(adapted_message)
+      end
     end
 
     def process_internal_message(message)
@@ -233,8 +239,8 @@ module Droonga
       adapted_message = adapter_runner.adapt_input(message)
       step_runner = @step_runners[dataset]
       plan = step_runner.plan(message)
-      distributor = Distributor.new(self)
-      distributor.distribute(plan)
+      distributor = Distributor.new(self, plan)
+      distributor.distribute
     rescue Droonga::UnsupportedMessageError => error
       target_message = error.message
       raise UnknownType.new(target_message["type"], target_message["dataset"])
@@ -252,27 +258,27 @@ module Droonga
 
     def create_runners
       runners = {}
-      @catalog.datasets.each do |name, configuration|
-        runners[name] = yield(configuration)
+      @catalog.datasets.each do |name, dataset|
+        runners[name] = yield(dataset)
       end
       runners
     end
 
     def create_adapter_runners
-      create_runners do |configuration|
-        AdapterRunner.new(self, configuration["plugins"] || [])
+      create_runners do |dataset|
+        AdapterRunner.new(self, dataset.plugins)
       end
     end
 
     def create_collector_runners
-      create_runners do |configuration|
-        CollectorRunner.new(configuration["plugins"] || [])
+      create_runners do |dataset|
+        CollectorRunner.new(dataset.plugins)
       end
     end
 
     def create_step_runners
-      create_runners do |configuration|
-        StepRunner.new(configuration["plugins"] || [])
+      create_runners do |dataset|
+        StepRunner.new(dataset, dataset.plugins)
       end
     end
 

data/lib/droonga/distributed_command_planner.rb

@@ -55,12 +55,12 @@ module Droonga
       end
     end
 
-    def scatter(options={})
+    def scatter(record, options={})
       @processor = {
         "command" => @source_message["type"],
         "dataset" => @dataset || @source_message["dataset"],
         "body"    => options[:body] || @source_message["body"],
-        "key"     => nil,
+        "record"  => record,
         "type"    => "scatter",
         "outputs" => [],
         "replica" => "all",
@@ -100,10 +100,10 @@ module Droonga
         unified = unified_reducers[type]
         if unified
           unified["body"] = unified["body"].merge(reducer["body"])
-          unified["inputs"] = unified["inputs"] + reducer["inputs"]
-          unified["outputs"] = unified["outputs"] + reducer["outputs"]
+          unified["inputs"] += reducer["inputs"]
+          unified["outputs"] += reducer["outputs"]
         else
-          unified_reducers[type] = Marshal.load(Marshal.dump(reducer))
+          unified_reducers[type] = reducer.dup
         end
       end
       unified_reducers.values
@@ -116,9 +116,9 @@ module Droonga
         unified = unified_gatherers[type]
         if unified
           unified["body"] = unified["body"].merge(gatherer["body"])
-          unified["inputs"] = unified["inputs"] + gatherer["inputs"]
+          unified["inputs"] += gatherer["inputs"]
         else
-          unified_gatherers[type] = Marshal.load(Marshal.dump(gatherer))
+          unified_gatherers[type] = gatherer.dup
         end
       end
       unified_gatherers.values
@@ -126,10 +126,6 @@ module Droonga
 
     def fixed_processor
       @processor["outputs"] = @outputs
-      if @processor["type"] == "scatter"
-        raise ErrorMessages::InternalServerError.new("missing key") unless @key
-        @processor["key"] = @key
-      end
       @processor
     end
 

data/lib/droonga/distributor.rb

@@ -37,38 +37,50 @@ module Droonga
 
     include TSort
 
-    def initialize(dispatcher)
+    def initialize(dispatcher, plan)
       @dispatcher = dispatcher
+      @plan = plan
+      build_dependencies
     end
 
-    def distribute(plan)
-      @dependency = {}
-      plan.each do |step|
-        @dependency[step] = step["inputs"]
-        next unless step["outputs"]
-        step["outputs"].each do |output|
-          @dependency[output] = [step]
-        end
-      end
+    def distribute
       steps = []
-      each_strongly_connected_component do |cs|
-        raise CyclicStepsError.new(cs) if cs.size > 1
-        steps.concat(cs) unless cs.first.is_a? String
+      each_strongly_connected_component do |nodes|
+        raise CyclicStepsError.new(nodes) if nodes.size > 1
+        nodes.each do |node|
+          steps << @step_maps[node] if node.is_a?(Integer)
+        end
       end
       @dispatcher.dispatch_steps(steps)
     end
 
     private
+    def build_dependencies
+      @dependencies = {}
+      @step_maps = {}
+      step_id = 0
+      @plan.each do |step|
+        step_id += 1
+        # Integer#hash (step_id.hash) is very faster than Hash#hash (step.hash).
+        @step_maps[step_id] = step
+        @dependencies[step_id] = step["inputs"]
+        next unless step["outputs"]
+        step["outputs"].each do |output|
+          @dependencies[output] = [step_id]
+        end
+      end
+    end
+
     def tsort_each_node(&block)
-      @dependency.each_key(&block)
+      @dependencies.each_key(&block)
     end
 
     def tsort_each_child(node, &block)
-      if node.is_a? String and @dependency[node].nil?
+      if node.is_a? String and @dependencies[node].nil?
         raise UndefinedInputError.new(node)
       end
-      if @dependency[node]
-        @dependency[node].each(&block)
+      if @dependencies[node]
+        @dependencies[node].each(&block)
       end
     end
   end

data/lib/droonga/event_loop.rb

@@ -19,21 +19,12 @@ require "coolio"
 
 module Droonga
   class EventLoop
-    def initialize
-      @loop = Coolio::Loop.new
+    def initialize(loop)
+      @loop = loop
       @loop_breaker = Coolio::AsyncWatcher.new
       @loop_breaker.attach(@loop)
     end
 
-    def run
-      @loop.run
-    end
-
-    def stop
-      @loop.stop
-      break_current_loop
-    end
-
     def attach(watcher)
       @loop.attach(watcher)
       break_current_loop