fluent-plugin-droonga 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f03ac63db8e9b4b082da89c26c8bbf7caea45e76
-  data.tar.gz: 20b6f9a80dc7f198003fb4932b483f85c6a1d976
+  metadata.gz: 80594c4e021918c28b58451b90293961f7b65174
+  data.tar.gz: ff89eb342312841a7d9eaa8831a7d190bec34c35
 SHA512:
-  metadata.gz: 06af50c670901878e0d42f72182cd9848194eb5b14f04428fbe788098b7a7b715cb57e94241742620a0bf89fa7cbea03f7e8ea825a1e4816c3aedf59e2a19e33
-  data.tar.gz: 69cc3370c3401f24bf1749ef71f7fe2b45cf1de4b7c06c8d5171c3650e296d75ac8576851375ddefd3856c137fa5836a414e2dafef21c03e908b0296d1e5c464
+  metadata.gz: 39351f8baffeae2e33ac2b3d1ad6f06c39fe08a626ee8f260c62501e03b5acb05a67f2e0667833250f200679664d518da774b57c2356c30f260cd0afb83f8940
+  data.tar.gz: c26dd8db11b63e7f6130826f6eb4ce97b05dae064e39b9502b3cc7cbf0f54a912c5fc3885bab2855ace0a21cba9af851e64e9a4d40c136310790c848d9e0aadd

data/.gitignore

@@ -2,3 +2,4 @@
 /test/command/tmp
 /doc/
 /.yardoc/
+/pkg/

data/Gemfile

@@ -50,3 +50,10 @@ if File.exist?(drntest_dir)
 else
   gem "drntest", :github => "droonga/drntest"
 end
+
+drnbench_dir = File.join(parent_dir, "drnbench")
+if File.exist?(drnbench_dir)
+  gem "drnbench", :path => drnbench_dir
+else
+  gem "drnbench", :github => "droonga/drnbench"
+end

data/benchmark/watch/benchmark-notify.rb

@@ -54,11 +54,11 @@ class NotifyBenchmark
       do_feed("#{WATCHING_KEYWORD} #{index}")
     end
 
-    notifications = []
-    while notifications.size != @n_times
-      notifications << @receiver.new_message
+    published_messages = []
+    while published_messages.size != @n_times
+      published_messages << @receiver.new_message
     end
-    notifications
+    published_messages
   end
 
   def add_subscribers(n_subscribers)
@@ -118,8 +118,8 @@ options[:n_steps].times do |try_count|
   percentage = nil
   result = Benchmark.bm do |benchmark|
     benchmark.report(label) do
-      sent_notifications = notify_benchmark.run
-      percentage = sent_notifications.size.to_f / options[:n_times] * 100
+      published_messages = notify_benchmark.run
+      percentage = published_messages.size.to_f / options[:n_times] * 100
     end
   end
   puts "=> #{percentage} % feeds are notified"

data/benchmark/watch/benchmark-notify.sh

@@ -1,13 +1,14 @@
 #!/bin/sh
 
 base_dir=$(cd $(dirname $0); pwd)
+work_dir=$base_dir/../../performance/watch
 
 rm -rf $base_dir/watch
 mkdir -p $base_dir/watch
 
-DROONGA_CATALOG=$base_dir/catalog.json \
+DROONGA_CATALOG=$work_dir/catalog.json \
   bundle exec fluentd \
-    --config $base_dir/fluentd.conf &
+    --config $work_dir/fluentd.conf &
 FLUENTD_PID=$!
 
 sleep 1

data/bin/grn2jsons

@@ -45,9 +45,6 @@ option_parser = OptionParser.new do |parser|
 end
 args = option_parser.parse!(ARGV)
 
-if options.reply_to.nil?
-  raise "You must specify the value of the \"replyTo\" field by --reply-to option."
-end
 if options.dataset.nil?
   raise "You must specify the name of the dataset by --dataset option."
 end

data/doc/text/news.md

@@ -1,5 +1,18 @@
 # News
 
+## 0.9.0: 2014-01-29
+
+### Improvements
+
+  * `search`: Supported `"attributes"` for `elements` of `output`.
+  * `table_remove`: Implemented Groonga compatible `table_remove`
+    command.
+  * `column_create`: Implemented error handling.
+  * `catalog`: Supported auto reloading.
+  * Supported reducing responses from two or more nodes for Groonga
+    compatible commands.
+  * Supported three or more partitions.
+
 ## 0.8.0: 2013-12-29
 
 ### Improvements

data/fluent-plugin-droonga.gemspec

@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "0.8.0"
+  gem.version       = "0.9.0"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
   gem.description   = "Droonga(distributed Groonga) plugin for Fluent event collector"

data/lib/droonga/catalog/base.rb

@@ -34,9 +34,7 @@ module Droonga
         @data["datasets"].each do |name, dataset|
           number_of_partitions = dataset["number_of_partitions"]
           next if number_of_partitions < 2
-          total_weight = dataset["ring"].reduce do |a, b|
-            a[1]["weight"] + b[1]["weight"]
-          end
+          total_weight = compute_total_weight(dataset)
           continuum = []
           dataset["ring"].each do |key, value|
             points = number_of_partitions * 160 * value["weight"] / total_weight
@@ -135,6 +133,13 @@ module Droonga
           end
         end
       end
+
+      private
+      def compute_total_weight(dataset)
+        dataset["ring"].reduce(0) do |result, zone|
+          result + zone[1]["weight"]
+        end
+      end
     end
   end
 end

data/lib/droonga/catalog.rb

@@ -19,21 +19,6 @@ require "droonga/catalog_loader"
 
 module Droonga
   class << self
-    def catalog
-      @catalog ||= Catalog.load
-    end
-  end
-
-  module Catalog
-    PATH = "catalog.json"
-
-    class << self
-      def load(path=nil)
-        path = ENV["DROONGA_CATALOG"] || PATH
-        path = File.expand_path(path)
-        loader = CatalogLoader.new(path)
-        loader.load
-      end
-    end
+    attr_accessor :catalog
   end
 end

data/lib/droonga/catalog_observer.rb

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class CatalogObserver
+    DEFAULT_CATALOG_PATH = "catalog.json"
+    CHECK_INTERVAL = 1
+
+    def initialize(loop)
+      @catalog_path = catalog_path
+      load_catalog
+
+      watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
+      observer = self
+      watcher.on_timer do
+        observer.ensure_latest_catalog_loaded
+      end
+      loop.attach(watcher)
+    end
+
+    def ensure_latest_catalog_loaded
+      if catalog_updated?
+        load_catalog
+      end
+    end
+
+    def catalog_path
+      path = ENV["DROONGA_CATALOG"] || DEFAULT_CATALOG_PATH
+      File.expand_path(path)
+    end
+
+    def catalog_updated?
+      File.mtime(catalog_path) > @catalog_mtime
+    end
+
+    def load_catalog
+      loader = CatalogLoader.new(@catalog_path)
+      Droonga.catalog = loader.load
+      @catalog_mtime = File.mtime(@catalog_path)
+      $log.info "catalog loaded", path: @catalog_path, mtime: @catalog_mtime
+    end
+  end
+end

data/lib/droonga/collector.rb

@@ -23,7 +23,7 @@ module Droonga
     include Pluggable
 
     def initialize
-      load_plugins(["basic"]) # TODO: make customizable
+      load_plugins(["basic", "search", "groonga"]) # TODO: make customizable
     end
 
     private

data/lib/droonga/dispatcher.rb

@@ -27,6 +27,7 @@ require "droonga/farm"
 require "droonga/session"
 require "droonga/replier"
 require "droonga/message_processing_error"
+require "droonga/catalog_observer"
 
 module Droonga
   class Dispatcher
@@ -48,6 +49,8 @@ module Droonga
     def initialize(options)
       @options = options
       @name = @options[:name]
+      @loop = EventLoop.new
+      @catalog_observer = CatalogObserver.new(@loop)
       @sessions = {}
       @current_id = 0
       @local = Regexp.new("^#{@name}")
@@ -55,7 +58,6 @@ module Droonga
         InputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
       @output_adapter =
         OutputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
-      @loop = EventLoop.new
       @farm = Farm.new(name, @loop, :dispatcher => self)
       @forwarder = Forwarder.new(@loop)
       @replier = Replier.new(@forwarder)

data/lib/droonga/distributor_plugin.rb

@@ -31,28 +31,13 @@ module Droonga
     end
 
     def scatter_all(message, key)
-      distribute_message = [{
-        "command"=> message["type"],
-        "dataset"=> message["dataset"],
-        "body"=> message["body"],
-        "key"=> key,
-        "type"=> "scatter",
-        "replica"=> "all",
-        "post"=> true
-      }]
-      distribute(distribute_message)
+      messages = [reducer(message), gatherer(message), scatterer(message, key)]
+      distribute(messages)
     end
 
     def broadcast_all(message)
-      distribute_message = [{
-        "command"=> message["type"],
-        "dataset"=> message["dataset"],
-        "body"=> message["body"],
-        "type"=> "broadcast",
-        "replica"=> "all",
-        "post"=> true
-      }]
-      distribute(distribute_message)
+      messages = [reducer(message), gatherer(message), broadcaster(message)]
+      distribute(messages)
     end
 
     private
@@ -63,5 +48,48 @@ module Droonga
         super
       end
     end
+
+    def scatterer(message, key)
+      {
+        "command" => message["type"],
+        "dataset" => message["dataset"],
+        "body"    => message["body"],
+        "key"     => key,
+        "type"    => "scatter",
+        "outputs" => [],
+        "replica" => "all",
+        "post"    => true
+      }
+    end
+
+    def broadcaster(message)
+      {
+        "command" => message["type"],
+        "dataset" => message["dataset"],
+        "body"    => message["body"],
+        "type"    => "broadcast",
+        "outputs" => [],
+        "replica" => "all",
+        "post"    => true
+      }
+    end
+
+    def reducer(message)
+      {
+        "type"    => "reduce",
+        "body"    => {},
+        "inputs"  => [],
+        "outputs" => [],
+      }
+    end
+
+    def gatherer(message)
+      {
+        "type"   => "gather",
+        "body"   => {},
+        "inputs" => [],
+        "post"   => true,
+      }
+    end
   end
 end

data/lib/droonga/handler_messenger.rb

@@ -18,12 +18,15 @@ require "droonga/forwarder"
 
 module Droonga
   class HandlerMessenger
+    attr_reader :database_name
+
     def initialize(forwarder, message, options={})
       @forwarder = forwarder
       @message = message
       @options = options
       @replier = Replier.new(@forwarder)
       @dispatcher = @options[:dispatcher]
+      @database_name = options[:database]
     end
 
     def emit(value)
@@ -60,8 +63,29 @@ module Droonga
                                      "body" => body)
         @replier.reply(response)
       else
-        #XXX IMPLEMENT ME!!
-        raise error
+        body = {
+          "id"    => @message.id,
+          "input" => "errors",
+          "value" => {
+            database_name => {
+              "statusCode" => status_code,
+              "body" => body,
+            },
+          },
+        }
+        all_dests = []
+        descendants.each do |name, dests|
+          all_dests += dests
+        end
+        all_dests.each do |dest|
+          if @dispatcher
+            @dispatcher.dispatch(body, dest)
+          else
+            message = raw_message.merge("statusCode" => status_code,
+                                        "body" => body,)
+            forward(message, "to" => dest, "type" => "dispatcher")
+          end
+        end
       end
     end
 

data/lib/droonga/message_processing_error.rb

@@ -15,8 +15,10 @@
 
 module Droonga
   class MessageProcessingError < StandardError
+    STATUS_CODE = 500
+
     attr_reader :message, :detail
- 
+
     def initialize(message, detail=nil)
       @message = message
       @detail = detail
@@ -27,7 +29,7 @@ module Droonga
     end
 
     def status_code
-      500
+      self.class::STATUS_CODE
     end
 
     def response_body
@@ -41,14 +43,10 @@ module Droonga
   end
 
   class BadRequest < MessageProcessingError
-    def status_code
-      400
-    end
+    STATUS_CODE = 400
   end
 
   class NotFound < MessageProcessingError
-    def status_code
-      404
-    end
+    STATUS_CODE = 404
   end
 end

data/lib/droonga/output_message.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -26,6 +26,22 @@ module Droonga
       @raw_message
     end
 
+    def status_code
+      @raw_message["statusCode"]
+    end
+
+    def status_code=(status_code)
+      @raw_message["statusCode"] = status_code
+    end
+
+    def errors
+      @raw_message["errors"]
+    end
+
+    def errors=(errors)
+      @raw_message["errors"] = errors
+    end
+
     def body
       @raw_message["body"]
     end

data/lib/droonga/plugin/collector/basic.rb

@@ -27,42 +27,49 @@ module Droonga
     def collector_gather(result)
       output = body ? body[input_name] : input_name
       if output.is_a?(Hash)
-        elements = output["elements"]
-        if elements && elements.is_a?(Hash)
-          # phase 1: pre-process
-          elements.each do |element, mapper|
-            case mapper["type"]
-            when "count"
-              result[element] = result[mapper["target"]].size
-            when "sort"
-              # do nothing on this phase!
-            end
-          end
-          # phase 2: post-process
-          elements.each do |element, mapper|
-            if mapper["no_output"]
-              result.delete(element)
-              next
-            end
-
-            case mapper["type"]
-            when "count"
-              # do nothing on this phase!
-            when "sort"
-              # because "count" type mapper requires all items of the array,
-              # I have to apply "sort" type mapper later.
-              if result[element]
-                result[element] = apply_output_range(result[element], mapper)
-                result[element] = apply_output_attributes_and_format(result[element], mapper)
-              end
-            end
-          end
-        end
         output = output["output"]
       end
       emit(output, result)
     end
 
+    command :collector_reduce
+    def collector_reduce(request)
+      body[input_name].each do |output, deal|
+        left_value = output_values[output]
+        right_value = request
+        value = reduce(deal, left_value, right_value)
+        emit(output, value)
+      end
+    end
+
+    def reduce(deal, left_value, right_value)
+      if left_value.nil? || right_value.nil?
+        return right_value || left_value
+      end
+
+      reduced_value = nil
+
+      case deal["type"]
+      when "and"
+        reduced_value = left_value && right_value
+      when "sum"
+        reduced_value = sum(left_value, right_value)
+        reduced_value = apply_output_range(reduced_value,
+                                           "limit" => deal["limit"])
+      when "average"
+        reduced_value = (left_value.to_f + right_value.to_f) / 2
+      when "sort"
+        reduced_value = merge(left_value,
+                              right_value,
+                              :operators => deal["operators"],
+                              :key_column => deal["key_column"])
+        reduced_value = apply_output_range(reduced_value,
+                                           "limit" => deal["limit"])
+      end
+
+      reduced_value
+    end
+
     def apply_output_range(items, output)
       if items && items.is_a?(Array)
         offset = output["offset"] || 0
@@ -78,58 +85,14 @@ module Droonga
       items
     end
 
-    def apply_output_attributes_and_format(items, output)
-      attributes = output["attributes"]
-      if attributes
-        format = output["format"]
-        if format == "complex"
-          items.collect! do |item|
-            complex_item = {}
-            attributes.each_with_index do |label, index|
-              complex_item[label] = item[index]
-            end
-            complex_item
-          end
-        else
-          items.collect! do |item|
-            item[0...attributes.size]
-          end
-        end
-      end
-      items
-    end
-
-    command :collector_reduce
-    def collector_reduce(request)
-      return unless request
-      body[input_name].each do |output, elements|
-        value = request
-        old_value = output_values[output]
-        value = reduce(elements, old_value, request) if old_value
-        emit(output, value)
-      end
-    end
-
-    def reduce(elements, *values)
-      result = {}
-      elements.each do |key, deal|
-        reduced_values = nil
-
-        case deal["type"]
-        when "sum"
-          reduced_values = values[0][key] + values[1][key]
-        when "sort"
-          reduced_values = merge(values[0][key],
-                                 values[1][key],
-                                 :operators => deal["operators"],
-                                 :key_column => deal["key_column"])
-        end
-
-        reduced_values = apply_output_range(reduced_values, "limit" => deal["limit"])
+    def sum(x, y)
+      return x || y if x.nil? or y.nil?
 
-        result[key] = reduced_values
+      if x.is_a?(Hash) && y.is_a?(Hash)
+        x.merge(y)
+      else
+        x + y
       end
-      return result
     end
 
     def merge(x, y, options={})

data/lib/droonga/plugin/collector/groonga.rb

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin/collector/basic"
+
+module Droonga
+  class GroongaCollector < BasicCollector
+    repository.register("groonga", self)
+
+    command :collector_groonga_gather
+    def collector_groonga_gather(result)
+      collector_gather(result)
+    end
+
+    command :collector_groonga_reduce
+    def collector_groonga_reduce(request)
+      collector_reduce(request)
+    end
+
+    def reduce(deal, left_value, right_value)
+      reduced_value = nil
+
+      case deal["type"]
+      when "groonga_result"
+        #XXX how to merge multiple erros?
+        #XXX how to mix regular results and erros?
+        # reduced_value = merge_groonga_result(left_value, right_value)
+        reduced_value = left_value || right_value
+      else
+        reduced_value = super
+      end
+
+      reduced_value
+    end
+
+    def merge_groonga_result(left_value, right_value)
+      result = []
+
+      result << merge_groonga_header(left_value.shift, right_value.shift)
+
+      left_value.each_with_index do |left, index|
+        right = right_value[index]
+        result << reduce({ "type" => "and" }, left, right)
+      end
+
+      result
+    end
+
+    def merge_groonga_header(left_header, right_header)
+      status = [left_header.shift, right_header.shift].min
+
+      start_time = reduce({ "type" => "average" },
+                          left_header.shift,
+                          right_header.shift)
+
+      elapsed_time = reduce({ "type" => "average" },
+                            left_header.shift,
+                            right_header.shift)
+
+      #XXX we should merge error informations more smarter...
+      error_information = reduce({ "type" => "sum",
+                                   "limit" => UNLIMITED },
+                                 left_header,
+                                 right_header)
+
+      [status, start_time, elapsed_time] + error_information
+    end
+  end
+end