fluent-plugin-droonga 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16568393cd233c1f8a54d3bce227e328b6e1b293
-  data.tar.gz: 48f158466e6f15a069f3f5c8f92add90f740114c
+  metadata.gz: f03ac63db8e9b4b082da89c26c8bbf7caea45e76
+  data.tar.gz: 20b6f9a80dc7f198003fb4932b483f85c6a1d976
 SHA512:
-  metadata.gz: aa5bedb70c57916313b56a3639f8567f3a3efc3a43ba9b8bd68fae5d547e91c481ab71865fa1864d9b9cd425f54488acdc60c8cc53d652754b6552ccbe6e1f40
-  data.tar.gz: 3a3c6fa0f130808929a5225ea02b5a89bf5b5b850bc2a227cf9a35d54702bb2b63857d136672bc6849455e2de350f7de95c12bdb5dac4313a67785881ad63ef4
+  metadata.gz: 06af50c670901878e0d42f72182cd9848194eb5b14f04428fbe788098b7a7b715cb57e94241742620a0bf89fa7cbea03f7e8ea825a1e4816c3aedf59e2a19e33
+  data.tar.gz: 69cc3370c3401f24bf1749ef71f7fe2b45cf1de4b7c06c8d5171c3650e296d75ac8576851375ddefd3856c137fa5836a414e2dafef21c03e908b0296d1e5c464

data/.gitignore

@@ -1,7 +1,4 @@
 /Gemfile.lock
-/test/command/.bundle/
-/test/command/Gemfile.lock
-/test/command/bin/
-/test/command/vendor/
+/test/command/tmp
 /doc/
 /.yardoc/

data/benchmark/watch/benchmark-notify.rb

@@ -66,14 +66,14 @@ class NotifyBenchmark
       message = DroongaBenchmark::MessageCreator.envelope_to_subscribe(WATCHING_KEYWORD)
       message["body"]["subscriber"] += " #{@n_subscribers + index}"
       message["body"]["route"] = @route
-      @client.connection.send_receive(message)
+      @client.connection.send(message, :response => :one)
     end
     @n_subscribers += n_subscribers
   end
 
   def do_feed(target)
     message = DroongaBenchmark::MessageCreator.envelope_to_feed(target)
-    @client.connection.send(message)
+    @client.connection.send(message, :response => :none)
   end
 end
 

data/benchmark/watch/benchmark-scan.rb

@@ -19,12 +19,15 @@ require "benchmark"
 require "fileutils"
 require "optparse"
 require "csv"
+require "fluent/log"
 
 require "groonga"
 
 require "droonga/watcher"
 require File.expand_path(File.join(__FILE__, "..", "..", "utils.rb"))
 
+$log = Fluent::Log.new
+
 class ScanBenchmark
   attr_reader :n_keywords
 

data/benchmark/watch/fluentd.conf

@@ -5,7 +5,6 @@
 <match droonga.message>
   name localhost:23003/droonga
   type droonga
-  proxy true
 </match>
 <match output.message>
   type stdout

data/fluent-plugin-droonga.gemspec

@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "0.7.0"
+  gem.version       = "0.8.0"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
   gem.description   = "Droonga(distributed Groonga) plugin for Fluent event collector"
@@ -28,9 +28,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
   gem.add_dependency "fluentd"
-  gem.add_dependency "rroonga", ">= 3.0.3"
+  gem.add_dependency "rroonga", ">= 3.1.0"
   gem.add_dependency "groonga-command-parser"
-  gem.add_dependency "fluent-logger"
   gem.add_dependency "serverengine"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "bundler"

data/lib/droonga/catalog.rb

@@ -15,139 +15,25 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "digest/sha1"
-require "zlib"
+require "droonga/catalog_loader"
 
 module Droonga
   class << self
     def catalog
-      @catalog ||= Catalog.new
+      @catalog ||= Catalog.load
     end
   end
 
-  class Catalog
-    CATALOG_FILE_PATH = "catalog.json"
+  module Catalog
+    PATH = "catalog.json"
 
-    attr_reader :path
-
-    def initialize(path=nil)
-      @path = path || default_path
-
-      open(@path) do |file|
-        @catalog = JSON.parse(file.read)
-      end
-      @catalog["datasets"].each do |name, dataset|
-        number_of_partitions = dataset["number_of_partitions"]
-        next if number_of_partitions < 2
-        total_weight = dataset["ring"].reduce do |a, b|
-          a[1]["weight"] + b[1]["weight"]
-        end
-        continuum = []
-        dataset["ring"].each do |key, value|
-          points = number_of_partitions * 160 * value["weight"] / total_weight
-          points.times do |point|
-            hash = Digest::SHA1.hexdigest("#{key}:#{point}")
-            continuum << [hash[0..7].to_i(16), key]
-          end
-        end
-        dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
-      end
-      @options = @catalog["options"] || {}
-    end
-
-    def base_path
-      @base_path ||= File.dirname(@path)
-    end
-
-    def option(name)
-      @options[name]
-    end
-
-    def get_partitions(name)
-      device = @catalog["farms"][name]["device"]
-      pattern = Regexp.new("^#{name}\.")
-      results = {}
-      @catalog["datasets"].each do |key, dataset|
-        workers = dataset["workers"]
-        plugins = dataset["plugins"]
-        dataset["ring"].each do |key, part|
-          part["partitions"].each do |range, partitions|
-            partitions.each do |partition|
-              if partition =~ pattern
-                path = File.join([device, $POSTMATCH, "db"])
-                path = File.expand_path(path, base_path)
-                options = {
-                  :database => path,
-                  :n_workers => workers,
-                  :handlers => plugins
-                }
-                results[partition] = options
-              end
-            end
-          end
-        end
+    class << self
+      def load(path=nil)
+        path = ENV["DROONGA_CATALOG"] || PATH
+        path = File.expand_path(path)
+        loader = CatalogLoader.new(path)
+        loader.load
       end
-      return results
-    end
-
-    def get_routes(name, args)
-      routes = []
-      dataset = dataset(name)
-      return routes unless dataset
-      case args["type"]
-      when "broadcast"
-        dataset["ring"].each do |key, partition|
-          select_range_and_replicas(partition, args, routes)
-        end
-      when "scatter"
-        name = get_partition(dataset, args["key"])
-        partition = dataset["ring"][name]
-        select_range_and_replicas(partition, args, routes)
-      end
-      return routes
-    end
-
-    def get_partition(dataset, key)
-      continuum = dataset["continuum"]
-      return dataset["ring"].keys[0] unless continuum
-      hash = Zlib.crc32(key)
-      min = 0
-      max = continuum.size - 1
-      while (min < max) do
-        index = (min + max) / 2
-        value, key = continuum[index]
-        return key if value == hash
-        if value > hash
-          max = index
-        else
-          min = index + 1
-        end
-      end
-      return continuum[max][1]
-    end
-
-    def dataset(name)
-      @catalog["datasets"][name]
-    end
-
-    def select_range_and_replicas(partition, args, routes)
-      date_range = args["date_range"] || 0..-1
-      partition["partitions"].sort[date_range].each do |time, replicas|
-        case args["replica"]
-        when "top"
-          routes << replicas[0]
-        when "random"
-          routes << replicas[rand(replicas.size)]
-        when "all"
-          routes.concat(replicas)
-        end
-      end
-    end
-
-    private
-    def default_path
-      path = ENV["DROONGA_CATALOG"] || Catalog::CATALOG_FILE_PATH
-      File.expand_path(path)
     end
   end
 end

data/lib/droonga/catalog/base.rb

@@ -0,0 +1,140 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "digest/sha1"
+require "zlib"
+require "droonga/message_processing_error"
+
+module Droonga
+  module Catalog
+    class Base
+      class UnknownDataset < NotFound
+        def initialize(dataset)
+          super("The dataset #{dataset.inspect} does not exist.")
+        end
+      end
+
+      attr_reader :base_path
+      def initialize(data, base_path)
+        @data = data
+        @base_path = base_path
+
+        @data["datasets"].each do |name, dataset|
+          number_of_partitions = dataset["number_of_partitions"]
+          next if number_of_partitions < 2
+          total_weight = dataset["ring"].reduce do |a, b|
+            a[1]["weight"] + b[1]["weight"]
+          end
+          continuum = []
+          dataset["ring"].each do |key, value|
+            points = number_of_partitions * 160 * value["weight"] / total_weight
+            points.times do |point|
+              hash = Digest::SHA1.hexdigest("#{key}:#{point}")
+              continuum << [hash[0..7].to_i(16), key]
+            end
+          end
+          dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
+        end
+        @options = @data["options"] || {}
+      end
+
+      def option(name)
+        @options[name]
+      end
+
+      def get_partitions(name)
+        device = @data["farms"][name]["device"]
+        pattern = Regexp.new("^#{name}\.")
+        results = {}
+        @data["datasets"].each do |key, dataset|
+          workers = dataset["workers"]
+          plugins = dataset["plugins"]
+          dataset["ring"].each do |key, part|
+            part["partitions"].each do |range, partitions|
+              partitions.each do |partition|
+                if partition =~ pattern
+                  path = File.join([device, $POSTMATCH, "db"])
+                  path = File.expand_path(path, base_path)
+                  options = {
+                    :database => path,
+                    :n_workers => workers,
+                    :handlers => plugins
+                  }
+                  results[partition] = options
+                end
+              end
+            end
+          end
+        end
+        return results
+      end
+
+      def get_routes(name, args)
+        routes = []
+        dataset = dataset(name)
+        case args["type"]
+        when "broadcast"
+          dataset["ring"].each do |key, partition|
+            select_range_and_replicas(partition, args, routes)
+          end
+        when "scatter"
+          name = get_partition(dataset, args["key"])
+          partition = dataset["ring"][name]
+          select_range_and_replicas(partition, args, routes)
+        end
+        return routes
+      end
+
+      def get_partition(dataset, key)
+        continuum = dataset["continuum"]
+        return dataset["ring"].keys[0] unless continuum
+        hash = Zlib.crc32(key)
+        min = 0
+        max = continuum.size - 1
+        while (min < max) do
+          index = (min + max) / 2
+          value, key = continuum[index]
+          return key if value == hash
+          if value > hash
+            max = index
+          else
+            min = index + 1
+          end
+        end
+        return continuum[max][1]
+      end
+
+      def dataset(name)
+        dataset = @data["datasets"][name]
+        raise UnknownDataset.new(name) unless dataset
+        dataset
+      end
+
+      def select_range_and_replicas(partition, args, routes)
+        date_range = args["date_range"] || 0..-1
+        partition["partitions"].sort[date_range].each do |time, replicas|
+          case args["replica"]
+          when "top"
+            routes << replicas[0]
+          when "random"
+            routes << replicas[rand(replicas.size)]
+          when "all"
+            routes.concat(replicas)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/version1.rb

@@ -0,0 +1,23 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/base"
+
+module Droonga
+  module Catalog
+    class Version1 < Base
+    end
+  end
+end

data/lib/droonga/catalog_loader.rb

@@ -0,0 +1,33 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "json"
+
+require "droonga/catalog/version1"
+
+module Droonga
+  class CatalogLoader
+    def initialize(path)
+      @path = path
+    end
+
+    def load
+      data = File.open(@path) do |file|
+        JSON.parse(file.read)
+      end
+      Catalog::Version1.new(data, File.dirname(@path))
+    end
+  end
+end

data/lib/droonga/collector.rb

@@ -22,82 +22,13 @@ module Droonga
   class Collector
     include Pluggable
 
-    def initialize(id, dispatcher, components, tasks, inputs)
-      @id = id
-      @dispatcher = dispatcher
-      @components = components
-      @tasks = tasks
-      @n_dones = 0
-      @inputs = inputs
+    def initialize
       load_plugins(["basic"]) # TODO: make customizable
     end
 
-    def handle(name, value)
-      tasks = @inputs[name]
-      unless tasks
-        #TODO: result arrived before its query
-        return
-      end
-      tasks.each do |task|
-        task["n_of_inputs"] += 1 if name
-        component = task["component"]
-        type = component["type"]
-        command = component["command"] || ("collector_" + type)
-        n_of_expects = component["n_of_expects"]
-        synchronous = nil
-        if command
-          synchronous = true unless n_of_expects.zero?
-          # TODO: check if asynchronous execution is available.
-          message = {
-            "task"=>task,
-            "name"=>name,
-            "value"=>value
-          }
-          unless synchronous
-            descendants = {}
-            component["descendants"].each do |name, indices|
-              descendants[name] = indices.collect do |index|
-                @components[index]["routes"].map do |route|
-                  @dispatcher.farm_path(route)
-                end
-              end
-            end
-            message["descendants"] = descendants
-            message["id"] = @id
-          end
-          if @id == task["route"]
-            process(command, message)
-          else
-            @dispatcher.deliver(@id, task["route"], message, command, synchronous)
-          end
-        end
-        return if task["n_of_inputs"] < n_of_expects
-        #the task is done
-        if synchronous
-          result = task["values"]
-          post = component["post"]
-          @dispatcher.post(result, post) if post
-          component["descendants"].each do |name, indices|
-            message = {
-              "id" => @id,
-              "input" => name,
-              "value" => result[name]
-            }
-            indices.each do |index|
-              @components[index]["routes"].each do |route|
-                @dispatcher.dispatch(message, route)
-              end
-            end
-          end
-        end
-        @n_dones += 1
-        @dispatcher.collectors.delete(@id) if @n_dones == @tasks.size
-      end
-    end
-
     private
     def instantiate_plugin(name)
-      CollectorPlugin.repository.instantiate(name, @dispatcher)
+      CollectorPlugin.repository.instantiate(name)
     end
 
     def log_tag