fluent-plugin-droonga 0.9.9 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24503ddfa77a2b4c19b83b771069a3c91f738ef2
-  data.tar.gz: 50f89c1a0fed25c58d8d16b5e16c41258cbd4e3a
+  metadata.gz: 611f2ebd66b47ba06e51dcba81545e17dc8759fb
+  data.tar.gz: 60378c2f9a04272698fbb64bbeb104ab7dd49b76
 SHA512:
-  metadata.gz: d6b1075946a566e9a92703c682a160ece90bfe8e46e7ec2f722617048d25e54b6453aa25ffc91582e12267bd773904d88605b16a20648dc05536f9943bd00e44
-  data.tar.gz: 4140cc01d656daed66444a853ea585c4aec6fc55b10361e29eac617f924b421d2a3aa2fe17519a5c585ffd8d044b55d01e32e76924bb2ed2ecd441af4ae82ed9
+  metadata.gz: 80eb92e66a2114940e77b7668941a2fd25d032d8caf1b3ca9752368b01027ed979049a4a90cfdcbb89a25e8ab85c3e91b7b9441da47fb2571a45b9e5fe271a71
+  data.tar.gz: a5dc67e0af98b68713b90ff8d15e8cf3923d86597d19777816a1a74391b5666295cfa74b3a569829b261e5eb629861cca79ecfcff3cbbb7259797f13e678a0e3

data/.dir-locals.el

@@ -0,0 +1,3 @@
+((js-mode . ((indent-tabs-mode . nil)
+             (js-indent-level . 2)
+             (show-trailing-whitespace . t))))

data/.travis.yml

@@ -1,10 +1,14 @@
 notifications:
   recipients:
     - groonga-commit@lists.sourceforge.jp
+env:
+  - DEFAULT_TEST_TASK=test:unit
+  - DEFAULT_TEST_TASK=test:command:default
+  - DEFAULT_TEST_TASK=test:command:version1
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
-  - ruby-head
+  - 2.1
+#  - ruby-head
 before_install:
   - curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/README.md

@@ -1,18 +1,17 @@
 # Fluent::Plugin::Droonga
 
-TODO: Write a gem description
+## What's "Droonga"?
 
-## What's "droonga"?
-
-Droonga is a distributed fulltext search engine, named from "distributed-groonga".
-This package provides droonga plugin for Fluent event collector.
+Droonga is a scalable data processing engine, named from "Distributed
+Groonga". Droonga can execute distributed fulltext search.
 
 ## Usage
 
-TODO: Write usage.
+See [droonga.org](http://droonga.org/). You can find overview, install
+manual, reference manual and tutorial at the site.
 
 ## Copyright
 
-* Copyright (c) 2013 Droonga Project
+* Copyright (c) 2013-2014 Droonga Project
 * License
   * GNU Lesser General Public License version 2.1

data/Rakefile

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 #
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -29,19 +29,36 @@ Packnga::DocumentTask.new(spec) do |task|
   task.translate_languages = ["ja"]
 end
 
+def run_command_test(*options)
+  ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb",),
+       *options)
+end
+
 namespace :test do
   desc "Run unit test"
   task :unit do
     ruby(File.join(File.dirname(__FILE__), "test", "unit", "run-test.rb"))
   end
 
-  desc "Run command test"
-  task :command do
-    ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb"))
+  namespace :command do
+    desc "Run command test: default"
+    task :default do
+      run_command_test
+    end
+
+    desc "Run command test: version1"
+    task :version1 do
+      run_command_test("--config", "version1")
+    end
   end
 end
 
 desc "Run test"
-task :test => ["test:unit", "test:command"]
+task :test => [
+  "test:unit",
+  "test:command:default",
+  "test:command:version1",
+]
 
-task :default => :test
+test_task = ENV["DEFAULT_TEST_TASK"] || "test"
+task :default => test_task

data/fluent-plugin-droonga.gemspec

@@ -17,10 +17,10 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "0.9.9"
+  gem.version       = "1.0.0"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
-  gem.description   = "Droonga(distributed Groonga) plugin for Fluent event collector"
+  gem.description   = "Droonga (distributed Groonga) plugin for Fluent event collector"
   gem.summary       = gem.description
   gem.homepage      = "https://github.com/droonga/fluent-plugin-droonga"
   gem.files         = `git ls-files`.split($/)

data/lib/droonga/adapter.rb

@@ -14,15 +14,24 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/pluggable"
-require "droonga/plugin/metadata/adapter_message"
+require "droonga/loggable"
+require "droonga/plugin/metadata/adapter_input_message"
+require "droonga/plugin/metadata/adapter_output_message"
+require "droonga/error_messages"
 
 module Droonga
   class Adapter
     extend Pluggable
+    include Loggable
+    include ErrorMessages
 
     class << self
-      def message
-        Plugin::Metadata::AdapterMessage.new(self)
+      def input_message
+        Plugin::Metadata::AdapterInputMessage.new(self)
+      end
+
+      def output_message
+        Plugin::Metadata::AdapterOutputMessage.new(self)
       end
 
       def id

data/lib/droonga/adapter_runner.rb

@@ -13,6 +13,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+require "droonga/loggable"
 require "droonga/message_matcher"
 require "droonga/input_message"
 require "droonga/output_message"
@@ -20,74 +21,78 @@ require "droonga/adapter"
 
 module Droonga
   class AdapterRunner
+    include Loggable
+
     def initialize(dispatcher, plugins)
       @dispatcher = dispatcher
       default_plugins = ["error"]
       plugins += (default_plugins - plugins)
+      logger.debug("activating plugins: #{plugins.join(", ")}")
       @adapter_classes = Adapter.find_sub_classes(plugins)
+      logger.debug("activated:\n#{@adapter_classes.join("\n")}")
     end
 
     def shutdown
     end
 
     def adapt_input(message)
-      $log.trace("#{log_tag}: adapt_input: start",
-                 :dataset => message["dataset"],
-                 :type => message["type"])
+      logger.trace("adapt_input: start",
+                   :dataset => message["dataset"],
+                   :type => message["type"])
       adapted_message = message
       adapted_message["appliedAdapters"] = []
       @adapter_classes.each do |adapter_class|
         adapter_class_id = adapter_class.id
-        pattern = adapter_class.message.input_pattern
+        pattern = adapter_class.input_message.pattern
         if pattern
           matcher = MessageMatcher.new(pattern)
-          $log.trace("#{log_tag}: adapt_input: skip: #{adapter_class_id}",
-                     :pattern => pattern)
+          logger.trace("adapt_input: skip: #{adapter_class_id}",
+                       :pattern => pattern)
           next unless matcher.match?(adapted_message)
         end
-        $log.trace("#{log_tag}: adapt_input: use: #{adapter_class_id}")
+        logger.trace("adapt_input: use: #{adapter_class_id}")
         input_message = InputMessage.new(adapted_message)
         adapter = adapter_class.new
         adapter.adapt_input(input_message)
         adapted_message = input_message.adapted_message
         adapted_message["appliedAdapters"] << adapter_class_id
       end
-      $log.trace("#{log_tag}: adapt_input: done",
-                 :dataset => adapted_message["dataset"],
-                 :type => adapted_message["type"])
+      logger.trace("adapt_input: done",
+                   :dataset => adapted_message["dataset"],
+                   :type => adapted_message["type"])
       adapted_message
     end
 
     def adapt_output(message)
-      $log.trace("#{log_tag}: adapt_output: start",
-                 :dataset => message["dataset"],
-                 :type => message["type"])
+      logger.trace("adapt_output: start",
+                   :dataset => message["dataset"],
+                   :type => message["type"])
       adapted_message = message
       applied_adapters = adapted_message["appliedAdapters"]
       @adapter_classes.reverse_each do |adapter_class|
         adapter_class_id = adapter_class.id
         if applied_adapters
-          $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}: " +
-                     "input adapter wasn't applied",
-                     :applied_adapters => applied_adapters)
+          logger.trace("adapt_output: skip: #{adapter_class_id}: " +
+                         "input adapter wasn't applied",
+                       :applied_adapters => applied_adapters)
           next unless applied_adapters.include?(adapter_class.id)
         end
-        pattern = adapter_class.message.output_pattern
+        pattern = adapter_class.output_message.pattern
         if pattern
           matcher = MessageMatcher.new(pattern)
-          $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}",
-                     :pattern => pattern)
+          logger.trace("adapt_output: skip: #{adapter_class_id}",
+                       :pattern => pattern)
           next unless matcher.match?(adapted_message)
         end
-        $log.trace("#{log_tag}: adapt_output: use: #{adapter_class_id}")
+        logger.trace("adapt_output: use: #{adapter_class_id}")
         output_message = OutputMessage.new(adapted_message)
         adapter = adapter_class.new
         adapter.adapt_output(output_message)
         adapted_message = output_message.adapted_message
       end
-      $log.trace("#{log_tag}: adapt_output: done",
-                 :dataset => adapted_message["dataset"],
-                 :type => adapted_message["type"])
+      logger.trace("adapt_output: done",
+                   :dataset => adapted_message["dataset"],
+                   :type => adapted_message["type"])
       adapted_message
     end
 

data/lib/droonga/catalog/base.rb

@@ -15,101 +15,18 @@
 
 require "digest/sha1"
 require "zlib"
-require "droonga/message_processing_error"
+require "time"
+require "droonga/error_messages"
+require "droonga/catalog/errors"
 
 module Droonga
   module Catalog
     class Base
-      attr_reader :base_path
-      def initialize(data, base_path)
+      attr_reader :path, :base_path
+      def initialize(data, path)
         @data = data
-        @base_path = base_path
-
-        @data["datasets"].each do |name, dataset|
-          number_of_partitions = dataset["number_of_partitions"]
-          next if number_of_partitions < 2
-          total_weight = compute_total_weight(dataset)
-          continuum = []
-          dataset["ring"].each do |key, value|
-            points = number_of_partitions * 160 * value["weight"] / total_weight
-            points.times do |point|
-              hash = Digest::SHA1.hexdigest("#{key}:#{point}")
-              continuum << [hash[0..7].to_i(16), key]
-            end
-          end
-          dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
-        end
-        @options = @data["options"] || {}
-      end
-
-      def option(name)
-        @options[name]
-      end
-
-      def get_partitions(name)
-        device = @data["farms"][name]["device"]
-        pattern = Regexp.new("^#{name}\.")
-        results = {}
-        @data["datasets"].each do |key, dataset|
-          workers = dataset["workers"]
-          plugins = dataset["plugins"]
-          dataset["ring"].each do |key, part|
-            part["partitions"].each do |range, partitions|
-              partitions.each do |partition|
-                if partition =~ pattern
-                  path = File.join([device, $POSTMATCH, "db"])
-                  path = File.expand_path(path, base_path)
-                  options = {
-                    :database => path,
-                    :n_workers => workers,
-                    :plugins => plugins
-                  }
-                  results[partition] = options
-                end
-              end
-            end
-          end
-        end
-        return results
-      end
-
-      def get_routes(name, args)
-        routes = []
-        dataset = dataset(name)
-        case args["type"]
-        when "broadcast"
-          dataset["ring"].each do |key, partition|
-            select_range_and_replicas(partition, args, routes)
-          end
-        when "scatter"
-          name = get_partition(dataset, args["key"])
-          partition = dataset["ring"][name]
-          select_range_and_replicas(partition, args, routes)
-        end
-        return routes
-      end
-
-      def get_partition(dataset, key)
-        continuum = dataset["continuum"]
-        return dataset["ring"].keys[0] unless continuum
-        hash = Zlib.crc32(key)
-        min = 0
-        max = continuum.size - 1
-        while (min < max) do
-          index = (min + max) / 2
-          value, key = continuum[index]
-          return key if value == hash
-          if value > hash
-            max = index
-          else
-            min = index + 1
-          end
-        end
-        return continuum[max][1]
-      end
-
-      def datasets
-        @data["datasets"] || {}
+        @path = path
+        @base_path = File.dirname(path)
       end
 
       def have_dataset?(name)
@@ -119,27 +36,6 @@ module Droonga
       def dataset(name)
         datasets[name]
       end
-
-      def select_range_and_replicas(partition, args, routes)
-        date_range = args["date_range"] || 0..-1
-        partition["partitions"].sort[date_range].each do |time, replicas|
-          case args["replica"]
-          when "top"
-            routes << replicas[0]
-          when "random"
-            routes << replicas[rand(replicas.size)]
-          when "all"
-            routes.concat(replicas)
-          end
-        end
-      end
-
-      private
-      def compute_total_weight(dataset)
-        dataset["ring"].reduce(0) do |result, zone|
-          result + zone[1]["weight"]
-        end
-      end
     end
   end
 end

data/lib/droonga/catalog/dataset.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,41 +13,29 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "digest/sha1"
+require "droonga/catalog/schema"
 
 module Droonga
   module Catalog
     class Dataset
-      def initialize(data)
+      def initialize(name, data)
+        @name = name
         @data = data
+        @schema = nil
       end
 
-      def continuum
-        @continuum ||= compute_continuum
+      # provided for compatibility
+      def [](key)
+        @data[key]
       end
 
-      private
-      def compute_continuum
-        number_of_partitions = @data["number_of_partitions"]
-        return [] if number_of_partitions < 2
-        total_weight = compute_total_weight
-        continuum = []
-        @data["ring"].each do |key, value|
-          points = number_of_partitions * 160 * value["weight"] / total_weight
-          points.times do |point|
-            hash = Digest::SHA1.hexdigest("#{key}:#{point}")
-            continuum << [hash[0..7].to_i(16), key]
-          end
-        end
-        continuum.sort do |a, b|
-          a[0] - b[0]
-        end
+      # provided for compatibility
+      def []=(key, value)
+        @data[key] = value
       end
 
-      def compute_total_weight
-        @data["ring"].reduce(0) do |result, zone|
-          result + zone[1]["weight"]
-        end
+      def schema
+        @schema ||= Droonga::Catalog::Schema.new(@name, @data["schema"])
       end
     end
   end