fluent-plugin-droonga 0.9.0 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80594c4e021918c28b58451b90293961f7b65174
-  data.tar.gz: ff89eb342312841a7d9eaa8831a7d190bec34c35
+  metadata.gz: 24503ddfa77a2b4c19b83b771069a3c91f738ef2
+  data.tar.gz: 50f89c1a0fed25c58d8d16b5e16c41258cbd4e3a
 SHA512:
-  metadata.gz: 39351f8baffeae2e33ac2b3d1ad6f06c39fe08a626ee8f260c62501e03b5acb05a67f2e0667833250f200679664d518da774b57c2356c30f260cd0afb83f8940
-  data.tar.gz: c26dd8db11b63e7f6130826f6eb4ce97b05dae064e39b9502b3cc7cbf0f54a912c5fc3885bab2855ace0a21cba9af851e64e9a4d40c136310790c848d9e0aadd
+  metadata.gz: d6b1075946a566e9a92703c682a160ece90bfe8e46e7ec2f722617048d25e54b6453aa25ffc91582e12267bd773904d88605b16a20648dc05536f9943bd00e44
+  data.tar.gz: 4140cc01d656daed66444a853ea585c4aec6fc55b10361e29eac617f924b421d2a3aa2fe17519a5c585ffd8d044b55d01e32e76924bb2ed2ecd441af4ae82ed9

data/.travis.yml

@@ -3,5 +3,8 @@ notifications:
     - groonga-commit@lists.sourceforge.jp
 rvm:
   - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - ruby-head
 before_install:
   - curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/Gemfile

@@ -27,7 +27,7 @@ if File.exist?(local_rroonga_path)
   gem "groonga-command", :path => local_groonga_command_path
   gem "groonga-command-parser", :path => local_groonga_command_parser_path
 elsif ENV["TRAVIS"] == "true"
-  require_unreleased_gems = false
+  require_unreleased_gems = true
   if require_unreleased_gems
     gem "rroonga", :git => "git://github.com/ranguba/rroonga.git"
     gem "groonga-command",
@@ -57,3 +57,10 @@ if File.exist?(drnbench_dir)
 else
   gem "drnbench", :github => "droonga/drnbench"
 end
+
+grn2drn_dir = File.join(parent_dir, "grn2drn")
+if File.exist?(grn2drn_dir)
+  gem "grn2drn", :path => grn2drn_dir
+else
+  gem "grn2drn", :github => "droonga/grn2drn"
+end

data/fluent-plugin-droonga.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "0.9.0"
+  gem.version       = "0.9.9"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
   gem.description   = "Droonga(distributed Groonga) plugin for Fluent event collector"

data/lib/droonga/adapter.rb

@@ -0,0 +1,39 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/pluggable"
+require "droonga/plugin/metadata/adapter_message"
+
+module Droonga
+  class Adapter
+    extend Pluggable
+
+    class << self
+      def message
+        Plugin::Metadata::AdapterMessage.new(self)
+      end
+
+      def id
+        options[:id] || name || object_id.to_s
+      end
+    end
+
+    def adapt_input(input_message)
+    end
+
+    def adapt_output(output_message)
+    end
+  end
+end

data/lib/droonga/adapter_runner.rb

@@ -0,0 +1,99 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/message_matcher"
+require "droonga/input_message"
+require "droonga/output_message"
+require "droonga/adapter"
+
+module Droonga
+  class AdapterRunner
+    def initialize(dispatcher, plugins)
+      @dispatcher = dispatcher
+      default_plugins = ["error"]
+      plugins += (default_plugins - plugins)
+      @adapter_classes = Adapter.find_sub_classes(plugins)
+    end
+
+    def shutdown
+    end
+
+    def adapt_input(message)
+      $log.trace("#{log_tag}: adapt_input: start",
+                 :dataset => message["dataset"],
+                 :type => message["type"])
+      adapted_message = message
+      adapted_message["appliedAdapters"] = []
+      @adapter_classes.each do |adapter_class|
+        adapter_class_id = adapter_class.id
+        pattern = adapter_class.message.input_pattern
+        if pattern
+          matcher = MessageMatcher.new(pattern)
+          $log.trace("#{log_tag}: adapt_input: skip: #{adapter_class_id}",
+                     :pattern => pattern)
+          next unless matcher.match?(adapted_message)
+        end
+        $log.trace("#{log_tag}: adapt_input: use: #{adapter_class_id}")
+        input_message = InputMessage.new(adapted_message)
+        adapter = adapter_class.new
+        adapter.adapt_input(input_message)
+        adapted_message = input_message.adapted_message
+        adapted_message["appliedAdapters"] << adapter_class_id
+      end
+      $log.trace("#{log_tag}: adapt_input: done",
+                 :dataset => adapted_message["dataset"],
+                 :type => adapted_message["type"])
+      adapted_message
+    end
+
+    def adapt_output(message)
+      $log.trace("#{log_tag}: adapt_output: start",
+                 :dataset => message["dataset"],
+                 :type => message["type"])
+      adapted_message = message
+      applied_adapters = adapted_message["appliedAdapters"]
+      @adapter_classes.reverse_each do |adapter_class|
+        adapter_class_id = adapter_class.id
+        if applied_adapters
+          $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}: " +
+                     "input adapter wasn't applied",
+                     :applied_adapters => applied_adapters)
+          next unless applied_adapters.include?(adapter_class.id)
+        end
+        pattern = adapter_class.message.output_pattern
+        if pattern
+          matcher = MessageMatcher.new(pattern)
+          $log.trace("#{log_tag}: adapt_output: skip: #{adapter_class_id}",
+                     :pattern => pattern)
+          next unless matcher.match?(adapted_message)
+        end
+        $log.trace("#{log_tag}: adapt_output: use: #{adapter_class_id}")
+        output_message = OutputMessage.new(adapted_message)
+        adapter = adapter_class.new
+        adapter.adapt_output(output_message)
+        adapted_message = output_message.adapted_message
+      end
+      $log.trace("#{log_tag}: adapt_output: done",
+                 :dataset => adapted_message["dataset"],
+                 :type => adapted_message["type"])
+      adapted_message
+    end
+
+    private
+    def log_tag
+      "adapter-runner"
+    end
+  end
+end

data/lib/droonga/catalog/base.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,12 +20,6 @@ require "droonga/message_processing_error"
 module Droonga
   module Catalog
     class Base
-      class UnknownDataset < NotFound
-        def initialize(dataset)
-          super("The dataset #{dataset.inspect} does not exist.")
-        end
-      end
-
       attr_reader :base_path
       def initialize(data, base_path)
         @data = data
@@ -68,7 +62,7 @@ module Droonga
                   options = {
                     :database => path,
                     :n_workers => workers,
-                    :handlers => plugins
+                    :plugins => plugins
                   }
                   results[partition] = options
                 end
@@ -114,10 +108,16 @@ module Droonga
         return continuum[max][1]
       end
 
+      def datasets
+        @data["datasets"] || {}
+      end
+
+      def have_dataset?(name)
+        datasets.key?(name)
+      end
+
       def dataset(name)
-        dataset = @data["datasets"][name]
-        raise UnknownDataset.new(name) unless dataset
-        dataset
+        datasets[name]
       end
 
       def select_range_and_replicas(partition, args, routes)

data/lib/droonga/catalog/dataset.rb

@@ -0,0 +1,54 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "digest/sha1"
+
+module Droonga
+  module Catalog
+    class Dataset
+      def initialize(data)
+        @data = data
+      end
+
+      def continuum
+        @continuum ||= compute_continuum
+      end
+
+      private
+      def compute_continuum
+        number_of_partitions = @data["number_of_partitions"]
+        return [] if number_of_partitions < 2
+        total_weight = compute_total_weight
+        continuum = []
+        @data["ring"].each do |key, value|
+          points = number_of_partitions * 160 * value["weight"] / total_weight
+          points.times do |point|
+            hash = Digest::SHA1.hexdigest("#{key}:#{point}")
+            continuum << [hash[0..7].to_i(16), key]
+          end
+        end
+        continuum.sort do |a, b|
+          a[0] - b[0]
+        end
+      end
+
+      def compute_total_weight
+        @data["ring"].reduce(0) do |result, zone|
+          result + zone[1]["weight"]
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/version1.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/lib/droonga/collector.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,15 +13,15 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/pluggable"
+require "droonga/legacy_pluggable"
 require "droonga/collector_plugin"
 
 module Droonga
   class Collector
-    include Pluggable
+    include LegacyPluggable
 
-    def initialize
-      load_plugins(["basic", "search", "groonga"]) # TODO: make customizable
+    def initialize(plugins)
+      load_plugins(plugins)
     end
 
     private

data/lib/droonga/collector_plugin.rb

@@ -15,13 +15,13 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/plugin"
+require "droonga/legacy_plugin"
 
 module Droonga
-  class CollectorPlugin < Plugin
+  class CollectorPlugin < LegacyPlugin
     extend PluginRegisterable
 
-    attr_reader :task, :input_name, :component, :output_values, :body, :output_names
+    attr_reader :task, :input_name, :step, :output_values, :body, :output_names
     def initialize
       super()
     end
@@ -30,11 +30,11 @@ module Droonga
       return false unless message.is_a? Hash
       @task = message["task"]
       return false unless @task.is_a? Hash
-      @component = @task["component"]
-      return false unless @component.is_a? Hash
+      @step = @task["step"]
+      return false unless @step.is_a? Hash
       @output_values = @task["values"]
-      @body = @component["body"]
-      @output_names = @component["outputs"]
+      @body = @step["body"]
+      @output_names = @step["outputs"]
       @id = message["id"]
       @value = message["value"]
       @input_name = message["name"]

data/lib/droonga/command.rb

@@ -0,0 +1,36 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/message_matcher"
+
+module Droonga
+  class Command
+    attr_reader :method_name
+    # @option options [Array] :pattern The pattern to be matched
+    #    against message. If the pattern is matched to a message,
+    #    the command will be applied.
+    #
+    # @see MessageMatcher
+    def initialize(method_name, options)
+      @method_name = method_name
+      @options = options
+      @matcher = MessageMatcher.new(@options[:pattern])
+    end
+
+    def match?(message)
+      @matcher.match?(message)
+    end
+  end
+end

data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb}

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,15 +13,21 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/input_adapter_plugin"
-
 module Droonga
-  class CRUDInputAdapter < Droonga::InputAdapterPlugin
-    repository.register("crud", self)
+  class CommandRepository
+    attr_reader :commands
+    def initialize
+      @commands = []
+    end
+
+    def register(command)
+      @commands << command
+    end
 
-    command :add
-    def add(input_message)
-      input_message.add_route("crud_generic_response")
+    def find(message)
+      @commands.find do |command|
+        command.match?(message)
+      end
     end
   end
 end