fluent-plugin-droonga 1.0.0 → 1.0.1

data/.travis.yml

@@ -11,4 +11,4 @@ rvm:
   - 2.1
 #  - ruby-head
 before_install:
-  - curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
+  - GROONGA_MASTER=yes curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/Gemfile

@@ -41,7 +41,7 @@ droonga_client_dir = File.join(parent_dir, "droonga-client-ruby")
 if File.exist?(droonga_client_dir)
   gem "droonga-client", :path => droonga_client_dir
 else
-  gem "droonga-client", github: "droonga/droonga-client-ruby"
+  gem "droonga-client", :github => "droonga/droonga-client-ruby"
 end
 
 drntest_dir = File.join(parent_dir, "drntest")

data/fluent-plugin-droonga.gemspec

@@ -17,7 +17,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "1.0.0"
+  gem.version       = "1.0.1"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
   gem.description   = "Droonga (distributed Groonga) plugin for Fluent event collector"

data/lib/droonga/catalog/collection_volume.rb

@@ -0,0 +1,97 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "digest/sha1"
+require "zlib"
+
+require "droonga/catalog/slice"
+
+module Droonga
+  module Catalog
+    class CollectionVolume
+      def initialize(dataset, data)
+        @dataset = dataset
+        @data = data
+        compute_continuum if ratio_scaled_slicer?
+      end
+
+      def dimension
+        @data["dimension"] || "_key"
+      end
+
+      def slicer
+        @data["slicer"] || "hash"
+      end
+
+      def slices
+        @slices ||= @data["slices"].collect do |raw_slice|
+          Slice.new(@dataset, raw_slice)
+        end
+      end
+
+      def select_slices(range=0..-1)
+        slices.sort_by(&:label)[range]
+      end
+
+      def choose_slice(record)
+        return slices.first unless ratio_scaled_slicer?
+
+        key = record[dimension]
+        hash = Zlib.crc32(key)
+        min = 0
+        max = @continuum.size - 1
+        while (min < max)
+          index = (min + max) / 2
+          value, key = @continuum[index]
+          return key if value == hash
+          if value > hash
+            max = index
+          else
+            min = index + 1
+          end
+        end
+        @continuum[max][1]
+      end
+
+      def ratio_scaled_slicer?
+        slicer == "hash"
+      end
+
+      private
+      def compute_continuum
+        total_weight = compute_total_weight
+        continuum = []
+        n_slices = slices.size
+        slices.each do |slice|
+          weight = slice.weight
+          points = n_slices * 160 * weight / total_weight
+          points.times do |point|
+            hash = Digest::SHA1.hexdigest("#{@dataset.name}:#{point}")
+            continuum << [hash[0..7].to_i(16), slice]
+          end
+        end
+        @continuum = continuum.sort do |a, b|
+          a[0] - b[0]
+        end
+      end
+
+      def compute_total_weight
+        slices.reduce(0) do |result, slice|
+          result + slice.weight
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/dataset.rb

@@ -14,10 +14,14 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 require "droonga/catalog/schema"
+require "droonga/catalog/volume"
+require "droonga/catalog/volume_collection"
 
 module Droonga
   module Catalog
     class Dataset
+      attr_reader :name
+
       def initialize(name, data)
         @name = name
         @data = data
@@ -35,7 +39,30 @@ module Droonga
       end
 
       def schema
-        @schema ||= Droonga::Catalog::Schema.new(@name, @data["schema"])
+        @schema ||= Schema.new(@name, @data["schema"])
+      end
+
+      def plugins
+        @data["plugins"] || []
+      end
+
+      def fact
+        @data["fact"]
+      end
+
+      def n_workers
+        @data["nWorkers"] || 0
+      end
+
+      def replicas
+        @replicas ||= VolumeCollection.new(create_volumes(@data["replicas"]))
+      end
+
+      private
+      def create_volumes(raw_volumes)
+        raw_volumes.collect do |raw_volume|
+          Volume.create(self, raw_volume)
+        end
       end
     end
   end

data/lib/droonga/catalog/errors.rb

@@ -18,6 +18,25 @@ require "droonga/error"
 module Droonga
   module Catalog
     class ValidationError < Error
+      class Detail
+        attr_reader :value_path, :message
+        def initialize(value_path, message)
+          @value_path = value_path
+          @message = message
+        end
+      end
+
+      attr_reader :path, :details
+      def initialize(path, details)
+        message = "validation error: <#{path}>"
+        details.each do |detail|
+          message << "\n  * #{detail.value_path}: #{detail.message}"
+        end
+        super(message)
+      end
+    end
+
+    class LegacyValidationError < Error
       def initialize(message, path)
         if path
           super("[Validation Error <#{path}>]#{message}")
@@ -27,13 +46,13 @@ module Droonga
       end
     end
 
-    class MissingRequiredParameter < ValidationError
+    class MissingRequiredParameter < LegacyValidationError
       def initialize(name, path)
         super("[#{name}] A required parameter is missing.", path)
       end
     end
 
-    class MismatchedParameterType < ValidationError
+    class MismatchedParameterType < LegacyValidationError
       def initialize(name, expected_types, actual, path)
         expected_types = [expected_types] unless expected_types.is_a?(Array)
         message = nil
@@ -49,43 +68,43 @@ module Droonga
       end
     end
 
-    class InvalidDate < ValidationError
+    class InvalidDate < LegacyValidationError
       def initialize(name, value, path)
         super("[#{name}] Invalid date string: <#{value}>", path)
       end
     end
 
-    class NegativeNumber < ValidationError
+    class NegativeNumber < LegacyValidationError
       def initialize(name, actual, path)
         super("[#{name}] A positive number is expected, but <#{actual}>", path)
       end
     end
 
-    class SmallerThanOne < ValidationError
+    class SmallerThanOne < LegacyValidationError
       def initialize(name, actual, path)
         super("[#{name}] A number 1 or larger is expected, but <#{actual}>", path)
       end
     end
 
-    class FarmNotZoned < ValidationError
+    class FarmNotZoned < LegacyValidationError
       def initialize(name, zones, path)
         super("The farm does not appear in zones: <#{name}>, zones=<#{zones}>", path)
       end
     end
 
-    class UnknownFarmInZones < ValidationError
+    class UnknownFarmInZones < LegacyValidationError
       def initialize(name, zones, path)
         super("The farm is unknown: <#{name}>, zones=<#{zones}>", path)
       end
     end
 
-    class UnknownFarmForPartition < ValidationError
+    class UnknownFarmForPartition < LegacyValidationError
       def initialize(name, slice, path)
         super("The farm is unknown: <{#name}>, slice=<#{slice}>", path)
       end
     end
 
-    class UnsupportedValue < ValidationError
+    class UnsupportedValue < LegacyValidationError
       def initialize(name, value, path)
         super("[#{name}] Not supported value: <#{value}>", path)
       end

data/lib/droonga/catalog/schema.rb

@@ -18,6 +18,22 @@ require "tsort"
 module Droonga
   module Catalog
     class Schema
+      class ColumnVectorOptions
+        def initialize(data)
+          @data = data
+        end
+
+        def weight
+          @data["weight"]
+        end
+
+        def flags
+          flags = []
+          flags << "WITH_WEIGHT" if weight
+          flags
+        end
+      end
+
       class ColumnIndexOptions
         def initialize(data)
           @data = data
@@ -49,11 +65,12 @@ module Droonga
       end
 
       class Column
-        attr_reader :table, :name, :data, :index_options
+        attr_reader :table, :name, :data, :vector_options, :index_options
         def initialize(table, name, data)
           @table = table
           @name = name
           @data = data
+          @vector_options = ColumnVectorOptions.new(vector_options_data)
           @index_options = ColumnIndexOptions.new(index_options_data)
         end
 
@@ -81,7 +98,7 @@ module Droonga
         end
 
         def flags
-          [type_flag] + index_options.flags
+          [type_flag] + vector_options.flags + index_options.flags
         end
 
         def value_type
@@ -112,6 +129,10 @@ module Droonga
         end
 
         private
+        def vector_options_data
+          @data["vectorOptions"] || {}
+        end
+
         def index_options_data
           @data["indexOptions"] || {}
         end

data/lib/droonga/catalog/single_volume.rb

@@ -0,0 +1,28 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  module Catalog
+    class SingleVolume
+      def initialize(data)
+        @data = data
+      end
+
+      def address
+        @data["address"]
+      end
+    end
+  end
+end

data/lib/droonga/catalog/slice.rb

@@ -0,0 +1,43 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/catalog/volume"
+
+module Droonga
+  module Catalog
+    class Slice
+      def initialize(dataset, data)
+        @dataset = dataset
+        @data = data
+      end
+
+      def weight
+        @data["weight"] || 1
+      end
+
+      def label
+        @data["label"]
+      end
+
+      def boundary
+        @data["boundary"]
+      end
+
+      def volume
+        @volume ||= Volume.create(@dataset, @data["volume"])
+      end
+    end
+  end
+end

data/lib/droonga/catalog/version1.rb

@@ -75,7 +75,7 @@ module Droonga
             select_range_and_replicas(partition, args, routes)
           end
         when "scatter"
-          name = get_partition(dataset, args["key"])
+          name = get_partition(dataset, args["record"]["_key"])
           partition = dataset["ring"][name]
           select_range_and_replicas(partition, args, routes)
         end
@@ -166,7 +166,7 @@ module Droonga
       def do_validation(&block)
         begin
           yield
-        rescue ValidationError => error
+        rescue LegacyValidationError => error
           @errors << error
         end
       end
@@ -413,7 +413,7 @@ module Droonga
                     if directory_name.nil? or directory_name.empty?
                       message = "\"#{partition}\" has no database name. " +
                                   "You mus specify a database name for \"#{name}\"."
-                      raise ValidationError.new(message, @path)
+                      raise LegacyValidationError.new(message, @path)
                     end
                   end
                 end

data/lib/droonga/catalog/version2.rb

@@ -15,12 +15,14 @@
 
 require "droonga/catalog/base"
 require "droonga/catalog/dataset"
+require "droonga/catalog/version2_validator"
 
 module Droonga
   module Catalog
     class Version2 < Base
       def initialize(data, path)
         super
+        validate
         prepare_data
       end
 
@@ -33,11 +35,11 @@ module Droonga
         pattern = Regexp.new("^#{name}\.")
         results = {}
         @datasets.each do |dataset_name, dataset|
-          n_workers = dataset["nWorkers"]
-          plugins = dataset["plugins"]
-          dataset["replicas"].each do |replica|
-            replica["slices"].each do |slice|
-              volume_address = slice["volume"]["address"]
+          n_workers = dataset.n_workers
+          plugins = dataset.plugins
+          dataset.replicas.each do |volume|
+            volume.slices.each do |slice|
+              volume_address = slice.volume.address
               if pattern =~ volume_address
                 path = File.join([device, $POSTMATCH, "db"])
                 path = File.expand_path(path, base_path)
@@ -60,18 +62,18 @@ module Droonga
         dataset = dataset(name)
         case args["type"]
         when "broadcast"
-          replicas = select_replicas(dataset["replicas"], args["replica"])
-          replicas.each do |replica|
-            slices = select_slices(replica)
+          volumes = dataset.replicas.select(args["replica"].to_sym)
+          volumes.each do |volume|
+            slices = volume.select_slices
             slices.each do |slice|
-              routes << slice["volume"]["address"]
+              routes << slice.volume.address
             end
           end
         when "scatter"
-          replicas = select_replicas(dataset["replicas"], args["replica"])
-          replicas.each do |replica|
-            slice = select_slice(replica, args["key"])
-            routes << slice["volume"]["address"]
+          volumes = dataset.replicas.select(args["replica"].to_sym)
+          volumes.each do |volume|
+            slice = volume.choose_slice(args["record"])
+            routes << slice.volume.address
           end
         end
         routes
@@ -79,82 +81,16 @@ module Droonga
 
       private
       def validate
-        # TODO: Implement me.
+        validator = Version2Validator.new(@data, @path)
+        validator.validate
       end
 
       def prepare_data
         @datasets = {}
         @data["datasets"].each do |name, dataset|
-          replicas = dataset["replicas"]
-          replicas.each do |replica|
-            total_weight = compute_total_weight(replica)
-            continuum = []
-            slices = replica["slices"]
-            n_slices = slices.size
-            slices.each do |slice|
-              weight = slice["weight"] || default_weight
-              points = n_slices * 160 * weight / total_weight
-              points.times do |point|
-                hash = Digest::SHA1.hexdigest("#{name}:#{point}")
-                continuum << [hash[0..7].to_i(16), slice]
-              end
-            end
-            replica["continuum"] = continuum.sort do |a, b|
-              a[0] - b[0]
-            end
-          end
           @datasets[name] = Dataset.new(name, dataset)
         end
       end
-
-      def default_weight
-        1
-      end
-
-      def compute_total_weight(replica)
-        slices = replica["slices"]
-        slices.reduce(0) do |result, slice|
-          result + (slice["weight"] || default_weight)
-        end
-      end
-
-      def select_replicas(replicas, how)
-        case how
-        when "top"
-          [replicas.first]
-        when "random"
-          [replicas.sample]
-        when "all"
-          replicas
-        end
-      end
-
-      def select_slices(replica, range=0..-1)
-        sorted_slices = replica["slices"].sort_by do |slice|
-          slice["label"]
-        end
-        sorted_slices[range]
-      end
-
-      def select_slice(replica, key)
-        continuum = replica["continuum"]
-        return replica["slices"].first unless continuum
-
-        hash = Zlib.crc32(key)
-        min = 0
-        max = continuum.size - 1
-        while (min < max) do
-          index = (min + max) / 2
-          value, key = continuum[index]
-          return key if value == hash
-          if value > hash
-            max = index
-          else
-            min = index + 1
-          end
-        end
-        continuum[max][1]
-      end
     end
   end
 end