droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/catalog/schema.rb

@@ -19,55 +19,55 @@ module Droonga
   module Catalog
     class Schema
       class ColumnVectorOptions
-        def initialize(data)
-          @data = data
+        def initialize(raw)
+          @raw = raw
         end
 
         def weight
-          @data["weight"]
+          @raw["weight"]
         end
       end
 
       class ColumnIndexOptions
-        def initialize(data)
-          @data = data
+        def initialize(raw)
+          @raw = raw
         end
 
         def section
-          @data["section"]
+          @raw["section"]
         end
 
         def weight
-          @data["weight"]
+          @raw["weight"]
         end
 
         def position
-          @data["position"]
+          @raw["position"]
         end
 
         def sources
-          @data["sources"]
+          @raw["sources"]
         end
       end
 
       class Column
-        attr_reader :table, :name, :data, :vector_options, :index_options
-        def initialize(table, name, data)
+        attr_reader :table, :name, :raw, :vector_options, :index_options
+        def initialize(table, name, raw)
           @table = table
           @name = name
-          @data = data
-          @vector_options = ColumnVectorOptions.new(vector_options_data)
-          @index_options = ColumnIndexOptions.new(index_options_data)
+          @raw = raw
+          @vector_options = ColumnVectorOptions.new(raw_vector_options)
+          @index_options = ColumnIndexOptions.new(raw_index_options)
         end
 
         def ==(other)
           self.class == other.class and
             name == other.name and
-            data == other.data
+            raw == other.raw
         end
 
         def type
-          @data["type"] || "Scalar"
+          @raw["type"] || "Scalar"
         end
 
         def type_symbol
@@ -82,7 +82,7 @@ module Droonga
         end
 
         def value_type
-          @data["valueType"]
+          @raw["valueType"]
         end
 
         def value_type_groonga
@@ -94,35 +94,35 @@ module Droonga
         end
 
         private
-        def vector_options_data
-          @data["vectorOptions"] || {}
+        def raw_vector_options
+          @raw["vectorOptions"] || {}
         end
 
-        def index_options_data
-          @data["indexOptions"] || {}
+        def raw_index_options
+          @raw["indexOptions"] || {}
         end
       end
 
       class Table
-        attr_reader :name, :columns, :data
-        def initialize(name, data)
+        attr_reader :name, :columns, :raw
+        def initialize(name, raw)
           @name = name
-          @data = data
+          @raw = raw
           @columns = {}
 
-          columns_data.each do |column_name, column_data|
-            @columns[column_name] = Column.new(name, column_name, column_data)
+          raw_columns.each do |column_name, raw_column|
+            @columns[column_name] = Column.new(name, column_name, raw_column)
           end
         end
 
         def ==(other)
           self.class == other.class and
             name == other.name and
-            data == other.data
+            raw == other.raw
         end
 
         def type
-          @data["type"] || "Hash"
+          @raw["type"] || "Hash"
         end
 
         def type_symbol
@@ -139,7 +139,7 @@ module Droonga
         end
 
         def key_type
-          @data["keyType"]
+          @raw["keyType"]
         end
 
         def key_type_groonga
@@ -154,26 +154,26 @@ module Droonga
         end
 
         def tokenizer
-          @data["tokenizer"]
+          @raw["tokenizer"]
         end
 
         def normalizer
-          @data["normalizer"]
+          @raw["normalizer"]
         end
 
         private
-        def columns_data
-          @data["columns"] || []
+        def raw_columns
+          @raw["columns"] || []
         end
       end
 
       attr_reader :tables
-      def initialize(dataset_name, data)
+      def initialize(dataset_name, raw)
         @dataset_name = dataset_name
-        @data = data || []
+        @raw = raw || []
         @tables = {}
-        @data.each do |table_name, table_data|
-          @tables[table_name] = Table.new(table_name, table_data)
+        @raw.each do |table_name, raw_table|
+          @tables[table_name] = Table.new(table_name, raw_table)
         end
       end
 

data/lib/droonga/catalog/single_volume.rb

@@ -19,9 +19,9 @@ module Droonga
   module Catalog
     class SingleVolume
       attr_reader :address
-      def initialize(data)
-        @data = data
-        @address = Address.parse(@data["address"])
+      def initialize(raw)
+        @raw = raw
+        @address = Address.parse(@raw["address"])
       end
 
       def node
@@ -31,6 +31,14 @@ module Droonga
       def all_nodes
         @all_nodes ||= [node]
       end
+
+      def compute_routes(message, active_nodes)
+        [address.to_s]
+      end
+
+      def sliced?
+        false
+      end
     end
   end
 end

data/lib/droonga/catalog/slice.rb

@@ -16,30 +16,34 @@
 module Droonga
   module Catalog
     class Slice
-      def initialize(dataset, data)
+      def initialize(dataset, raw)
         @dataset = dataset
-        @data = data
+        @raw = raw
       end
 
       def weight
-        @data["weight"] || 1
+        @raw["weight"] || 1
       end
 
       def label
-        @data["label"]
+        @raw["label"]
       end
 
       def boundary
-        @data["boundary"]
+        @raw["boundary"]
       end
 
       def volume
-        @volume ||= Volume.create(@dataset, @data["volume"])
+        @volume ||= Catalog::Volume.create(@dataset, @raw["volume"])
       end
 
       def all_nodes
         @all_nodes ||= volume.all_nodes
       end
+
+      def compute_routes(message, active_nodes)
+        volume.compute_routes(message, active_nodes)
+      end
     end
   end
 end

data/lib/droonga/catalog/{collection_volume.rb → slices_volume.rb}

@@ -20,29 +20,35 @@ require "droonga/catalog/slice"
 
 module Droonga
   module Catalog
-    class CollectionVolume
-      def initialize(dataset, data)
+    class SlicesVolume
+      def initialize(dataset, raw)
         @dataset = dataset
-        @data = data
+        @raw = raw
         compute_continuum if ratio_scaled_slicer?
       end
 
       def dimension
-        @data["dimension"] || "_key"
+        @raw["dimension"] || "_key"
       end
 
       def slicer
-        @data["slicer"] || "hash"
+        @raw["slicer"] || "hash"
       end
 
       def slices
-        @slices ||= @data["slices"].collect do |raw_slice|
-          Slice.new(@dataset, raw_slice)
-        end
+        @slices ||= create_slices
       end
 
-      def select_slices(range=0..-1)
-        slices.sort_by(&:label)[range]
+      def select_slices(how=:all, range=0..-1)
+        selected = slices.sort_by(&:label)[range]
+        case how
+        when :random
+          [selected.sample]
+        when :all
+          selected
+        else
+          selected
+        end
       end
 
       def choose_slice(record)
@@ -73,6 +79,30 @@ module Droonga
         @all_nodes ||= collect_all_nodes
       end
 
+      def compute_routes(message, active_nodes)
+        routes = []
+        slices = []
+        case message["type"]
+        when "broadcast"
+          slices = select_slices(message["slice"].to_sym)
+        when "scatter"
+          record = message["record"]
+          if record
+            slices = [choose_slice(record)]
+          else
+            slices = select_slices(message["slice"].to_sym)
+          end
+        end
+        slices.each do |slice|
+          routes.concat(slice.compute_routes(message, active_nodes))
+        end
+        routes
+      end
+
+      def sliced?
+        slices.size > 1
+      end
+
       private
       def compute_continuum
         total_weight = compute_total_weight
@@ -106,10 +136,16 @@ module Droonga
         end
       end
 
+      def create_slices
+        @raw["slices"].collect do |raw_slice|
+          Slice.new(@dataset, raw_slice)
+        end
+      end
+
       def collect_all_nodes
         nodes = []
         slices.each do |slice|
-          nodes += slice.all_nodes
+          nodes.concat(slice.all_nodes)
         end
         nodes.uniq.sort
       end

data/lib/droonga/catalog/version1.rb

@@ -23,7 +23,7 @@ require "droonga/catalog/dataset"
 module Droonga
   module Catalog
     class Version1 < Base
-      def initialize(data, path)
+      def initialize(raw, path)
         super
         @errors = []
 
@@ -42,10 +42,10 @@ module Droonga
       end
 
       def get_partitions(name)
-        device = @data["farms"][name]["device"]
+        device = @raw["farms"][name]["device"]
         pattern = Regexp.new("^#{name}\.")
         results = {}
-        @data["datasets"].each do |dataset_name, dataset_data|
+        @raw["datasets"].each do |dataset_name, dataset_data|
           dataset = Dataset.new(dataset_name, dataset_data)
           workers = dataset["workers"]
           plugins = dataset["plugins"]
@@ -79,7 +79,7 @@ module Droonga
       private
       def prepare_data
         @datasets = {}
-        @data["datasets"].each do |name, dataset|
+        @raw["datasets"].each do |name, dataset|
           @datasets[name] = Dataset.new(name, dataset)
           number_of_partitions = dataset["number_of_partitions"]
           next if number_of_partitions < 2
@@ -94,7 +94,7 @@ module Droonga
           end
           dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
         end
-        @options = @data["options"] || {}
+        @options = @raw["options"] || {}
       end
 
       def compute_total_weight(dataset)
@@ -104,7 +104,7 @@ module Droonga
       end
 
       def collect_all_nodes
-        @data["zones"].sort
+        @raw["zones"].sort
       end
 
       def validate
@@ -190,13 +190,13 @@ module Droonga
       end
 
       def validate_effective_date
-        date = @data["effective_date"]
+        date = @raw["effective_date"]
         validate_required_parameter(date, "effective_date")
         validate_valid_datetime(date, "effective_date")
       end
 
       def validate_farms
-        farms = @data["farms"]
+        farms = @raw["farms"]
 
         validate_required_parameter(farms, "farms")
         validate_parameter_type(Hash, farms, "farms")
@@ -214,7 +214,7 @@ module Droonga
       end
 
       def validate_zones
-        zones = @data["zones"]
+        zones = @raw["zones"]
 
         validate_required_parameter(zones, "zones")
         validate_parameter_type(Array, zones, "zones")
@@ -236,7 +236,7 @@ module Droonga
       end
 
       def validate_datasets
-        datasets = @data["datasets"]
+        datasets = @raw["datasets"]
 
         validate_required_parameter(datasets, "datasets")
         validate_parameter_type(Hash, datasets, "datasets")
@@ -328,11 +328,11 @@ module Droonga
       end
 
       def validate_zone_relations
-        return unless @data["zones"].is_a?(Array)
-        return unless @data["farms"].is_a?(Hash)
+        return unless @raw["zones"].is_a?(Array)
+        return unless @raw["farms"].is_a?(Hash)
 
-        farms = @data["farms"]
-        zones = @data["zones"]
+        farms = @raw["farms"]
+        zones = @raw["zones"]
 
         all_farms = farms.keys
         all_zones = zones.flatten
@@ -351,14 +351,14 @@ module Droonga
       end
 
       def validate_database_relations
-        return unless @data["farms"]
+        return unless @raw["farms"]
 
-        farm_names = @data["farms"].keys.collect do |name|
+        farm_names = @raw["farms"].keys.collect do |name|
           Regexp.escape(name)
         end
         valid_farms_matcher = Regexp.new("^(#{farm_names.join("|")})\.")
 
-        @data["datasets"].each do |dataset_name, dataset|
+        @raw["datasets"].each do |dataset_name, dataset|
           ring = dataset["ring"]
           next if ring.nil? or !ring.is_a?(Hash)
           ring.each do |ring_key, part|
@@ -389,7 +389,7 @@ module Droonga
       end
 
       class Dataset < Catalog::Dataset
-        def compute_routes(args, live_nodes=nil)
+        def compute_routes(args, active_nodes=nil)
           routes = []
           case args["type"]
           when "broadcast"
@@ -397,7 +397,13 @@ module Droonga
               select_range_and_replicas(partition, args, routes)
             end
           when "scatter"
-            name = get_partition(args["record"]["_key"])
+            record = args["record"]
+            if record
+              key = record["_key"]
+            else
+              key = nil
+            end
+            name = get_partition(key)
             partition = self["ring"][name]
             select_range_and_replicas(partition, args, routes)
           end
@@ -407,6 +413,8 @@ module Droonga
         def get_partition(key)
           continuum = self["continuum"]
           return self["ring"].keys[0] unless continuum
+          return continuum.sample[1] unless key
+          key = key.to_s unless key.is_a?(String)
           hash = Zlib.crc32(key)
           min = 0
           max = continuum.size - 1
@@ -436,6 +444,26 @@ module Droonga
             end
           end
         end
+
+        def all_nodes
+          @all_nodes ||= collect_all_nodes
+        end
+
+        def sliced?
+          self["ring"].keys.size > 1
+        end
+
+        private
+        def collect_all_nodes
+          nodes = []
+          self["ring"].each do |ring_key, part|
+            part["partitions"].each do |partition|
+              partition =~ /\A([^:]+:[0-9]+\/[^\.]+)/
+              nodes << $1
+            end
+          end
+          nodes.sort.uniq
+        end
       end
     end
   end