droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/catalog/generator.rb

@@ -0,0 +1,245 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "time"
+
+require "droonga/node_name"
+require "droonga/catalog/dataset"
+
+module Droonga
+  module Catalog
+    class Generator
+      DEFAULT_DATASET = Catalog::Dataset::DEFAULT_NAME
+      DEFAULT_HOSTS     = [NodeName::DEFAULT_HOST]
+      DEFAULT_N_WORKERS = 4
+      DEFAULT_N_SLICES  = 1
+      DEFAULT_PLUGINS   = ["groonga", "search", "crud", "dump", "system", "catalog"]
+      DEFAULT_PORT      = NodeName::DEFAULT_PORT
+      DEFAULT_TAG       = NodeName::DEFAULT_TAG
+
+      attr_reader :datasets
+
+      class << self
+        def generate(datasets_params)
+          generator = new
+          datasets_params.each do |name, params|
+            generator.add_dataset(name, params)
+          end
+          generator.generate
+        end
+      end
+
+      def initialize
+        @version        = 2
+        @effective_date = Time.now
+        @datasets       = {}
+      end
+
+      def add_dataset(name, options)
+        @datasets[name] = Dataset.new(name, options)
+      end
+
+      def generate
+        {
+          "version"       => @version,
+          "effectiveDate" => @effective_date.iso8601,
+          "datasets"      => catalog_datasets,
+        }
+      end
+
+      def load(catalog)
+        catalog["datasets"].each do |name, catalog_dataset|
+          load_dataset(name, catalog_dataset)
+        end
+        self
+      end
+
+      def dataset_for_host(host)
+        @datasets.each do |name, dataset|
+          if dataset.replicas.hosts.include?(host)
+            return dataset
+          end
+        end
+        nil
+      end
+
+      def modify(dataset_modifications)
+        dataset_modifications.each do |name, modification|
+          dataset = @datasets[name]
+          next unless dataset
+
+          replicas = dataset.replicas
+
+          if modification[:replica_hosts]
+            replicas.hosts = modification[:replica_hosts]
+          end
+
+          if modification[:add_replica_hosts]
+            replicas.hosts += modification[:add_replica_hosts]
+            replicas.hosts.uniq!
+          end
+
+          if modification[:remove_replica_hosts]
+            replicas.hosts -= modification[:remove_replica_hosts]
+          end
+        end
+      end
+
+      private
+      def catalog_datasets
+        catalog_datasets = {}
+        @datasets.each do |name, dataset|
+          catalog_datasets[name] = dataset.to_catalog
+        end
+        catalog_datasets
+      end
+
+      def load_dataset(name, catalog_dataset)
+        options = {}
+        options[:n_workers] = catalog_dataset["nWorkers"]
+        options[:plugins]   = catalog_dataset["plugins"]
+        options[:schema]    = catalog_dataset["schema"]
+        options[:fact]      = catalog_dataset["fact"]
+        options[:replicas]  = catalog_dataset["replicas"]
+        add_dataset(name, options)
+      end
+
+      class Dataset
+        attr_reader :name
+
+        def initialize(name, options)
+          @name = name
+          @options = options
+        end
+
+        def n_workers
+          @options[:n_workers] || DEFAULT_N_WORKERS
+        end
+
+        def plugins
+          @options[:plugins] || DEFAULT_PLUGINS
+        end
+
+        def schema
+          @options[:schema] || {}
+        end
+
+        def fact
+          @options[:fact]
+        end
+
+        def replicas
+          @replicas ||= create_replicas
+        end
+
+        def to_catalog
+          catalog = {
+            "nWorkers" => n_workers,
+            "plugins"  => plugins,
+            "schema"   => schema,
+            "replicas" => replicas.to_catalog,
+          }
+          catalog["fact"] = fact if fact
+          catalog
+        end
+
+        private
+        def create_replicas
+          catalog_replicas = @options[:replicas]
+          if catalog_replicas
+            replicas = Replicas.new
+            replicas.load(catalog_replicas)
+            replicas
+          else
+            Replicas.new(@options)
+          end
+        end
+      end
+
+      class Replicas
+        attr_accessor :hosts
+        attr_reader :port, :tag, :n_slices
+
+        def initialize(options={})
+          @hosts    = options[:hosts] || DEFAULT_HOSTS
+          @port     = options[:port]
+          @tag      = options[:tag]
+          @n_slices = options[:n_slices]
+        end
+
+        def load(catalog_replicas)
+          dataset = Catalog::Dataset.new("temporary",
+                                         "replicas" => catalog_replicas)
+          @hosts = dataset.replicas.collect do |replica|
+            replica.slices.first.volume.address.host
+          end
+          collection_volume = dataset.replicas.first
+          slices = collection_volume.slices
+          @n_slices = slices.size
+          single_volume_address = slices.first.volume.address
+          @port = single_volume_address.port
+          @tag  = single_volume_address.tag
+        end
+
+        def to_catalog
+          catalog_replicas = []
+          @hosts.each do |host|
+            replica = Replica.new(host, :port => @port,
+                                        :tag => @tag,
+                                        :n_slices => @n_slices)
+            catalog_replicas << replica.to_catalog
+          end
+          catalog_replicas
+        end
+      end
+
+      class Replica
+        def initialize(host, options={})
+          @host     = host
+          @port     = options[:port]     || DEFAULT_PORT
+          @tag      = options[:tag]      || DEFAULT_TAG
+          @n_slices = options[:n_slices] || DEFAULT_N_SLICES
+        end
+
+        def to_catalog
+          slices = []
+          @n_slices.times do |i|
+            slices << catalog_slice(i)
+          end
+          {
+            "dimension" => "_key",
+            "slicer" => "hash",
+            "slices" => slices,
+          }
+        end
+
+        private
+        def catalog_slice(nth_slice)
+          local_name = "%03d" % nth_slice
+          {
+            "weight" => weight,
+            "volume" => {
+              "address" => "#{@host}:#{@port}/#{@tag}.#{local_name}",
+            },
+          }
+        end
+
+        def weight
+          @weight ||= 100 / @n_slices
+        end
+      end
+    end
+  end
+end

data/lib/droonga/catalog/loader.rb

@@ -0,0 +1,66 @@
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "json"
+
+require "droonga/catalog/version1"
+require "droonga/catalog/version2"
+
+module Droonga
+  module Catalog
+    class Loader
+      def initialize(path=nil)
+        @path = path
+      end
+
+      def load
+        unless @path
+          raise Error.new("nothing specified")
+        end
+
+        data = nil
+        begin
+          data = File.open(@path) do |file|
+            JSON.parse(file.read)
+          end
+        rescue Errno::ENOENT => error
+          raise Error.new("Missing catalog file #{@path}")
+        rescue JSON::ParserError => error
+          raise Error.new("Syntax error in #{@path}:\n#{error.to_s}")
+        end
+        parse(data)
+      end
+
+      def parse(data)
+        unless data.is_a?(Hash)
+          raise Error.new("Root element of catalog must be an object in #{@path}: " +
+                            "#{JSON.generate(data)}")
+        end
+
+        version = data["version"]
+        case version
+        when 1
+          Catalog::Version1.new(data, @path)
+        when 2
+          Catalog::Version2.new(data, @path)
+        when nil
+          raise Error.new("Catalog version must be specified in #{@path}")
+        else
+          raise Error.new("Unsupported catalog version <#{version}> is specified in #{@path}")
+        end
+      end
+    end
+  end
+end

data/lib/droonga/{catalog_modifier.rb → catalog/modifier.rb}

@@ -16,29 +16,22 @@
 require "json"
 
 require "droonga/path"
-require "droonga/catalog_generator"
+require "droonga/catalog/generator"
 require "droonga/safe_file_writer"
 
 module Droonga
-  class CatalogModifier
-    class << self
-      def modify
-        new.modify do |generator, file|
-          yield(generator, file)
-        end
+  module Catalog
+    class Modifier
+      def initialize(source_catalog)
+        @generator = Catalog::Generator.new
+        @generator.load(source_catalog)
       end
-    end
 
-    def initialize
-      @generator = CatalogGenerator.new
-      @catalog = JSON.parse(Path.catalog.read)
-      @generator.load(@catalog)
-    end
-
-    def modify
-      SafeFileWriter.write(Path.catalog) do |output, file|
-        yield(@generator, file)
-        output.puts(JSON.pretty_generate(@generator.generate))
+      def modify
+        SafeFileWriter.write(Path.catalog) do |output, file|
+          yield(@generator, file)
+          output.puts(JSON.pretty_generate(@generator.generate))
+        end
       end
     end
   end

data/lib/droonga/catalog/replicas_volume.rb

@@ -0,0 +1,123 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  module Catalog
+    class ReplicasVolume
+      include Enumerable
+
+      def initialize(dataset, raw)
+        @dataset = dataset
+        @raw = raw
+      end
+
+      def each(&block)
+        replicas.each(&block)
+      end
+
+      def ==(other)
+        other.is_a?(self.class) and
+          to_a == other.to_a
+      end
+
+      def eql?(other)
+        self == other
+      end
+
+      def hash
+        to_a.hash
+      end
+
+      def select(how=nil, active_nodes=nil)
+        case how
+        when :top
+          replicas = live_replicas(active_nodes)
+          if replicas.empty?
+            []
+          else
+            [replicas.first]
+          end
+        when :random
+          replicas = live_replicas(active_nodes)
+          if replicas.empty?
+            []
+          else
+            [replicas.sample]
+          end
+        when :all
+          live_replicas(active_nodes)
+        else
+          super
+        end
+      end
+
+      def replicas
+        @replicas ||= create_replicas
+      end
+
+      def all_nodes
+        @all_nodes ||= collect_all_nodes
+      end
+
+      def live_replicas(active_nodes=nil)
+        return replicas unless active_nodes
+
+        replicas.select do |volume|
+          dead_nodes = volume.all_nodes - active_nodes
+          dead_nodes.empty?
+        end
+      end
+
+      def compute_routes(message, active_nodes)
+        routes = []
+        case message["type"]
+        when "broadcast"
+          replicas = select(message["replica"].to_sym, active_nodes)
+          replicas.each do |volume|
+            routes.concat(volume.compute_routes(message, active_nodes))
+          end
+        when "scatter"
+          #TODO: we should choose suitable replica from differently sliced replicas.
+          replicas = select(message["replica"].to_sym, active_nodes)
+          replicas.each do |volume|
+            routes.concat(volume.compute_routes(message, active_nodes))
+          end
+        end
+        routes
+      end
+
+      def sliced?
+        replicas.any? do |volume|
+          volume.sliced?
+        end
+      end
+
+      private
+      def create_replicas
+        @raw["replicas"].collect do |raw_replica|
+          Catalog::Volume.create(@dataset, raw_replica)
+        end
+      end
+
+      def collect_all_nodes
+        nodes = []
+        replicas.each do |volume|
+          nodes.concat(volume.all_nodes)
+        end
+        nodes.sort.uniq
+      end
+    end
+  end
+end