droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/catalog/volume_collection.rb

@@ -1,79 +0,0 @@
-# Copyright (C) 2014 Droonga Project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-module Droonga
-  module Catalog
-    class VolumeCollection
-      include Enumerable
-
-      def initialize(volumes)
-        @volumes = volumes
-      end
-
-      def each(&block)
-        @volumes.each(&block)
-      end
-
-      def ==(other)
-        other.is_a?(self.class) and
-          to_a == other.to_a
-      end
-
-      def eql?(other)
-        self == other
-      end
-
-      def hash
-        to_a.hash
-      end
-
-      def select(how=nil, live_nodes=nil)
-        volumes = live_volumes(live_nodes)
-        case how
-        when :top
-          [volumes.first]
-        when :random
-          [volumes.sample]
-        when :all
-          @volumes
-        else
-          super
-        end
-      end
-
-      def all_nodes
-        @all_nodes ||= collect_all_nodes
-      end
-
-      def live_volumes(live_nodes=nil)
-        return @volumes unless live_nodes
-
-        @volumes.select do |volume|
-          dead_nodes = volume.all_nodes - live_nodes
-          dead_nodes.empty?
-        end
-      end
-
-      private
-      def collect_all_nodes
-        nodes = []
-        @volumes.each do |volume|
-          nodes += volume.all_nodes
-        end
-        nodes.sort.uniq
-      end
-    end
-  end
-end

data/lib/droonga/catalog_fetcher.rb

@@ -1,53 +0,0 @@
-# Copyright (C) 2014 Droonga Project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-require "socket"
-
-require "droonga/client"
-
-require "droonga/address"
-require "droonga/catalog/dataset"
-
-module Droonga
-  class CatalogFetcher
-    def initialize(client_options)
-      @client_options = default_options.merge(client_options)
-    end
-
-    def fetch(options={})
-      message = {
-        "dataset" => options[:dataset] || Catalog::Dataset::DEFAULT_NAME,
-        "type"    => "catalog.fetch"
-      }
-      Droonga::Client.open(@client_options) do |client|
-        response = client.request(message)
-        response["body"]
-      end
-    end
-
-    private
-    def default_options
-      {
-        :host          => "127.0.0.1",
-        :port          => Address::DEFAULT_PORT,
-        :tag           => Address::DEFAULT_TAG,
-        :protocol      => :droonga,
-        :timeout       => 1,
-        :receiver_host => Socket.gethostname,
-        :receiver_port => 0,
-      }
-    end
-  end
-end

data/lib/droonga/catalog_generator.rb

@@ -1,243 +0,0 @@
-# Copyright (C) 2014 Droonga Project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-require "time"
-
-require "droonga/address"
-require "droonga/catalog/dataset"
-
-module Droonga
-  class CatalogGenerator
-    DEFAULT_DATASET = Catalog::Dataset::DEFAULT_NAME
-    DEFAULT_HOSTS = [Address::DEFAULT_HOST]
-    DEFAULT_N_WORKERS = 4
-    DEFAULT_N_SLICES = 1
-    DEFAULT_PLUGINS = ["groonga", "search", "crud", "dump", "system", "catalog"]
-    DEFAULT_PORT = Address::DEFAULT_PORT
-    DEFAULT_TAG = Address::DEFAULT_TAG
-
-    attr_reader :datasets
-
-    class << self
-      def generate(datasets_params)
-        generator = new
-        datasets_params.each do |name, params|
-          generator.add_dataset(name, params)
-        end
-        generator.generate
-      end
-    end
-
-    def initialize
-      @version = 2
-      @effective_date = Time.now
-      @datasets = {}
-    end
-
-    def add_dataset(name, options)
-      @datasets[name] = Dataset.new(name, options)
-    end
-
-    def generate
-      {
-        "version"       => @version,
-        "effectiveDate" => @effective_date.iso8601,
-        "datasets"      => catalog_datasets,
-      }
-    end
-
-    def load(catalog)
-      catalog["datasets"].each do |name, catalog_dataset|
-        load_dataset(name, catalog_dataset)
-      end
-      self
-    end
-
-    def dataset_for_host(host)
-      @datasets.each do |name, dataset|
-        if dataset.replicas.hosts.include?(host)
-          return dataset
-        end
-      end
-      nil
-    end
-
-    def modify(dataset_modifications)
-      dataset_modifications.each do |name, modification|
-        dataset = @datasets[name]
-        next unless dataset
-
-        replicas = dataset.replicas
-
-        if modification[:replica_hosts]
-          replicas.hosts = modification[:replica_hosts]
-        end
-
-        if modification[:add_replica_hosts]
-          replicas.hosts += modification[:add_replica_hosts]
-          replicas.hosts.uniq!
-        end
-
-        if modification[:remove_replica_hosts]
-          replicas.hosts -= modification[:remove_replica_hosts]
-        end
-      end
-    end
-
-    private
-    def catalog_datasets
-      catalog_datasets = {}
-      @datasets.each do |name, dataset|
-        catalog_datasets[name] = dataset.to_catalog
-      end
-      catalog_datasets
-    end
-
-    def load_dataset(name, catalog_dataset)
-      options = {}
-      options[:n_workers] = catalog_dataset["nWorkers"]
-      options[:plugins]   = catalog_dataset["plugins"]
-      options[:schema]    = catalog_dataset["schema"]
-      options[:fact]      = catalog_dataset["fact"]
-      options[:replicas]  = catalog_dataset["replicas"]
-      add_dataset(name, options)
-    end
-
-    class Dataset
-      attr_reader :name
-
-      def initialize(name, options)
-        @name = name
-        @options = options
-      end
-
-      def n_workers
-        @options[:n_workers] || DEFAULT_N_WORKERS
-      end
-
-      def plugins
-        @options[:plugins] || DEFAULT_PLUGINS
-      end
-
-      def schema
-        @options[:schema] || {}
-      end
-
-      def fact
-        @options[:fact]
-      end
-
-      def replicas
-        @replicas ||= create_replicas
-      end
-
-      def to_catalog
-        catalog = {
-          "nWorkers" => n_workers,
-          "plugins"  => plugins,
-          "schema"   => schema,
-          "replicas" => replicas.to_catalog,
-        }
-        catalog["fact"] = fact if fact
-        catalog
-      end
-
-      private
-      def create_replicas
-        catalog_replicas = @options[:replicas]
-        if catalog_replicas
-          replicas = Replicas.new
-          replicas.load(catalog_replicas)
-          replicas
-        else
-          Replicas.new(@options)
-        end
-      end
-    end
-
-    class Replicas
-      attr_accessor :hosts
-      attr_reader :port, :tag, :n_slices
-
-      def initialize(options={})
-        @hosts      = options[:hosts] || DEFAULT_HOSTS
-        @port       = options[:port]
-        @tag        = options[:tag]
-        @n_slices   = options[:n_slices]
-      end
-
-      def load(catalog_replicas)
-        dataset = Catalog::Dataset.new("temporary",
-                                       "replicas" => catalog_replicas)
-        @hosts = dataset.replicas.collect do |replica|
-          replica.slices.first.volume.address.host
-        end
-        collection_volume = dataset.replicas.first
-        slices = collection_volume.slices
-        @n_slices = slices.size
-        single_volume_address = slices.first.volume.address
-        @port = single_volume_address.port
-        @tag = single_volume_address.tag
-      end
-
-      def to_catalog
-        catalog_replicas = []
-        @hosts.each do |host|
-          replica = Replica.new(host, :port => @port,
-                                      :tag => @tag,
-                                      :n_slices => @n_slices)
-          catalog_replicas << replica.to_catalog
-        end
-        catalog_replicas
-      end
-    end
-
-    class Replica
-      def initialize(host, options={})
-        @host       = host
-        @port       = options[:port]     || DEFAULT_PORT
-        @tag        = options[:tag]      || DEFAULT_TAG
-        @n_slices   = options[:n_slices] || DEFAULT_N_SLICES
-      end
-
-      def to_catalog
-        slices = []
-        @n_slices.times do |i|
-          slices << catalog_slice(i)
-        end
-        {
-          "dimension" => "_key",
-          "slicer" => "hash",
-          "slices" => slices,
-        }
-      end
-
-      private
-      def catalog_slice(nth_slice)
-        name = "%03d" % nth_slice
-        {
-          "weight" => weight,
-          "volume" => {
-            "address" => "#{@host}:#{@port}/#{@tag}.#{name}",
-          },
-        }
-      end
-
-      def weight
-        @weight ||= 100 / @n_slices
-      end
-    end
-  end
-end

data/lib/droonga/catalog_loader.rb

@@ -1,56 +0,0 @@
-# Copyright (C) 2013-2014 Droonga Project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-require "json"
-
-require "droonga/catalog/version1"
-require "droonga/catalog/version2"
-
-module Droonga
-  class CatalogLoader
-    def initialize(path)
-      @path = path
-    end
-
-    def load
-      data = nil
-      begin
-        data = File.open(@path) do |file|
-          JSON.parse(file.read)
-        end
-      rescue Errno::ENOENT => error
-        raise Error.new("Missing catalog file #{@path}")
-      rescue JSON::ParserError => error
-        raise Error.new("Syntax error in #{@path}:\n#{error.to_s}")
-      end
-
-      unless data.is_a?(Hash)
-        raise Error.new("Root element of catalog must be an object in #{@path}")
-      end
-
-      version = data["version"]
-      case version
-      when 1
-        Catalog::Version1.new(data, @path)
-      when 2
-        Catalog::Version2.new(data, @path)
-      when nil
-        raise Error.new("Catalog version must be specified in #{@path}")
-      else
-        raise Error.new("Unsupported catalog version <#{version}> is specified in #{@path}")
-      end
-    end
-  end
-end

data/lib/droonga/command/remote.rb

@@ -1,404 +0,0 @@
-# Copyright (C) 2014 Droonga Project
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-require "json"
-
-require "droonga/path"
-require "droonga/serf"
-require "droonga/node_status"
-require "droonga/catalog_generator"
-require "droonga/catalog_modifier"
-require "droonga/catalog_fetcher"
-require "droonga/data_absorber"
-require "droonga/safe_file_writer"
-require "droonga/service_installation"
-
-module Droonga
-  module Command
-    module Remote
-      class Base
-        attr_reader :response
-
-        def initialize(serf_name, params)
-          @serf_name = serf_name
-          @params    = params
-          @response  = {
-            "log" => []
-          }
-          @serf = Serf.new(nil, @serf_name)
-
-          @service_installation = ServiceInstallation.new
-          @service_installation.ensure_using_service_base_directory
-
-          log("params = #{params}")
-        end
-
-        def process
-          # override me!
-        end
-
-        def should_process?
-          for_me? or @params.nil? or not @params.include?("node")
-        end
-
-        private
-        def node
-          @serf_name
-        end
-
-        def host
-          node.split(":").first
-        end
-
-        def target_node
-          @params && @params["node"]
-        end
-
-        def for_me?
-          target_node == @serf_name
-        end
-
-        def log(message)
-          @response["log"] << message
-        end
-      end
-
-      class ChangeRole < Base
-        def process
-          status = NodeStatus.new
-          status.set(:role, @params["role"])
-        end
-      end
-
-      class ReportStatus < Base
-        def process
-          status = NodeStatus.new
-          @response["value"] = status.get(@params["key"])
-        end
-      end
-
-      class SetStatus < Base
-        def process
-          status = NodeStatus.new
-          status.set(@params["key"], @params["value"])
-        end
-      end
-
-      class Join < Base
-        def process
-          log("type = #{type}")
-          case type
-          when "replica"
-            join_as_replica
-          end
-        end
-
-        private
-        def type
-          @params["type"]
-        end
-
-        def source_node
-          @params["source"]
-        end
-
-        def joining_node
-          @params["node"]
-        end
-
-        def dataset_name
-          @params["dataset"]
-        end
-
-        def messages_per_second
-          @params["messages_per_second"]
-        end
-
-        def valid_params?
-          have_required_params? and
-            valid_node?(source_node) and
-            valid_node?(joining_node)
-        end
-
-        def have_required_params?
-          required_params = [
-            source_node,
-            joining_node,
-            dataset_name,
-          ]
-          required_params.all? do |param|
-            not param.nil?
-          end
-        end
-
-        NODE_PATTERN = /\A([^:]+):(\d+)\/(.+)\z/
-
-        def valid_node?(node)
-          node =~ NODE_PATTERN
-        end
-
-        def source_host
-          @source_host ||= (source_node =~ NODE_PATTERN && $1)
-        end
-
-        def joining_host
-          @joining_host ||= (joining_node =~ NODE_PATTERN && $1)
-        end
-
-        def port
-          @port ||= (source_node =~ NODE_PATTERN && $2 && $2.to_i)
-        end
-
-        def tag
-          @tag ||= (source_node =~ NODE_PATTERN && $3)
-        end
-
-        def should_absorb_data?
-          @params["copy"]
-        end
-
-        def join_as_replica
-          return unless valid_params?
-
-          log("source_node = #{source_node}")
-
-          @catalog = fetch_catalog
-
-          other_hosts = replica_hosts
-          log("other_hosts = #{other_hosts}")
-          return if other_hosts.empty?
-
-          # restart self with the fetched catalog.
-          SafeFileWriter.write(Path.catalog) do |output, file|
-            output.puts(JSON.pretty_generate(@catalog))
-            @service_installation.ensure_correct_file_permission(file)
-          end
-
-          absorb_data if should_absorb_data?
-
-          log("joining to the cluster: update myself")
-
-          CatalogModifier.modify do |modifier, file|
-            modifier.datasets[dataset_name].replicas.hosts += other_hosts
-            modifier.datasets[dataset_name].replicas.hosts.uniq!
-            @service_installation.ensure_correct_file_permission(file)
-          end
-
-          @serf.join(*other_hosts)
-        end
-
-        def replica_hosts
-          generator = CatalogGenerator.new
-          generator.load(@catalog)
-          dataset = generator.dataset_for_host(source_host) ||
-                      generator.dataset_for_host(host)
-          return [] unless dataset
-          dataset.replicas.hosts
-        end
-
-        def fetch_catalog
-          fetcher = CatalogFetcher.new(:host          => source_host,
-                                       :port          => port,
-                                       :tag           => tag,
-                                       :receiver_host => host)
-          fetcher.fetch(:dataset => dataset_name)
-        end
-
-        def absorb_data
-          log("starting to copy data from #{source_host}")
-
-          CatalogModifier.modify do |modifier, file|
-            modifier.datasets[dataset_name].replicas.hosts = [host]
-            @service_installation.ensure_correct_file_permission(file)
-          end
-          sleep(5) #TODO: wait for restart. this should be done more safely, to avoid starting of absorbing with old catalog.json.
-
-          status = NodeStatus.new
-          status.set(:absorbing, true)
-          DataAbsorber.absorb(:dataset          => dataset_name,
-                              :source_host      => source_host,
-                              :destination_host => joining_host,
-                              :port             => port,
-                              :tag              => tag,
-                              :messages_per_second => messages_per_second)
-          status.delete(:absorbing)
-          sleep(1)
-        end
-      end
-
-      class AbsorbData < Base
-        attr_writer :dataset_name, :port, :tag
-
-        def process
-          return unless source
-
-          log("start to absorb data from #{source}")
-
-          if dataset_name.nil? or port.nil? or tag.nil?
-            current_catalog = JSON.parse(Path.catalog.read)
-            generator = CatalogGenerator.new
-            generator.load(current_catalog)
-
-            dataset = generator.dataset_for_host(source)
-            return unless dataset
-
-            self.dataset_name = dataset.name
-            self.port         = dataset.replicas.port
-            self.tag          = dataset.replicas.tag
-          end
-
-          log("dataset = #{dataset_name}")
-          log("port    = #{port}")
-          log("tag     = #{tag}")
-
-          status = NodeStatus.new
-          status.set(:absorbing, true)
-          DataAbsorber.absorb(:dataset          => dataset_name,
-                              :source_host      => source,
-                              :destination_host => host,
-                              :port             => port,
-                              :tag              => tag,
-                              :messages_per_second => messages_per_second,
-                              :client           => "droonga-send")
-          status.delete(:absorbing)
-        end
-
-        private
-        def source
-          @params["source"]
-        end
-
-        def dataset_name
-          @dataset_name ||= @params["dataset"]
-        end
-
-        def port
-          @port ||= @params["port"]
-        end
-
-        def tag
-          @tag ||= @params["tag"]
-        end
-
-        def messages_per_second
-          @messages_per_second ||= @params["messages_per_second"]
-        end
-      end
-
-      class ModifyReplicasBase < Base
-        private
-        def dataset
-          @params["dataset"]
-        end
-
-        def hosts
-          @hosts ||= prepare_hosts
-        end
-
-        def prepare_hosts
-          hosts = @params["hosts"]
-          return nil unless hosts
-          hosts = [hosts] if hosts.is_a?(String)
-          hosts
-        end
-      end
-
-      class SetReplicas < ModifyReplicasBase
-        def process
-          return if dataset.nil? or hosts.nil?
-
-          log("new replicas: #{hosts.join(",")}")
-
-          CatalogModifier.modify do |modifier, file|
-            modifier.datasets[dataset].replicas.hosts = hosts
-            @service_installation.ensure_correct_file_permission(file)
-          end
-
-          @serf.join(*hosts)
-        end
-      end
-
-      class AddReplicas < ModifyReplicasBase
-        def process
-          return if dataset.nil? or hosts.nil?
-
-          added_hosts = hosts - [host]
-          log("adding replicas: #{added_hosts.join(",")}")
-          return if added_hosts.empty?
-
-          CatalogModifier.modify do |modifier, file|
-            modifier.datasets[dataset].replicas.hosts += added_hosts
-            modifier.datasets[dataset].replicas.hosts.uniq!
-            @service_installation.ensure_correct_file_permission(file)
-          end
-
-          @serf.join(*added_hosts)
-        end
-      end
-
-      class RemoveReplicas < ModifyReplicasBase
-        def process
-          return if dataset.nil? or hosts.nil?
-
-          log("removing replicas: #{hosts.join(",")}")
-
-          CatalogModifier.modify do |modifier, file|
-            modifier.datasets[dataset].replicas.hosts -= hosts
-            @service_installation.ensure_correct_file_permission(file)
-          end
-        end
-      end
-
-      class Unjoin < ModifyReplicasBase
-        def process
-          return if dataset.nil? or hosts.nil?
-
-          log("unjoining replicas: #{hosts.join(",")}")
-
-          CatalogModifier.modify do |modifier, file|
-            if unjoining_node?
-              modifier.datasets[dataset].replicas.hosts = hosts
-            else
-              modifier.datasets[dataset].replicas.hosts -= hosts
-            end
-            @service_installation.ensure_correct_file_permission(file)
-          end
-        end
-
-        private
-        def unjoining_node?
-          hosts.include?(host)
-        end
-      end
-
-      class UpdateLiveNodes < Base
-        def process
-          path = Path.live_nodes
-          nodes = live_nodes
-          file_contents = JSON.pretty_generate(nodes)
-          SafeFileWriter.write(path) do |output, file|
-            output.puts(file_contents)
-            @service_installation.ensure_correct_file_permission(file)
-          end
-        end
-
-        private
-        def live_nodes
-          @serf.live_nodes
-        end
-      end
-    end
-  end
-end