droonga-engine 1.0.8 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e7b344e7b567ecc524d5f5cfb52ed6a67342a672
-  data.tar.gz: 79ecc5c743f0f01f350a1855d55368dc024515a9
+  metadata.gz: f2574dab76722981c14a23373e838f1e01d18220
+  data.tar.gz: 9776825164dda315f20af0f0d610d2391cce93fd
 SHA512:
-  metadata.gz: 1f94a76e05f59f76bbd6e92774bd928c10a01c8c731df1e9b548a0b255b61abfab538e3c374d1e1fd74244639cb140d9913ee09649226a2226ad54d458afd0a2
-  data.tar.gz: 343bf2deb3d726858be67b61f222689535b939ea1ecce9d564f8f69a474e41cd22190600cf95e0b2e42f3e609ff244f44fcd9fafc946901fb2a89197de33a0b7
+  metadata.gz: 55b359cab3e5ca1bd0b7008233c54bbf413d27b97ee8d179ca09be701fa4f2749935cb37eaf743bfb0099600794c80d319edce22bedde28b69f685d944c8f193
+  data.tar.gz: 09add0504d2886e51065af0c09a4d350db98c8fe07e9a6de6f317ca08af34971016f57c2ac3706d4317df110d125077045c8973c996f4bda382a187cd903ffa0

data/bin/droonga-engine-absorb-data

@@ -18,6 +18,7 @@
 require "ostruct"
 require "optparse"
 require "open3"
+require "socket"
 
 require "droonga/engine/version"
 require "droonga/catalog_generator"
@@ -34,6 +35,7 @@ class AbsorbDataCommand
 
     puts "Start to absorb data from #{@options.source_host}"
     puts "                       to #{@options.destination_host}"
+    puts "                      via #{@options.receiver_host} (this host)"
     puts "  dataset = #{@options.dataset}"
     puts "  port    = #{@options.port}"
     puts "  tag     = #{@options.tag}"
@@ -57,6 +59,7 @@ class AbsorbDataCommand
     options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
     options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
     options.remote  = true
+    options.receiver_host = Socket.gethostname
     options.messages_per_second = Droonga::DataAbsorber::DEFAULT_MESSAGES_PER_SECOND
     parser = OptionParser.new
     parser.version = Droonga::Engine::VERSION
@@ -71,6 +74,11 @@ class AbsorbDataCommand
               "Host name of this cluster to be connected.") do |host|
       options.destination_host = host
     end
+    parser.on("--receiver-host=HOST",
+              "Host name of this computer.",
+              "(#{options.receiver_host})") do |host|
+      options.receiver_host = host
+    end
     parser.on("--port=PORT", Integer,
               "Port number of the source cluster to be connected.",
               "(#{options.port})") do |port|
@@ -139,6 +147,7 @@ class AbsorbDataCommand
       :dataset          => @options.dataset,
       :source_host      => @options.source_host,
       :destination_host => @options.destination_host,
+      :receiver_host    => @options.receiver_host,
       :port             => @options.port,
       :tag              => @options.tag,
       :messages_per_second => @options.messages_per_second,

data/bin/droonga-engine-join

@@ -18,6 +18,7 @@
 require "slop"
 require "json"
 require "pathname"
+require "socket"
 
 require "droonga/engine/version"
 require "droonga/path"
@@ -30,6 +31,12 @@ class JoinCommand
   def run
     parse_options
     trap_signals
+
+    puts "Start to join a new node #{@options[:host]}"
+    puts "       to the cluster of #{@options["replica-source-host"]}"
+    puts "                     via #{@options["receiver-host"]} (this host)"
+    puts ""
+
     set_node_role
     do_join
     sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
@@ -54,6 +61,9 @@ class JoinCommand
       option.on("replica-source-host=",
                 "Host name of the soruce node in the cluster to be connected.",
                 :required => true)
+      option.on("receiver-host=",
+                "Host name of this host.",
+                :default => Socket.gethostname)
       option.on(:dataset=,
                 "Dataset name of for the node to be joined.",
                 :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
@@ -101,6 +111,7 @@ class JoinCommand
       :dataset          => @options[:dataset],
       :source_host      => @options["replica-source-host"],
       :destination_host => @options[:host],
+      :receiver_host    => @options["receiver-host"],
       :port             => @options[:port],
       :tag              => @options[:tag],
       :messages_per_second => @options["records-per-second"],

data/bin/droonga-engine-unjoin

@@ -15,96 +15,121 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "ostruct"
-require "optparse"
+require "slop"
 require "json"
 require "pathname"
 
 require "droonga/engine/version"
 require "droonga/path"
+require "droonga/catalog_fetcher"
 require "droonga/catalog_generator"
-require "droonga/safe_file_writer"
-require "droonga/service_installation"
 require "droonga/serf"
 
-service_installation = Droonga::ServiceInstallation.new
-service_installation.ensure_using_service_base_directory
-
-options = OpenStruct.new
-options.base_dir = ENV[Droonga::Path::BASE_DIR_ENV_NAME] || Dir.pwd
-
-parser = OptionParser.new
-parser.version = Droonga::Engine::VERSION
-
-parser.on("--base-dir=PATH",
-          "Path to the base directory the catalog.json is located in.",
-          "(#{options.base_dir})") do |path|
-  options.base_dir = path
-end
-parser.on("--host=HOST",
-          "Host name of the replica removed from cluster.") do |host|
-  options.replica_remove_host = host
-end
-
-parser.parse!(ARGV)
-
-
-base_dir = Pathname(options.base_dir).expand_path
-ENV[Droonga::Path::BASE_DIR_ENV_NAME] = base_dir.to_s
-
-
-catalog_path = Droonga::Path.catalog
-unless catalog_path.exist?
-  raise "Cannot load 'catalog.json'. You must specify correct path " +
-          "to the base directory via --base-dir option."
-end
-
-unless options.replica_remove_host
-  raise "You must specify the host name or the IP address of a node to " +
-          "be removed from the cluster via --replica-remove-host option."
-end
-
-unless service_installation.have_read_permission?
-  puts("You have no permission to read files under " +
-         "<#{Droonga::Path.base.to_s}>.")
-  puts("Try again with right permission.")
-  exit(false)
-end
-
-source_catalog = JSON.parse(catalog_path.read)
-generator = Droonga::CatalogGenerator.new
-generator.load(source_catalog)
-
-dataset = generator.dataset_for_host(options.replica_remove_host)
-unless dataset
-  raise "Specified host #{options.replica_remove_host} is not a member of "+
-          "the cluster. You must specify correct host via --replica-remove-host " +
-          "option."
+class UnjoinCommand
+  def run
+    parse_options
+
+    puts "Start to unjoin a node #{@options[:host]}"
+    puts "                    by #{@options["receiver-host"]} (this host)"
+    puts ""
+
+    do_unjoin
+
+    puts("Done.")
+    exit(true)
+  end
+
+  private
+  def parse_options
+    options = Slop.parse(:help => true) do |option|
+      option.on(:host=,
+                "Host name of the replica removed from cluster.",
+                :required => true)
+      option.on("receiver-host=",
+                "Host name of this host.",
+                :default => Socket.gethostname)
+      option.on(:dataset=,
+                "Dataset name of for the node to be unjoined.",
+                :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
+      option.on(:port=,
+                "Port number of the source cluster to be connected.",
+                :as => Integer,
+                :default => Droonga::CatalogGenerator::DEFAULT_PORT)
+      option.on(:tag=,
+                "Tag name of the soruce cluster to be connected.",
+                :default => Droonga::CatalogGenerator::DEFAULT_TAG)
+    end
+    @options = options
+  rescue Slop::MissingOptionError => error
+    $stderr.puts(error)
+    exit(false)
+  end
+
+  def replica_remove_host
+    @options[:host]
+  end
+
+  def tag
+    @options[:tag]
+  end
+
+  def port
+    @options[:port]
+  end
+
+  def dataset_name
+    @options[:dataset]
+  end
+
+  def replica_remove_node
+    "#{replica_remove_host}:#{port}/#{tag}"
+  end
+
+  def fetch_catalog
+    fetcher = Droonga::CatalogFetcher.new(:host          => replica_remove_host,
+                                          :port          => port,
+                                          :tag           => tag,
+                                          :receiver_host => @options["receiver-host"])
+    fetcher.fetch(:dataset => dataset_name)
+  end
+
+  def remaining_node
+    @remaining_node ||= detect_remaining_node
+  end
+
+  def detect_remaining_node
+    catalog = fetch_catalog
+    generator = Droonga::CatalogGenerator.new
+    generator.load(catalog)
+
+    dataset = generator.dataset_for_host(replica_remove_host)
+    unless dataset
+      raise "Specified host #{replica_remove_host} is not a member of "+
+              "the cluster. You must specify correct host via --replica-remove-host " +
+              "option."
+    end
+
+    other_hosts = dataset.replicas.hosts
+
+    remaining_host = other_hosts.first || replica_remove_host
+    "#{remaining_host}:#{port}/#{tag}"
+  end
+
+  def run_remote_command(target, command, options)
+    serf = Droonga::Serf.new(nil, target)
+    result = serf.send_query(command, options)
+    puts(result[:result])
+    puts(result[:error]) unless result[:error].empty?
+    result[:response]
+  end
+
+  def do_unjoin
+    puts "Unjoining replica from the cluster..."
+
+    run_remote_command(remaining_node, "unjoin",
+                       "dataset" => dataset_name,
+                       "hosts"   => [replica_remove_host])
+  end
 end
 
-options.dataset     = dataset.name
-options.tag         = dataset.replicas.tag
-options.port        = dataset.replicas.port
-options.other_hosts = dataset.replicas.hosts
-
-remaining_host = options.other_hosts.first || options.replica_remove_host
-options.remaining_node = "#{remaining_host}:#{options.port}/#{options.tag}"
-
-
-def run_remote_command(target, command, options)
-  serf = Droonga::Serf.new(nil, target)
-  result = serf.send_query(command, options)
-  puts result[:result]
-  puts result[:error] unless result[:error].empty?
-end
-
-
-puts "Unjoining replica from the cluster..."
-
-run_remote_command(options.remaining_node, "remove_replicas",
-                   "dataset" => options.dataset,
-                   "hosts"   => [options.replica_remove_host])
-
-puts "Done."
-
-exit(true)
+UnjoinCommand.new.run

data/droonga-engine.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "archive-zip"
   gem.add_dependency "cool.io"
   gem.add_dependency "drndump"
-  gem.add_dependency "droonga-client", ">= 0.2.0"
+  gem.add_dependency "droonga-client", ">= 0.1.9"
   gem.add_dependency "droonga-message-pack-packer", ">= 1.0.2"
   gem.add_dependency "groonga-command-parser"
   gem.add_dependency "faraday"

data/lib/droonga/catalog/base.rb

@@ -16,6 +16,7 @@
 require "digest/sha1"
 require "zlib"
 require "time"
+require "digest"
 require "droonga/error_messages"
 require "droonga/catalog/errors"
 
@@ -37,7 +38,16 @@ module Droonga
         datasets[name]
       end
 
+      def cluster_id
+        @cluster_id ||= calculate_cluster_id
+      end
+
       private
+      def calculate_cluster_id
+        raw_id = all_nodes.sort.join(",")
+        Digest::SHA1.hexdigest(raw_id)
+      end
+
       def migrate_database_location(current_db_path, device, name)
         return if current_db_path.exist?
 

data/lib/droonga/catalog_fetcher.rb

@@ -13,6 +13,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "socket"
+
 require "droonga/client"
 
 require "droonga/address"
@@ -43,7 +45,7 @@ module Droonga
         :tag           => Address::DEFAULT_TAG,
         :protocol      => :droonga,
         :timeout       => 1,
-        :receiver_host => "127.0.0.1",
+        :receiver_host => Socket.gethostname,
         :receiver_port => 0,
       }
     end

data/lib/droonga/command/droonga_engine.rb

@@ -382,6 +382,7 @@ module Droonga
           catalog_observer = FileObserver.new(@loop, Path.catalog)
           catalog_observer.on_change = lambda do
             restart_graceful
+            @serf.update_cluster_id if @serf and @serf.running?
           end
           catalog_observer.start
           catalog_observer

data/lib/droonga/command/remote.rb

@@ -327,7 +327,6 @@ module Droonga
           end
 
           @serf.join(*hosts)
-          #XXX Now we should restart serf agent to remove unjoined nodes from the list of members...
         end
       end
 
@@ -359,8 +358,28 @@ module Droonga
             modifier.datasets[dataset].replicas.hosts -= hosts
             @service_installation.ensure_correct_file_permission(file)
           end
+        end
+      end
+
+      class Unjoin < ModifyReplicasBase
+        def process
+          return if dataset.nil? or hosts.nil?
+
+          log("unjoining replicas: #{hosts.join(",")}")
 
-          #XXX Now we should restart serf agent to remove unjoined nodes from the list of members...
+          CatalogModifier.modify do |modifier, file|
+            if unjoining_node?
+              modifier.datasets[dataset].replicas.hosts = hosts
+            else
+              modifier.datasets[dataset].replicas.hosts -= hosts
+            end
+            @service_installation.ensure_correct_file_permission(file)
+          end
+        end
+
+        private
+        def unjoining_node?
+          hosts.include?(host)
         end
       end
 

data/lib/droonga/command/serf_event_handler.rb

@@ -77,6 +77,8 @@ module Droonga
           Remote::SetStatus
         when "join"
           Remote::Join
+        when "unjoin"
+          Remote::Unjoin
         when "set_replicas"
           Remote::SetReplicas
         when "add_replicas"

data/lib/droonga/data_absorber.rb

@@ -52,6 +52,7 @@ module Droonga
 
       @source_host      = @params[:source_host]
       @destination_host = @params[:destination_host]
+      @receiver_host    = @params[:receiver_host] || @destination_host
 
       @receiver_port = @params[:receiver_port]
     end
@@ -115,7 +116,7 @@ module Droonga
         :port          => @port,
         :tag           => @tag,
         :progocol      => :droonga,
-        :receiver_host => @destination_host,
+        :receiver_host => @receiver_host,
         :receiver_port => 0,
       }
       @source_client ||= Droonga::Client.new(options)
@@ -127,7 +128,7 @@ module Droonga
         :port          => @port,
         :tag           => @tag,
         :progocol      => :droonga,
-        :receiver_host => @destination_host,
+        :receiver_host => @receiver_host,
         :receiver_port => 0,
       }
       @destination_client ||= Droonga::Client.new(options)
@@ -152,7 +153,7 @@ module Droonga
       options += ["--port", @port]
       options += ["--tag", @tag]
       options += ["--dataset", @dataset]
-      options += ["--receiver-host", @destination_host]
+      options += ["--receiver-host", @receiver_host]
       options += ["--receiver-port", @receiver_port] if @receiver_port
       options.collect(&:to_s)
     end
@@ -162,7 +163,7 @@ module Droonga
       options += ["--host", @destination_host]
       options += ["--port", @port]
       options += ["--tag", @tag]
-      options += ["--receiver-host", @destination_host]
+      options += ["--receiver-host", @receiver_host]
       options += ["--receiver-port", @receiver_port] if @receiver_port
       options.collect(&:to_s)
     end
@@ -252,7 +253,7 @@ module Droonga
       fetcher = CatalogFetcher.new(:host          => @source_host,
                                    :port          => @port,
                                    :tag           => @tag,
-                                   :receiver_host => @destination_host)
+                                   :receiver_host => @receiver_host)
       fetcher.fetch(:dataset => @dataset)
     end
 

data/lib/droonga/engine/version.rb

@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.0.8"
+    VERSION = "1.0.9"
   end
 end

data/lib/droonga/serf.rb

@@ -71,6 +71,7 @@ module Droonga
                    "-event-handler", "droonga-engine-serf-event-handler",
                    "-log-level", log_level,
                    "-tag", "role=engine",
+                   "-tag", "cluster_id=#{cluster_id}",
                    *retry_joins)
       logger.trace("start: done")
     end
@@ -87,6 +88,13 @@ module Droonga
       logger.trace("stop: done")
     end
 
+    def restart
+      logger.trace("restart: start")
+      stop
+      start
+      logger.trace("restart: done")
+    end
+
     def join(*hosts)
       ensure_serf
       nodes = hosts.collect do |host|
@@ -123,8 +131,10 @@ module Droonga
       result = run_once("members", "-format", "json")
       result[:result] = JSON.parse(result[:result])
       members = result[:result]
+      current_cluster_id = cluster_id
       members["members"].each do |member|
-        if member["status"] == "alive"
+        if member["status"] == "alive" and
+           member["tags"]["cluster_id"] == current_cluster_id
           nodes[member["name"]] = {
             "serfAddress" => member["addr"],
             "tags"        => member["tags"],
@@ -134,6 +144,21 @@ module Droonga
       nodes
     end
 
+    def set_tag(name, value)
+      ensure_serf
+      run_once("tags", "-set", "#{name}=#{value}")
+    end
+
+    def update_cluster_id
+      set_tag("cluster_id", cluster_id)
+    end
+
+    def cluster_id
+      loader = CatalogLoader.new(Path.catalog.to_s)
+      catalog = loader.load
+      catalog.cluster_id
+    end
+
     private
     def ensure_serf
       @serf = find_system_serf

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: droonga-engine
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
 platform: ruby
 authors:
 - Droonga Project
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-28 00:00:00.000000000 Z
+date: 2014-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: archive-zip
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.1.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.1.9
 - !ruby/object:Gem::Dependency
   name: droonga-message-pack-packer
   requirement: !ruby/object:Gem::Requirement