droonga-engine 1.0.9 → 1.1.0

data/bin/droonga-engine-catalog-generate

@@ -21,17 +21,17 @@ require "json"
 require "pathname"
 
 require "droonga/engine/version"
-require "droonga/catalog_generator"
+require "droonga/catalog/generator"
 require "droonga/safe_file_writer"
 require "droonga/service_installation"
 
 service_installation = Droonga::ServiceInstallation.new
 service_installation.ensure_using_service_base_directory
 
-generator = Droonga::CatalogGenerator.new
+generator = Droonga::Catalog::Generator.new
 current_dataset = {}
 datasets = {
-  Droonga::CatalogGenerator::DEFAULT_DATASET => current_dataset
+  Droonga::Catalog::Generator::DEFAULT_DATASET => current_dataset
 }
 
 options = OpenStruct.new
@@ -51,37 +51,37 @@ end
 parser.on("--dataset=NAME",
           "Add a dataset its name is NAME.",
           "And set the NAME to the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_DATASET})") do |name|
+          "(#{Droonga::Catalog::Generator::DEFAULT_DATASET})") do |name|
   current_dataset = datasets[name] = {}
 end
 parser.on("--n-workers=N", Integer,
           "Use N workers for the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_N_WORKERS})") do |n|
+          "(#{Droonga::Catalog::Generator::DEFAULT_N_WORKERS})") do |n|
   current_dataset[:n_workers] = n
 end
 parser.on("--hosts=NAME1,NAME2,...", Array,
           "Use given hosts for replicas of the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_HOSTS.join(",")})") do |hosts|
+          "(#{Droonga::Catalog::Generator::DEFAULT_HOSTS.join(",")})") do |hosts|
   current_dataset[:hosts] = hosts
 end
 parser.on("--port=PORT", Integer,
           "Use the PORT as the port for the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_PORT})") do |port|
+          "(#{Droonga::Catalog::Generator::DEFAULT_PORT})") do |port|
   current_dataset[:port] = port
 end
 parser.on("--tag=TAG",
           "Use the TAG as the tag for the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_TAG})") do |tag|
+          "(#{Droonga::Catalog::Generator::DEFAULT_TAG})") do |tag|
   current_dataset[:tag] = tag
 end
 parser.on("--n-slices=N", Integer,
           "Use N slices for each replica.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_N_SLICES})") do |n|
+          "(#{Droonga::Catalog::Generator::DEFAULT_N_SLICES})") do |n|
   current_dataset[:n_slices] = n
 end
 parser.on("--plugins=PLUGIN1,PLUGIN2,...", Array,
           "Use PLUGINS for the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_PLUGINS.join(",")})") do |plugins|
+          "(#{Droonga::Catalog::Generator::DEFAULT_PLUGINS.join(",")})") do |plugins|
   current_dataset[:plugins] = plugins
 end
 parser.on("--schema=PATH",
@@ -102,8 +102,8 @@ parser.on("--replicas=PATH",
 end
 parser.parse!(ARGV)
 
-if datasets[Droonga::CatalogGenerator::DEFAULT_DATASET].empty?
-  datasets.delete(Droonga::CatalogGenerator::DEFAULT_DATASET)
+if datasets[Droonga::Catalog::Generator::DEFAULT_DATASET].empty?
+  datasets.delete(Droonga::Catalog::Generator::DEFAULT_DATASET)
 end
 
 if service_installation.user_exist? and

data/bin/droonga-engine-catalog-modify

@@ -21,17 +21,17 @@ require "json"
 require "pathname"
 
 require "droonga/engine/version"
-require "droonga/catalog_generator"
+require "droonga/catalog/generator"
 require "droonga/safe_file_writer"
 require "droonga/service_installation"
 
 service_installation = Droonga::ServiceInstallation.new
 service_installation.ensure_using_service_base_directory
 
-generator = Droonga::CatalogGenerator.new
+generator = Droonga::Catalog::Generator.new
 current_dataset = {}
 datasets = {
-  Droonga::CatalogGenerator::DEFAULT_DATASET => current_dataset
+  Droonga::Catalog::Generator::DEFAULT_DATASET => current_dataset
 }
 
 options = OpenStruct.new
@@ -65,7 +65,7 @@ end
 parser.on("--dataset=NAME",
           "Add a dataset its name is NAME.",
           "And set the NAME to the current dataset.",
-          "(#{Droonga::CatalogGenerator::DEFAULT_DATASET})") do |name|
+          "(#{Droonga::Catalog::Generator::DEFAULT_DATASET})") do |name|
   current_dataset = datasets[name] = {}
 end
 parser.on("--replica-hosts=NAME1,NAME2,...", Array,

data/bin/droonga-engine-join

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,207 +19,388 @@ require "slop"
 require "json"
 require "pathname"
 require "socket"
+require "coolio"
 
 require "droonga/engine/version"
 require "droonga/path"
-require "droonga/catalog_generator"
+require "droonga/node_name"
+require "droonga/node_role"
+require "droonga/catalog/dataset"
+require "droonga/catalog/fetcher"
+require "droonga/catalog/loader"
 require "droonga/safe_file_writer"
-require "droonga/data_absorber"
+require "droonga/data_absorber_client"
 require "droonga/serf"
 
-class JoinCommand
-  def run
-    parse_options
-    trap_signals
-
-    puts "Start to join a new node #{@options[:host]}"
-    puts "       to the cluster of #{@options["replica-source-host"]}"
-    puts "                     via #{@options["receiver-host"]} (this host)"
-    puts ""
-
-    set_node_role
-    do_join
-    sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
-    do_copy unless @options["no-copy"]
-    set_effective_message_timestamp
-    update_other_nodes
-    reset_node_role
-    puts("Done.")
-    exit(true)
-  end
+module Droonga
+  module Command
+    class Join
+    class MissingRequiredParameter < StandardError
+    end
 
-  private
-  def parse_options
-    options = Slop.parse(:help => true) do |option|
-      option.on("no-copy", "Don't copy data from the source cluster.",
-                :default => false)
-
-      option.separator("Connections:")
-      option.on(:host=,
-                "Host name of the new node to be joined.",
-                :required => true)
-      option.on("replica-source-host=",
-                "Host name of the soruce node in the cluster to be connected.",
-                :required => true)
-      option.on("receiver-host=",
-                "Host name of this host.",
-                :default => Socket.gethostname)
-      option.on(:dataset=,
-                "Dataset name of for the node to be joined.",
-                :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
-      option.on(:port=,
-                "Port number of the source cluster to be connected.",
-                :as => Integer,
-                :default => Droonga::CatalogGenerator::DEFAULT_PORT)
-      option.on(:tag=,
-                "Tag name of the soruce cluster to be connected.",
-                :default => Droonga::CatalogGenerator::DEFAULT_TAG)
-      option.on("records-per-second=",
-                "Maximum number of records per second to be copied. " +
-                  "'#{Droonga::Client::RateLimiter::NO_LIMIT}' means no limit.",
-                :as => Integer,
-                :default => Droonga::DataAbsorber::DEFAULT_MESSAGES_PER_SECOND)
-    end
-    @options = options
-  rescue Slop::MissingOptionError => error
-    $stderr.puts(error)
-    exit(false)
-  end
+    def run
+      @loop = Coolio::Loop.default
 
-  def joining_node
-    "#{@options[:host]}:#{@options[:port]}/#{@options[:tag]}"
-  end
+      parse_options
+      trap_signals
 
-  def source_node
-    "#{@options["replica-source-host"]}:#{@options[:port]}/#{@options[:tag]}"
-  end
+      puts "Start to join a new node #{joining_node.host}"
+      puts "       to the cluster of #{source_node.host}"
+      puts "                     via #{@options["receiver-host"]} (this host)"
+      puts "    port    = #{joining_node.port}"
+      puts "    tag     = #{joining_node.tag}"
+      puts "    dataset = #{dataset}"
+      puts ""
 
-  def run_remote_command(target, command, options)
-    serf = Droonga::Serf.new(nil, target)
-    result = serf.send_query(command, options)
-    #puts(result[:result])
-    puts(result[:error]) unless result[:error].empty?
-    result[:response]
-  end
+      if should_copy? and not absorber.empty_destination?
+        $stderr.puts("Error: The joining node's dataset #{dataset} is not empty.")
+        $stderr.puts("       You must clear all data of the node before joining.")
+        return false
+      end
 
-  def absorber
-    @absorber ||= prepare_absorber
-  end
+      puts "Source Cluster ID: #{source_cluster_id}"
+      puts ""
 
-  def prepare_absorber
-    absorber_options = {
-      :dataset          => @options[:dataset],
-      :source_host      => @options["replica-source-host"],
-      :destination_host => @options[:host],
-      :receiver_host    => @options["receiver-host"],
-      :port             => @options[:port],
-      :tag              => @options[:tag],
-      :messages_per_second => @options["records-per-second"],
-    }
-    Droonga::DataAbsorber.new(absorber_options)
-  end
+      begin
+        set_joining_node_role
+        do_join
+        register_to_existing_nodes
+        set_source_node_role
+        if should_copy?
+          #XXX If any command is received by the source node after changing of its role,
+          #    the timestamp of last processed mesasge is unexpectedly updated by them.
+          #    Be careful to not send any command to the source node on this timing!!
+          update_accept_messages_newer_than_timestamp
+          successed = copy_data
+          unless successed
+            do_cancel
+            return false
+          end
+        end
+        reset_source_node_role
+        reset_joining_node_role
+        puts("Done.")
+        true
+      rescue Exception => exception
+        puts("Unexpected exception: #{exception.message}")
+        puts(exception.backtrace.join("\n"))
+        do_cancel
+        false
+      end
+    end
+
+    private
+    def parse_options
+      options = Slop.parse(:help => true) do |option|
+        option.on("no-copy", "Don't copy data from the source cluster.",
+                  :default => false)
 
-  def set_node_role
-    if absorber.source_node_suspendable?
-      run_remote_command(source_node, "change_role",
-                         "node" => source_node,
-                         "role" => "source")
+        option.separator("Target:")
+        option.on(:host=,
+                  "Host name of the new node to be joined.",
+                  :required => true)
+        option.on("replica-source-host=",
+                  "Host name of the soruce node in the cluster to be connected.",
+                  :required => true)
+
+        option.on(:port=,
+                  "Port number of the source cluster to be connected.",
+                  :as => Integer,
+                  :default => NodeName::DEFAULT_PORT)
+        option.on(:tag=,
+                  "Tag name of the soruce cluster to be connected.",
+                  :default => NodeName::DEFAULT_TAG)
+        option.on(:dataset=,
+                  "Dataset name of for the node to be joined.",
+                  :default => Catalog::Dataset::DEFAULT_NAME)
+
+        option.separator("Connections:")
+        option.on("receiver-host=",
+                  "Host name of this host.",
+                  :default => Socket.gethostname)
+
+        option.separator("Miscellaneous:")
+        option.on("records-per-second=",
+                  "Maximum number of records per second to be copied. " +
+                    "'#{Client::RateLimiter::NO_LIMIT}' means no limit.",
+                  :as => Integer,
+                  :default => DataAbsorberClient::DEFAULT_MESSAGES_PER_SECOND)
+        option.on("progress-interval-seconds=",
+                  "Interval seconds to report progress.",
+                  :as => Integer,
+                  :default => DataAbsorberClient::DEFAULT_PROGRESS_INTERVAL_SECONDS)
+        option.on(:verbose, "Output details for internal operations.",
+                  :default => false)
+      end
+      @options = options
+    rescue Slop::MissingOptionError => error
+      $stderr.puts(error)
+      raise MissingRequiredParameter.new
     end
-    run_remote_command(joining_node, "change_role",
-                       "node" => joining_node,
-                       "role" => "destination")
-    @node_role_changed = true
-  end
 
-  def reset_node_role
-    if absorber.source_node_suspendable?
-      run_remote_command(source_node, "change_role",
-                         "node" => source_node,
-                         "role" => "")
+    def dataset
+      @options[:dataset]
     end
-    run_remote_command(joining_node, "change_role",
-                       "node" => joining_node,
-                       "role" => "")
-    @node_role_changed = false
-  end
 
-  def do_join
-    puts("Joining new replica to the cluster...")
-    run_remote_command(joining_node, "join",
-                       "node"    => joining_node,
-                       "type"    => "replica",
-                       "source"  => source_node,
-                       "dataset" => @options[:dataset],
-                       "copy"    => !@options["no-copy"])
-  end
+    def should_copy?
+      not @options["no-copy"]
+    end
+
+    def joining_node
+      @joining_node ||= NodeName.new(:host => @options[:host],
+                                     :port => @options[:port],
+                                     :tag  => @options[:tag])
+    end
+
+    def source_node
+      @source_node ||= NodeName.new(:host => @options["replica-source-host"],
+                                    :port => @options[:port],
+                                    :tag  => @options[:tag])
+    end
+
+    def source_node_serf
+      @source_node_serf ||= Serf.new(source_node.to_s,
+                                     :verbose => @options[:verbose])
+    end
+
+    def joining_node_serf
+      @joining_node_serf ||= Serf.new(joining_node.to_s,
+                                      :verbose => @options[:verbose])
+    end
+
+    def source_cluster_id
+      source_catalog.cluster_id
+    end
+
+    def all_nodes
+      existing_nodes + [joining_node]
+    end
+
+    def existing_nodes
+      @existing_nodes ||= prepare_existing_nodes
+    end
+
+    def prepare_existing_nodes
+      generator = Catalog::Generator.new
+      generator.load(raw_source_catalog)
+
+      dataset = generator.dataset_for_host(source_node.host)
+      other_hosts = dataset.replicas.hosts
+      other_hosts.collect do |host|
+        NodeName.new(:host => host,
+                     :port => source_node.port,
+                     :tag  => source_node.tag)
+      end
+    end
+
+    def source_catalog
+      @source_catalog ||= parse_source_catalog
+    end
+
+    def parse_source_catalog
+      loader = Catalog::Loader.new
+      loader.parse(raw_source_catalog)
+    end
+
+    def raw_source_catalog
+      @raw_source_catalog ||= fetch_source_catalog
+    end
+
+    def fetch_source_catalog
+      fetcher = Catalog::Fetcher.new(:host          => source_node.host,
+                                     :port          => source_node.port,
+                                     :tag           => source_node.tag,
+                                     :receiver_host => @options["receiver-host"])
+      fetcher.fetch(:dataset => dataset)
+    end
+
+    def absorber
+      @absorber ||= prepare_absorber
+    end
+
+    def prepare_absorber
+      absorber_options = {
+        :host    => joining_node.host,
+        :port    => joining_node.port,
+        :tag     => joining_node.tag,
+        :dataset => dataset,
+
+        :source_host    => source_node.host,
+        :source_port    => source_node.port,
+        :source_tag     => source_node.tag,
+        :source_dataset => dataset,
+
+        :receiver_host    => @options["receiver-host"],
+
+        :messages_per_second => @options["records-per-second"],
+        :progress_interval_seconds => @options["progress-interval-seconds"],
+        :target_role => NodeRole::ABSORB_DESTINATION,
+
+        :client_options   => {
+          :backend => :coolio,
+          :loop    => @loop,
+        },
+      }
+      DataAbsorberClient.new(absorber_options)
+    end
+
+    def set_source_node_role
+      return if source_node_serf.role == NodeRole::ABSORB_SOURCE
+      if absorber.source_node_suspendable?
+        puts("Changing role of the source node...")
+        source_node_serf.ensure_restarted do
+          source_node_serf.send_query("change_role",
+                                      "node" => source_node.to_s,
+                                      "role" => NodeRole::ABSORB_SOURCE)
+        end
+      end
+    end
 
-  def do_copy
-    @start_time_in_seconds = Time.new.to_i
-    puts("Copying data from the source node...")
-    last_progress = ""
-    while true
-      sleep(3)
-      response = run_remote_command(joining_node, "report_status",
-                                    "node" => joining_node,
-                                    "key"  => "absorbing")
-      if response
-        absorbing = response["value"]
-        break unless absorbing
-      end
-
-      progress = absorber.report_progress(@start_time_in_seconds)
-      if progress
-        printf("%s", "#{" " * last_progress.size}\r")
-        printf("%s", "#{progress}\r")
+    def set_joining_node_role
+      return if joining_node_serf.role == NodeRole::ABSORB_DESTINATION
+      puts("Changing role of the joining node...")
+      joining_node_serf.ensure_restarted do
+        joining_node_serf.send_query("change_role",
+                                     "node" => joining_node.to_s,
+                                     "role" => NodeRole::ABSORB_DESTINATION)
+      end
+    end
+
+    def reset_source_node_role
+      return if source_node_serf.role == NodeRole::SERVICE_PROVIDER
+      if absorber.source_node_suspendable?
+        puts("Restoring role of the source node...")
+        source_node_serf.ensure_restarted do
+          source_node_serf.send_query("change_role",
+                                      "node" => source_node.to_s,
+                                      "role" => NodeRole::SERVICE_PROVIDER)
+        end
+      end
+    end
+
+    def reset_joining_node_role
+      return if joining_node_serf.role == NodeRole::SERVICE_PROVIDER
+      puts("Restoring role of the joining node...")
+      joining_node_serf.ensure_restarted do
+        joining_node_serf.send_query("change_role",
+                                     "node" => joining_node.to_s,
+                                     "role" => NodeRole::SERVICE_PROVIDER)
+      end
+    end
+
+    def do_join
+      puts("Configuring the joining node as a new replica for the cluster...")
+      joining_node_serf.ensure_restarted do
+        joining_node_serf.send_query("join",
+                                     "node"    => joining_node.to_s,
+                                     "type"    => "replica",
+                                     "source"  => source_node.to_s,
+                                     "dataset" => dataset)
+      end
+    end
+
+    def copy_data
+      puts("Copying data from the source node...")
+
+      last_progress = nil
+      absorber.run do |progress|
+        if last_progress
+          printf("%s", "#{" " * last_progress[:message].size}\r")
+        end
+        printf("%s", "#{progress[:message]}\r")
         last_progress = progress
       end
+      @loop.run
+
+      if absorber.error_message
+        puts(absorber.error_message)
+        return false
+      end
+
+      puts ""
+      true
     end
-    puts ""
-  end
 
-  def set_effective_message_timestamp
-    response = run_remote_command(source_node, "report_status",
-                                  "node" => source_node,
-                                  "key" => "last_processed_message_timestamp")
-    timestamp = response["value"]
-    if timestamp and not timestamp.empty?
-      puts "The timestamp of the last processed message in the source node: #{timestamp}"
-      puts "Setting effective message timestamp for the destination node..."
-      response = run_remote_command(joining_node, "set_status",
-                                    "node" => joining_node,
-                                    "key" => "effective_message_timestamp",
-                                    "value" => timestamp)
+    GETTING_LAST_MESSAGE_TIMESTAMP_MAX_RETRY_COUNT = 10
+    GETTING_LAST_MESSAGE_TIMESTAMP_RETRY_INTERVAL_SECONDS = 10
+
+    def try_get_last_message_timestamp(retry_count=0)
+      puts "Getting the timestamp of the last processed message in the source node..."
+      timestamp = source_node_serf.last_message_timestamp
+      unless timestamp
+        if retry_count < GETTING_LAST_MESSAGE_TIMESTAMP_MAX_RETRY_COUNT
+          puts "Failed. Retrying..."
+          sleep(GETTING_LAST_MESSAGE_TIMESTAMP_RETRY_INTERVAL_SECONDS)
+          timestamp = try_get_last_message_timestamp(retry_count + 1)
+        end
+      end
+      timestamp
     end
-  end
 
-  def update_other_nodes
-    puts("Update existing hosts in the cluster...")
-    run_remote_command(source_node, "add_replicas",
-                       "dataset" => @options[:dataset],
-                       "hosts"   => [@options[:host]])
-  end
+    def update_accept_messages_newer_than_timestamp
+      timestamp = try_get_last_message_timestamp
+      if timestamp and not timestamp.empty?
+        puts "The timestamp of the last processed message at the source node: #{timestamp}"
+        puts "Setting new node to ignore messages older than the timestamp..."
+        joining_node_serf.ensure_restarted do
+          joining_node_serf.send_query("accept_messages_newer_than",
+                                       "node"      => joining_node.to_s,
+                                       "timestamp" => timestamp)
+        end
+      else
+        $stderr.puts("WARNING: Couldn't get the time stamp of " +
+                       "the last processed message from the source node. " +
+                       "Any message will be forwarded to the joining node.")
+      end
+    end
 
-  def trap_signals
-    trap(:TERM) do
-      do_cancel
-      trap(:TERM, "DEFAULT")
+    def register_to_existing_nodes
+      puts("Registering new node to existing nodes...")
+      source_node_serf.ensure_restarted(*existing_nodes) do
+        source_node_serf.send_query("add_replicas",
+                                    "cluster_id" => source_cluster_id,
+                                    "dataset"    => dataset,
+                                    "hosts"      => [joining_node.host])
+      end
+      @node_registered = true
     end
 
-    trap(:INT) do
-      do_cancel
-      trap(:INT, "DEFAULT")
+    def unregister_from_existing_nodes
+      puts("Unregistering new node from existing nodes...")
+      source_node_serf.ensure_restarted(*existing_nodes) do
+        source_node_serf.send_query("remove_replicas",
+                                    "cluster_id" => source_cluster_id,
+                                    "dataset"    => dataset,
+                                    "hosts"      => [joining_node.host])
+      end
+      @node_registered = false
     end
 
-    trap(:QUIT) do
-      do_cancel
-      trap(:QUIT, "DEFAULT")
+    def trap_signals
+      trap(:TERM) do
+        trap(:TERM, "DEFAULT")
+        do_cancel
+      end
+
+      trap(:INT) do
+        trap(:INT, "DEFAULT")
+        do_cancel
+      end
+
+      trap(:QUIT) do
+        trap(:QUIT, "DEFAULT")
+        do_cancel
+      end
     end
-  end
 
-  def do_cancel
-    #XXX we have to write more codes to cancel remote processes!
-    reset_node_role if @node_role_changed
+    def do_cancel
+      #XXX we have to write more codes to cancel remote processes!
+      unregister_from_existing_nodes if @node_registered
+      reset_joining_node_role
+      reset_source_node_role
+    end
+    end
   end
 end
 
-JoinCommand.new.run
+exit(Droonga::Command::Join.new.run)