droonga-engine 1.0.7 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9d356f5ce7ea305000e8d982a0bacd9c02c1db4
-  data.tar.gz: 700367d322a8b3df05ed73a6b57782d30568bf8f
+  metadata.gz: e7b344e7b567ecc524d5f5cfb52ed6a67342a672
+  data.tar.gz: 79ecc5c743f0f01f350a1855d55368dc024515a9
 SHA512:
-  metadata.gz: b99cb2c1e4fa3df7d96700499277e37c6c7c37ba3eabeba5eb0721232712911d1cb691fc10f3159196b5633224d96f47fc66f4e279479643042282d2b3bc0654
-  data.tar.gz: 766e53f931fa1ea0c5ca58614bdaa9627cd382b84be60673cdcd35a9842739d097d9b3b9f9105eb6b675fec6cdbe480994b7ededd7802d287d512cc6234096b5
+  metadata.gz: 1f94a76e05f59f76bbd6e92774bd928c10a01c8c731df1e9b548a0b255b61abfab538e3c374d1e1fd74244639cb140d9913ee09649226a2226ad54d458afd0a2
+  data.tar.gz: 343bf2deb3d726858be67b61f222689535b939ea1ecce9d564f8f69a474e41cd22190600cf95e0b2e42f3e609ff244f44fcd9fafc946901fb2a89197de33a0b7

data/.travis.yml

@@ -13,6 +13,6 @@ rvm:
 before_install:
 #  - GROONGA_MASTER=yes curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
   - curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
-  - curl --location --remote-name https://dl.bintray.com/mitchellh/serf/0.6.0_linux_amd64.zip
-  - unzip 0.6.0_linux_amd64.zip
+  - curl --location --remote-name https://dl.bintray.com/mitchellh/serf/0.6.3_linux_amd64.zip
+  - unzip 0.6.3_linux_amd64.zip
   - sudo install serf /usr/local/bin/

data/bin/droonga-engine-absorb-data

@@ -24,100 +24,214 @@ require "droonga/catalog_generator"
 require "droonga/path"
 require "droonga/data_absorber"
 require "droonga/serf"
+require "droonga/client"
 
-options = OpenStruct.new
-options.port    = Droonga::CatalogGenerator::DEFAULT_PORT
-options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
-options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
-options.remote  = true
-parser = OptionParser.new
-parser.version = Droonga::Engine::VERSION
-
-parser.separator("")
-parser.separator("Connection:")
-parser.on("--source-host=HOST",
-          "Host name of the source cluster to be connected.") do |host|
-  options.source_host = host
-end
-parser.on("--destination-host=HOST",
-          "Host name of this cluster to be connected.") do |host|
-  options.destination_host = host
-end
-parser.on("--port=PORT", Integer,
-          "Port number of the source cluster to be connected.",
-          "(#{options.port})") do |port|
-  options.port = port
-end
-parser.on("--[no-]remote",
-          "Run command in remote node or not.",
-          "(#{options.remote})") do |remote|
-  options.remote = remote
-end
+class AbsorbDataCommand
+  def run
+    parse_options
+    assert_valid_options
+    trap_signals
 
-parser.separator("")
-parser.separator("Data:")
-parser.on("--tag=TAG",
-          "Tag name to be used to communicate with Droonga system.",
-          "(#{options.tag})") do |tag|
-  options.tag = tag
-end
-parser.on("--dataset=DATASET",
-          "Dataset to be absorbed.",
-          "(#{options.dataset})") do |dataset|
-  options.dataset = dataset
-end
+    puts "Start to absorb data from #{@options.source_host}"
+    puts "                       to #{@options.destination_host}"
+    puts "  dataset = #{@options.dataset}"
+    puts "  port    = #{@options.port}"
+    puts "  tag     = #{@options.tag}"
+    puts ""
+    puts "Absorbing..."
 
-parser.parse!(ARGV)
+    if @options.remote
+      absorb_on_remote
+    else
+      absorb_on_local
+    end
 
-unless options.source_host
-  raise "You must specify the source host via --source-host option."
-end
-unless options.destination_host
-  raise "You must specify the destination host via --destination-host option."
-end
+    puts "Done."
+    exit(true)
+  end
 
-destination_node = "#{options.destination_host}:#{options.port}/#{options.tag}"
+  private
+  def parse_options
+    options = OpenStruct.new
+    options.port    = Droonga::CatalogGenerator::DEFAULT_PORT
+    options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
+    options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
+    options.remote  = true
+    options.messages_per_second = Droonga::DataAbsorber::DEFAULT_MESSAGES_PER_SECOND
+    parser = OptionParser.new
+    parser.version = Droonga::Engine::VERSION
 
-def run_remote_command(target, command, options)
-  serf = Droonga::Serf.new(nil, target)
-  result = serf.send_query(command, options)
-  puts result[:result]
-  puts result[:error] unless result[:error].empty?
-  result[:response]
-end
+    parser.separator("")
+    parser.separator("Connection:")
+    parser.on("--source-host=HOST",
+              "Host name of the source cluster to be connected.") do |host|
+      options.source_host = host
+    end
+    parser.on("--destination-host=HOST",
+              "Host name of this cluster to be connected.") do |host|
+      options.destination_host = host
+    end
+    parser.on("--port=PORT", Integer,
+              "Port number of the source cluster to be connected.",
+              "(#{options.port})") do |port|
+      options.port = port
+    end
+    parser.on("--[no-]remote",
+              "Run command in remote node or not.",
+              "(#{options.remote})") do |remote|
+      options.remote = remote
+    end
+
+    parser.separator("")
+    parser.separator("Data:")
+    parser.on("--tag=TAG",
+              "Tag name to be used to communicate with Droonga system.",
+              "(#{options.tag})") do |tag|
+      options.tag = tag
+    end
+    parser.on("--dataset=DATASET",
+              "Dataset to be absorbed.",
+              "(#{options.dataset})") do |dataset|
+      options.dataset = dataset
+    end
+    parser.on("--records-per-second=N", Integer,
+              "Maximum number of records per second to be absorbed.",
+              "'#{Droonga::Client::RateLimiter::NO_LIMIT}' means no limit.",
+              "(#{options.messages_per_second})") do |n|
+      options.messages_per_second = n
+    end
+
+    parser.parse!(ARGV)
+    @options = options
+  end
+
+  def assert_valid_options
+    unless @options.source_host
+      raise "You must specify the source host via --source-host option."
+    end
+    unless @options.destination_host
+      raise "You must specify the destination host via --destination-host option."
+    end
+  end
+
+  def source_node
+    "#{@options.source_host}:#{@options.port}/#{@options.tag}"
+  end
 
-puts "Start to absorb data from #{options.source_host}"
-puts "                       to #{options.destination_host}"
-puts "  dataset = #{options.dataset}"
-puts "  port    = #{options.port}"
-puts "  tag     = #{options.tag}"
-puts ""
-puts "Absorbing..."
-
-if options.remote
-  run_remote_command(destination_node, "absorb_data",
-                     "node"    => destination_node,
-                     "source"  => options.source_host,
-                     "port"    => options.port,
-                     "tag"     => options.tag,
-                     "dataset" => options.dataset)
-  while true
-    sleep(3)
-    response = run_remote_command(destination_node, "report_status",
-                                  "node" => destination_node,
-                                  "key" => "absorbing")
-    absorbing = response["value"]
-    break unless absorbing
+  def destination_node
+    "#{@options.destination_host}:#{@options.port}/#{@options.tag}"
   end
-else
-  Droonga::DataAbsorber.absorb(:dataset          => options.dataset,
-                               :source_host      => options.source_host,
-                               :destination_host => options.destination_host,
-                               :port             => options.port,
-                               :tag              => options.tag) do |output|
-    puts output
+
+  def run_remote_command(target, command, options)
+    serf = Droonga::Serf.new(nil, target)
+    result = serf.send_query(command, options)
+    #puts result[:result]
+    puts result[:error] unless result[:error].empty?
+    result[:response]
+  end
+
+  def absorber
+    @absorber ||= prepare_absorber
+  end
+
+  def prepare_absorber
+    absorber_options = {
+      :dataset          => @options.dataset,
+      :source_host      => @options.source_host,
+      :destination_host => @options.destination_host,
+      :port             => @options.port,
+      :tag              => @options.tag,
+      :messages_per_second => @options.messages_per_second,
+    }
+    Droonga::DataAbsorber.new(absorber_options)
+  end
+
+  def absorb_on_remote
+    start_time_in_seconds = Time.new.to_i
+    run_remote_command(destination_node, "absorb_data",
+                       "node"    => destination_node,
+                       "source"  => @options.source_host,
+                       "port"    => @options.port,
+                       "tag"     => @options.tag,
+                       "dataset" => @options.dataset,
+                       "messages_per_second" => @options.messages_per_second)
+    last_progress = ""
+    while true
+      sleep(3)
+      response = run_remote_command(destination_node, "report_status",
+                                    "node" => destination_node,
+                                    "key" => "absorbing")
+      if response
+        absorbing = response["value"]
+        break unless absorbing
+      end
+
+      progress = absorber.report_progress(start_time_in_seconds)
+      if progress
+        printf("%s", "#{" " * last_progress.size}\r")
+        printf("%s", "#{progress}\r")
+        last_progress = progress
+      end
+    end
+    puts ""
+
+    response = run_remote_command(source_node, "report_status",
+                                  "node" => source_node,
+                                  "key" => "last_processed_message_timestamp")
+    timestamp = response["value"]
+    if timestamp and not timestamp.empty?
+      puts "The timestamp of the last processed message in the source node: #{timestamp}"
+      puts "Setting effective message timestamp for the destination node..."
+      response = run_remote_command(destination_node, "set_status",
+                                    "node" => destination_node,
+                                    "key" => "effective_message_timestamp",
+                                    "value" => timestamp)
+    end
+  end
+
+  def absorb_on_local
+    last_progress = ""
+    absorber.absorb do |live_status|
+      if live_status[:progress]
+        progress = live_status[:progress]
+      else
+        progress = live_status[:output]
+      end
+      printf("%s", "#{" " * last_progress.size}\r")
+      printf("%s", "#{progress}\r")
+      last_progress = progress
+    end
+    response = run_remote_command(source_node, "report_status",
+                                  "node" => source_node,
+                                  "key" => "last_processed_message_timestamp")
+    timestamp = response["value"]
+    puts "The timestamp of the last processed message in the source node: #{timestamp}"
+    if timestamp and not timestamp.empty?
+      status = NodeStatus.new
+      status.set(:effective_message_timestamp, timestamp)
+    end
+  end
+
+  def trap_signals
+    trap(:TERM) do
+      do_cancel
+      trap(:TERM, "DEFAULT")
+    end
+
+    trap(:INT) do
+      do_cancel
+      trap(:INT, "DEFAULT")
+    end
+
+    trap(:QUIT) do
+      do_cancel
+      trap(:QUIT, "DEFAULT")
+    end
+  end
+
+  def do_cancel
+    #XXX we have to write more codes to cancel remote processes!
   end
 end
-puts "Done."
 
-exit(true)
+AbsorbDataCommand.new.run

data/bin/droonga-engine-configure

@@ -26,7 +26,7 @@ require "droonga/service_installation"
 require "droonga/logger"
 
 options = {
-  :quiet => nil,
+  :no_prompt => nil,
   :clear => nil,
   :reset_config  => nil,
   :reset_catalog => nil,
@@ -34,8 +34,11 @@ options = {
 
 configuration = Droonga::Command::DroongaEngine::Configuration.new
 parser = OptionParser.new
-parser.on("--quiet", "Run with no prompt.") do |host|
-  options[:quiet] = true
+parser.on("--no-prompt", "Run with no prompt.") do |host|
+  options[:no_prompt] = true
+end
+parser.on("--quiet", "Same to --no-prompt. For backward compatibility.") do |host|
+  options[:no_prompt] = true
 end
 parser.on("--clear", "Clear any existing data.") do |host|
   options[:clear] = true
@@ -97,7 +100,7 @@ service_installation.ensure_using_service_base_directory
 running = false
 begin
   if service_installation.running?
-    if !options[:quiet]
+    if !options[:no_prompt]
       puts("The droonga-engine service is now running.")
       puts("Before reconfiguration, the service is going to be stopped " +
              "and this node will be unjoined from the cluster.")
@@ -126,19 +129,19 @@ data_files = [
 have_data = data_files.any?(&:exist?)
 options[:clear] = false unless have_data
 
-if !options[:quiet] and options[:clear].nil?
+if !options[:no_prompt] and options[:clear].nil?
   options[:clear] = confirmed?("Do you want all data to be cleared?")
 end
 
 
 options[:reset_config] = true unless Droonga::Path.config.exist?
-if !options[:quiet] and options[:reset_config].nil?
+if !options[:no_prompt] and options[:reset_config].nil?
   options[:reset_config] = confirmed?("Do you want the configuration file " +
                                         "\"droonga-engine.yaml\" to be regenerated?")
 end
 
 options[:reset_catalog] = true unless Droonga::Path.catalog.exist?
-if !options[:quiet] and options[:reset_catalog].nil?
+if !options[:no_prompt] and options[:reset_catalog].nil?
   options[:reset_catalog] = confirmed?("Do you want the file \"catalog.json\" " +
                                          "to be regenerated?")
 end
@@ -156,19 +159,19 @@ if options[:clear]
 end
 
 if options[:reset_config] or options[:reset_catalog]
-  if configuration.have_given_host? or options[:quiet]
+  if options[:no_prompt]
     host = configuration.host
   else
     host = input("host", configuration.host)
   end
 
-  if configuration.have_given_port? or options[:quiet]
+  if options[:no_prompt]
     port = configuration.port
   else
     port = input("port", configuration.port).to_i
   end
 
-  if configuration.have_given_tag? or options[:quiet]
+  if options[:no_prompt]
     tag = configuration.tag
   else
     tag = input("tag", configuration.tag)
@@ -176,7 +179,7 @@ if options[:reset_config] or options[:reset_catalog]
 end
 
 if options[:reset_config]
-  if configuration.have_given_log_level? or options[:quiet]
+  if options[:no_prompt]
     log_level = configuration.log_level
   else
     levels = Droonga::Logger::Level::LABELS

data/bin/droonga-engine-join

@@ -26,70 +26,189 @@ require "droonga/safe_file_writer"
 require "droonga/data_absorber"
 require "droonga/serf"
 
-options = nil
-begin
-  options = Slop.parse(:help => true) do |option|
-    option.on("no-copy", "Don't copy data from the source cluster.",
-              :default => false)
-
-    option.separator("Connections:")
-    option.on(:host=,
-              "Host name of the node to be joined.",
-              :required => true)
-    option.on("replica-source-host=",
-              "Host name of the soruce cluster to be connected.",
-              :required => true)
-    option.on(:dataset=,
-              "Tag dataset name of the cluster to be joined as a node.",
-              :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
-    option.on(:port=,
-              "Port number of the source cluster to be connected.",
-              :as => Integer,
-              :default => Droonga::CatalogGenerator::DEFAULT_PORT)
-    option.on(:tag=,
-              "Tag name of the soruce cluster to be connected.",
-              :default => Droonga::CatalogGenerator::DEFAULT_TAG)
+class JoinCommand
+  def run
+    parse_options
+    trap_signals
+    set_node_role
+    do_join
+    sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
+    do_copy unless @options["no-copy"]
+    set_effective_message_timestamp
+    update_other_nodes
+    reset_node_role
+    puts("Done.")
+    exit(true)
   end
-rescue Slop::MissingOptionError => e
-  $stderr.puts(e)
-  exit(false)
-end
 
-joining_node = "#{options[:host]}:#{options[:port]}/#{options[:tag]}"
-source_node  = "#{options["replica-source-host"]}:#{options[:port]}/#{options[:tag]}"
+  private
+  def parse_options
+    options = Slop.parse(:help => true) do |option|
+      option.on("no-copy", "Don't copy data from the source cluster.",
+                :default => false)
 
-def run_remote_command(target, command, options)
-  serf = Droonga::Serf.new(nil, target)
-  result = serf.send_query(command, options)
-  puts(result[:result])
-  puts(result[:error]) unless result[:error].empty?
-  result[:response]
-end
+      option.separator("Connections:")
+      option.on(:host=,
+                "Host name of the new node to be joined.",
+                :required => true)
+      option.on("replica-source-host=",
+                "Host name of the soruce node in the cluster to be connected.",
+                :required => true)
+      option.on(:dataset=,
+                "Dataset name of for the node to be joined.",
+                :default => Droonga::CatalogGenerator::DEFAULT_DATASET)
+      option.on(:port=,
+                "Port number of the source cluster to be connected.",
+                :as => Integer,
+                :default => Droonga::CatalogGenerator::DEFAULT_PORT)
+      option.on(:tag=,
+                "Tag name of the soruce cluster to be connected.",
+                :default => Droonga::CatalogGenerator::DEFAULT_TAG)
+      option.on("records-per-second=",
+                "Maximum number of records per second to be copied. " +
+                  "'#{Droonga::Client::RateLimiter::NO_LIMIT}' means no limit.",
+                :as => Integer,
+                :default => Droonga::DataAbsorber::DEFAULT_MESSAGES_PER_SECOND)
+    end
+    @options = options
+  rescue Slop::MissingOptionError => error
+    $stderr.puts(error)
+    exit(false)
+  end
 
-puts("Joining new replica to the cluster...")
-run_remote_command(joining_node, "join",
-                   "node"    => joining_node,
-                   "type"    => "replica",
-                   "source"  => source_node,
-                   "dataset" => options[:dataset],
-                   "copy"    => !options["no-copy"])
-sleep(5) #TODO: wait for restarting of the joining node. this should be done more safely.
-
-while true
-  sleep(3)
-  response = run_remote_command(joining_node, "report_status",
-                                "node" => joining_node,
-                                "key"  => "absorbing")
-  absorbing = response["value"]
-  break unless absorbing
-end
+  def joining_node
+    "#{@options[:host]}:#{@options[:port]}/#{@options[:tag]}"
+  end
+
+  def source_node
+    "#{@options["replica-source-host"]}:#{@options[:port]}/#{@options[:tag]}"
+  end
+
+  def run_remote_command(target, command, options)
+    serf = Droonga::Serf.new(nil, target)
+    result = serf.send_query(command, options)
+    #puts(result[:result])
+    puts(result[:error]) unless result[:error].empty?
+    result[:response]
+  end
+
+  def absorber
+    @absorber ||= prepare_absorber
+  end
+
+  def prepare_absorber
+    absorber_options = {
+      :dataset          => @options[:dataset],
+      :source_host      => @options["replica-source-host"],
+      :destination_host => @options[:host],
+      :port             => @options[:port],
+      :tag              => @options[:tag],
+      :messages_per_second => @options["records-per-second"],
+    }
+    Droonga::DataAbsorber.new(absorber_options)
+  end
+
+  def set_node_role
+    if absorber.source_node_suspendable?
+      run_remote_command(source_node, "change_role",
+                         "node" => source_node,
+                         "role" => "source")
+    end
+    run_remote_command(joining_node, "change_role",
+                       "node" => joining_node,
+                       "role" => "destination")
+    @node_role_changed = true
+  end
 
-puts("Update existing hosts in the cluster...")
-run_remote_command(source_node, "add_replicas",
-                   "dataset" => options[:dataset],
-                   "hosts"   => [options[:host]])
+  def reset_node_role
+    if absorber.source_node_suspendable?
+      run_remote_command(source_node, "change_role",
+                         "node" => source_node,
+                         "role" => "")
+    end
+    run_remote_command(joining_node, "change_role",
+                       "node" => joining_node,
+                       "role" => "")
+    @node_role_changed = false
+  end
+
+  def do_join
+    puts("Joining new replica to the cluster...")
+    run_remote_command(joining_node, "join",
+                       "node"    => joining_node,
+                       "type"    => "replica",
+                       "source"  => source_node,
+                       "dataset" => @options[:dataset],
+                       "copy"    => !@options["no-copy"])
+  end
+
+  def do_copy
+    @start_time_in_seconds = Time.new.to_i
+    puts("Copying data from the source node...")
+    last_progress = ""
+    while true
+      sleep(3)
+      response = run_remote_command(joining_node, "report_status",
+                                    "node" => joining_node,
+                                    "key"  => "absorbing")
+      if response
+        absorbing = response["value"]
+        break unless absorbing
+      end
 
+      progress = absorber.report_progress(@start_time_in_seconds)
+      if progress
+        printf("%s", "#{" " * last_progress.size}\r")
+        printf("%s", "#{progress}\r")
+        last_progress = progress
+      end
+    end
+    puts ""
+  end
+
+  def set_effective_message_timestamp
+    response = run_remote_command(source_node, "report_status",
+                                  "node" => source_node,
+                                  "key" => "last_processed_message_timestamp")
+    timestamp = response["value"]
+    if timestamp and not timestamp.empty?
+      puts "The timestamp of the last processed message in the source node: #{timestamp}"
+      puts "Setting effective message timestamp for the destination node..."
+      response = run_remote_command(joining_node, "set_status",
+                                    "node" => joining_node,
+                                    "key" => "effective_message_timestamp",
+                                    "value" => timestamp)
+    end
+  end
 
-puts("Done.")
+  def update_other_nodes
+    puts("Update existing hosts in the cluster...")
+    run_remote_command(source_node, "add_replicas",
+                       "dataset" => @options[:dataset],
+                       "hosts"   => [@options[:host]])
+  end
+
+  def trap_signals
+    trap(:TERM) do
+      do_cancel
+      trap(:TERM, "DEFAULT")
+    end
+
+    trap(:INT) do
+      do_cancel
+      trap(:INT, "DEFAULT")
+    end
+
+    trap(:QUIT) do
+      do_cancel
+      trap(:QUIT, "DEFAULT")
+    end
+  end
+
+  def do_cancel
+    #XXX we have to write more codes to cancel remote processes!
+    reset_node_role if @node_role_changed
+  end
+end
 
-exit(true)
+JoinCommand.new.run