droonga-engine 1.0.9 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2574dab76722981c14a23373e838f1e01d18220
-  data.tar.gz: 9776825164dda315f20af0f0d610d2391cce93fd
+  metadata.gz: bd5aad47933f58aee93072477c535d458556b183
+  data.tar.gz: fc6e82289c99cc3ef4cdc269d0153ca6588dec7f
 SHA512:
-  metadata.gz: 55b359cab3e5ca1bd0b7008233c54bbf413d27b97ee8d179ca09be701fa4f2749935cb37eaf743bfb0099600794c80d319edce22bedde28b69f685d944c8f193
-  data.tar.gz: 09add0504d2886e51065af0c09a4d350db98c8fe07e9a6de6f317ca08af34971016f57c2ac3706d4317df110d125077045c8973c996f4bda382a187cd903ffa0
+  metadata.gz: 08909575ab516f97f404396904a53a0920bdc7a6ad7ab8673e7065d681698dee2f7029c71cb74731824892df0eeb87b8de4383d4aa9d0107823e7a00f363a7dc
+  data.tar.gz: fc2acfa0c8c133bc70ebad99f7d326e96fcc316fdedfb45347431e5ec3a96a751d4f00db03b9c21c4c44d03d26b8da5b0263615926e9321d50f44c60277f0cf9

data/.travis.yml

@@ -9,6 +9,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1
+  - 2.2
 #  - ruby-head
 before_install:
 #  - GROONGA_MASTER=yes curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/benchmark/timer-watcher/benchmark.rb

@@ -0,0 +1,44 @@
+require "coolio"
+
+$times = 100000
+
+def add_timeout_timer(loop)
+  timeout_timer = Coolio::TimerWatcher.new(60)
+  timeout_timer.on_timer do
+    timeout_timer.detach
+    timeout_timer = nil
+  end
+  loop.attach(timeout_timer)
+  timeout_timer
+end
+
+puts "start."
+
+$start = Time.now
+$timers = []
+$loop = Cool.io::Loop.default
+$times.times do
+  $timers << add_timeout_timer($loop)
+end
+
+$timeout_before = Time.now
+timeout_timer = Coolio::TimerWatcher.new(1)
+timeout_timer.on_timer do
+  $timeout_after = Time.now
+
+  timeout_timer.detach
+  $timers.each do |timer|
+    timer.detach
+  end
+
+  $finish = Time.now
+  delta = $finish - $start
+  delta -= $timeout_after - $timeout_before
+  puts "done."
+  puts "overhead: #{delta} seconds for #{$times} timers."
+end
+$loop.attach(timeout_timer)
+
+$loop.run
+
+

data/bin/droonga-engine-absorb-data

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,230 +17,289 @@
 
 require "ostruct"
 require "optparse"
-require "open3"
 require "socket"
+require "coolio"
 
 require "droonga/engine/version"
-require "droonga/catalog_generator"
 require "droonga/path"
-require "droonga/data_absorber"
+require "droonga/node_name"
+require "droonga/data_absorber_client"
 require "droonga/serf"
 require "droonga/client"
 
-class AbsorbDataCommand
-  def run
-    parse_options
-    assert_valid_options
-    trap_signals
-
-    puts "Start to absorb data from #{@options.source_host}"
-    puts "                       to #{@options.destination_host}"
-    puts "                      via #{@options.receiver_host} (this host)"
-    puts "  dataset = #{@options.dataset}"
-    puts "  port    = #{@options.port}"
-    puts "  tag     = #{@options.tag}"
-    puts ""
-    puts "Absorbing..."
-
-    if @options.remote
-      absorb_on_remote
-    else
-      absorb_on_local
-    end
+module Droonga
+  module Command
+    class AbsorbData
+    def run
+      @loop = Coolio::Loop.default
 
-    puts "Done."
-    exit(true)
-  end
+      parse_options
+      assert_valid_options
+      trap_signals
 
-  private
-  def parse_options
-    options = OpenStruct.new
-    options.port    = Droonga::CatalogGenerator::DEFAULT_PORT
-    options.tag     = Droonga::CatalogGenerator::DEFAULT_TAG
-    options.dataset = Droonga::CatalogGenerator::DEFAULT_DATASET
-    options.remote  = true
-    options.receiver_host = Socket.gethostname
-    options.messages_per_second = Droonga::DataAbsorber::DEFAULT_MESSAGES_PER_SECOND
-    parser = OptionParser.new
-    parser.version = Droonga::Engine::VERSION
-
-    parser.separator("")
-    parser.separator("Connection:")
-    parser.on("--source-host=HOST",
-              "Host name of the source cluster to be connected.") do |host|
-      options.source_host = host
-    end
-    parser.on("--destination-host=HOST",
-              "Host name of this cluster to be connected.") do |host|
-      options.destination_host = host
-    end
-    parser.on("--receiver-host=HOST",
-              "Host name of this computer.",
-              "(#{options.receiver_host})") do |host|
-      options.receiver_host = host
-    end
-    parser.on("--port=PORT", Integer,
-              "Port number of the source cluster to be connected.",
-              "(#{options.port})") do |port|
-      options.port = port
+      puts "Start to absorb data from #{@options.source_dataset} at #{source_node.to_s}"
+      puts "                       to #{@options.dataset} at #{destination_node.to_s}"
+      puts "                      via #{@options.receiver_host} (this host)"
+      puts ""
+      puts "Absorbing..."
+
+      #XXX If any command is received by the source node after changing of its role,
+      #    the timestamp of last processed mesasge is unexpectedly updated by them.
+      #    Be careful to not send any command to the source node on this timing!!
+      update_accept_messages_newer_than_timestamp
+
+      succeeded = absorb
+
+      if succeeded
+        puts "Done."
+      else
+        do_cancel
+      end
+      succeeded
     end
-    parser.on("--[no-]remote",
-              "Run command in remote node or not.",
-              "(#{options.remote})") do |remote|
-      options.remote = remote
+
+    private
+    def parse_options
+      options = OpenStruct.new
+
+      options.host    = Socket.gethostname
+      options.port    = DataAbsorberClient::DEFAULT_PORT
+      options.tag     = DataAbsorberClient::DEFAULT_TAG
+      options.dataset = DataAbsorberClient::DEFAULT_DATASET
+
+      options.source_host    = DataAbsorberClient::DEFAULT_HOST
+      options.source_port    = DataAbsorberClient::DEFAULT_PORT
+      options.source_tag     = DataAbsorberClient::DEFAULT_TAG
+      options.source_dataset = DataAbsorberClient::DEFAULT_DATASET
+
+      options.receiver_host = Socket.gethostname
+
+      options.messages_per_second       = DataAbsorberClient::DEFAULT_MESSAGES_PER_SECOND
+      options.progress_interval_seconds = DataAbsorberClient::DEFAULT_PROGRESS_INTERVAL_SECONDS
+
+      options.verbose = false
+
+      parser = OptionParser.new
+      parser.version = Engine::VERSION
+
+      parser.separator("")
+      parser.separator("Destination node:")
+      parser.on("--host=HOST",
+                "Host name of the destination node.") do |host|
+        options.host = host
+      end
+      parser.on("--port=PORT", Integer,
+                "Port number of the destination node.",
+                "(#{options.port})") do |port|
+        options.port = port
+      end
+      parser.on("--tag=TAG", Integer,
+                "Tag name of the destination node.",
+                "(#{options.tag})") do |tag|
+        options.tag = tag
+      end
+      parser.on("--dataset=DATASET",
+                "Name of the destination dataset.",
+                "(#{options.dataset})") do |dataset|
+        options.dataset = dataset
+      end
+
+      parser.separator("")
+      parser.separator("Source node:")
+      parser.on("--source-host=HOST",
+                "Host name of the source node.",
+                "(#{options.source_host})") do |host|
+        options.source_host = host
+      end
+      parser.on("--source-port=PORT", Integer,
+                "Port number of the source node.",
+                "(#{options.source_port})") do |host|
+        options.source_host = host
+      end
+      parser.on("--source-tag=TAG",
+                "Tag name of the source node.",
+                "(#{options.source_tag})") do |tag|
+        options.source_tag = tag
+      end
+      parser.on("--source-dataset=DATASET",
+                "Name of the source dataset.",
+                "(#{options.source_dataset})") do |dataset|
+        options.source_dataset = dataset
+      end
+
+      parser.separator("")
+      parser.separator("Connection:")
+      parser.on("--receiver-host=HOST",
+                "Host name of this computer.",
+                "(#{options.receiver_host})") do |host|
+        options.receiver_host = host
+      end
+
+      parser.separator("")
+      parser.separator("Miscellaneous:")
+      parser.on("--records-per-second=N", Integer,
+                "Maximum number of records per second to be absorbed.",
+                "'#{Client::RateLimiter::NO_LIMIT}' means no limit.",
+                "(#{options.messages_per_second})") do |n|
+        options.messages_per_second = n
+      end
+      parser.on("--progress-interval-seconds=N", Integer,
+                "Interval seconds to report progress.",
+                "(#{options.progress_interval_seconds})") do |n|
+        options.progress_interval_seconds = n
+      end
+      parser.on("--[no-]verbose",
+                "Output details for internal operations.",
+                "(#{options.verbose})") do |verbose|
+        options.verbose = verbose
+      end
+
+      parser.separator("")
+      parser.separator("For backward compatibility:")
+      parser.on("--destination-host=HOST",
+                "Alias to \"--host\".") do |host|
+        options.host = host
+      end
+
+      parser.parse!(ARGV)
+      @options = options
     end
 
-    parser.separator("")
-    parser.separator("Data:")
-    parser.on("--tag=TAG",
-              "Tag name to be used to communicate with Droonga system.",
-              "(#{options.tag})") do |tag|
-      options.tag = tag
+    def assert_valid_options
+      unless @options.source_host
+        raise "You must specify the source host via --source-host option."
+      end
+      unless @options.host
+        raise "You must specify the destination host via --host option."
+      end
     end
-    parser.on("--dataset=DATASET",
-              "Dataset to be absorbed.",
-              "(#{options.dataset})") do |dataset|
-      options.dataset = dataset
+
+    def source_node
+      @source_node ||= NodeName.new(:host => @options.source_host,
+                                    :port => @options.source_port,
+                                    :tag  => @options.source_tag)
     end
-    parser.on("--records-per-second=N", Integer,
-              "Maximum number of records per second to be absorbed.",
-              "'#{Droonga::Client::RateLimiter::NO_LIMIT}' means no limit.",
-              "(#{options.messages_per_second})") do |n|
-      options.messages_per_second = n
+
+    def destination_node
+      @destination_node ||= NodeName.new(:host => @options.host,
+                                         :port => @options.port,
+                                         :tag  => @options.tag)
     end
 
-    parser.parse!(ARGV)
-    @options = options
-  end
+    def source_node_serf
+      @source_node_serf ||= Serf.new(source_node.to_s,
+                                     :verbose => @options.verbose)
+    end
 
-  def assert_valid_options
-    unless @options.source_host
-      raise "You must specify the source host via --source-host option."
+    def destination_node_serf
+      @destination_node_serf ||= Serf.new(destination_node.to_s,
+                                          :verbose => @options.verbose)
     end
-    unless @options.destination_host
-      raise "You must specify the destination host via --destination-host option."
+
+    def absorber
+      @absorber ||= prepare_absorber
     end
-  end
 
-  def source_node
-    "#{@options.source_host}:#{@options.port}/#{@options.tag}"
-  end
+    def prepare_absorber
+      absorber_options = {
+        :host    => @options.host,
+        :port    => @options.port,
+        :tag     => @options.tag,
+        :dataset => @options.dataset,
 
-  def destination_node
-    "#{@options.destination_host}:#{@options.port}/#{@options.tag}"
-  end
+        :source_host    => @options.source_host,
+        :source_port    => @options.source_port,
+        :source_tag     => @options.source_tag,
+        :source_dataset => @options.source_dataset,
 
-  def run_remote_command(target, command, options)
-    serf = Droonga::Serf.new(nil, target)
-    result = serf.send_query(command, options)
-    #puts result[:result]
-    puts result[:error] unless result[:error].empty?
-    result[:response]
-  end
+        :receiver_host    => @options.receiver_host,
 
-  def absorber
-    @absorber ||= prepare_absorber
-  end
+        :messages_per_second => @options.messages_per_second,
+        :progress_interval_seconds => @options.progress_interval_seconds,
 
-  def prepare_absorber
-    absorber_options = {
-      :dataset          => @options.dataset,
-      :source_host      => @options.source_host,
-      :destination_host => @options.destination_host,
-      :receiver_host    => @options.receiver_host,
-      :port             => @options.port,
-      :tag              => @options.tag,
-      :messages_per_second => @options.messages_per_second,
-    }
-    Droonga::DataAbsorber.new(absorber_options)
-  end
+        :client_options   => {
+          :backend => :coolio,
+          :loop    => @loop,
+        },
+      }
+      DataAbsorberClient.new(absorber_options)
+    end
 
-  def absorb_on_remote
-    start_time_in_seconds = Time.new.to_i
-    run_remote_command(destination_node, "absorb_data",
-                       "node"    => destination_node,
-                       "source"  => @options.source_host,
-                       "port"    => @options.port,
-                       "tag"     => @options.tag,
-                       "dataset" => @options.dataset,
-                       "messages_per_second" => @options.messages_per_second)
-    last_progress = ""
-    while true
-      sleep(3)
-      response = run_remote_command(destination_node, "report_status",
-                                    "node" => destination_node,
-                                    "key" => "absorbing")
-      if response
-        absorbing = response["value"]
-        break unless absorbing
-      end
-
-      progress = absorber.report_progress(start_time_in_seconds)
-      if progress
-        printf("%s", "#{" " * last_progress.size}\r")
-        printf("%s", "#{progress}\r")
+    def absorb
+      last_progress = nil
+      absorber.run do |progress|
+        if last_progress
+          printf("%s", "#{" " * last_progress[:message].size}\r")
+        end
+        printf("%s", "#{progress[:message]}\r")
         last_progress = progress
       end
+      @loop.run
+
+      if absorber.error_message
+        puts(absorber.error_message)
+        return false
+      end
+
+      puts ""
+      true
     end
-    puts ""
-
-    response = run_remote_command(source_node, "report_status",
-                                  "node" => source_node,
-                                  "key" => "last_processed_message_timestamp")
-    timestamp = response["value"]
-    if timestamp and not timestamp.empty?
-      puts "The timestamp of the last processed message in the source node: #{timestamp}"
-      puts "Setting effective message timestamp for the destination node..."
-      response = run_remote_command(destination_node, "set_status",
-                                    "node" => destination_node,
-                                    "key" => "effective_message_timestamp",
-                                    "value" => timestamp)
+
+    GETTING_LAST_MESSAGE_TIMESTAMP_MAX_RETRY_COUNT = 10
+    GETTING_LAST_MESSAGE_TIMESTAMP_RETRY_INTERVAL_SECONDS = 10
+
+    def try_get_last_message_timestamp(retry_count=0)
+      puts "Getting the timestamp of the last processed message in the source node..."
+      timestamp = source_node_serf.last_message_timestamp
+      unless timestamp
+        if retry_count < GETTING_LAST_MESSAGE_TIMESTAMP_MAX_RETRY_COUNT
+          puts "Failed. Retrying..."
+          sleep(GETTING_LAST_MESSAGE_TIMESTAMP_RETRY_INTERVAL_SECONDS)
+          timestamp = try_get_last_message_timestamp(retry_count + 1)
+        end
+      end
+      timestamp
     end
-  end
 
-  def absorb_on_local
-    last_progress = ""
-    absorber.absorb do |live_status|
-      if live_status[:progress]
-        progress = live_status[:progress]
+    def update_accept_messages_newer_than_timestamp
+      timestamp = try_get_last_message_timestamp
+      if timestamp and not timestamp.empty?
+        puts "The timestamp of the last processed message in the source node: #{timestamp}"
+        puts "Setting the destination node to ignore messages older than the timestamp..."
+        destination_node_serf.ensure_restarted do
+          destination_node_serf.send_query("accept_messages_newer_than",
+                                           "node"      => destination_node.to_s,
+                                           "timestamp" => timestamp)
+        end
       else
-        progress = live_status[:output]
+        $stderr.puts("WARNING: Couldn't get the time stamp of " +
+                       "the last processed message from the source node. " +
+                       "Any message will be forwarded to the destination node.")
       end
-      printf("%s", "#{" " * last_progress.size}\r")
-      printf("%s", "#{progress}\r")
-      last_progress = progress
     end
-    response = run_remote_command(source_node, "report_status",
-                                  "node" => source_node,
-                                  "key" => "last_processed_message_timestamp")
-    timestamp = response["value"]
-    puts "The timestamp of the last processed message in the source node: #{timestamp}"
-    if timestamp and not timestamp.empty?
-      status = NodeStatus.new
-      status.set(:effective_message_timestamp, timestamp)
-    end
-  end
 
-  def trap_signals
-    trap(:TERM) do
-      do_cancel
-      trap(:TERM, "DEFAULT")
-    end
+    def trap_signals
+      trap(:TERM) do
+        trap(:TERM, "DEFAULT")
+        do_cancel
+      end
 
-    trap(:INT) do
-      do_cancel
-      trap(:INT, "DEFAULT")
-    end
+      trap(:INT) do
+        trap(:INT, "DEFAULT")
+        do_cancel
+      end
 
-    trap(:QUIT) do
-      do_cancel
-      trap(:QUIT, "DEFAULT")
+      trap(:QUIT) do
+        trap(:QUIT, "DEFAULT")
+        do_cancel
+      end
     end
-  end
 
-  def do_cancel
-    #XXX we have to write more codes to cancel remote processes!
+    def do_cancel
+      #XXX we have to write more codes to cancel remote processes!
+    end
+    end
   end
 end
 
-AbsorbDataCommand.new.run
+exit(Droonga::Command::AbsorbData.new.run)