droonga-engine 1.0.7 → 1.0.8

data/droonga-engine.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "archive-zip"
   gem.add_dependency "cool.io"
   gem.add_dependency "drndump"
-  gem.add_dependency "droonga-client", ">= 0.1.9"
+  gem.add_dependency "droonga-client", ">= 0.2.0"
   gem.add_dependency "droonga-message-pack-packer", ">= 1.0.2"
   gem.add_dependency "groonga-command-parser"
   gem.add_dependency "faraday"

data/install.sh

@@ -133,6 +133,7 @@ setup_configuration_directory() {
     echo "This node is configured with a hostname $HOST."
   fi
 
+  # we should use --no-prompt instead of --quiet, for droonga-engine 1.0.8 and later.
   droonga-engine-configure --quiet \
     --host=$HOST --port=$PORT
   if [ $? -ne 0 ]; then

data/lib/droonga/buffered_tcp_socket.rb

@@ -140,14 +140,14 @@ module Droonga
 
       def written_partial(size)
         written
-        @data = @data[size..-1]
+        @data = data[size..-1]
         @revision += 1
         buffering
       end
 
       private
       def path
-        @data_directory + "#{@time_stamp.iso8601(6)}.#{@revision}.chunk"
+        @data_directory + "#{@time_stamp.iso8601(6)}.#{@revision}.#{SUFFIX}"
       end
     end
   end

data/lib/droonga/catalog/base.rb

@@ -36,6 +36,24 @@ module Droonga
       def dataset(name)
         datasets[name]
       end
+
+      private
+      def migrate_database_location(current_db_path, device, name)
+        return if current_db_path.exist?
+
+        common_base_path = Pathname(@base_path)
+        old_db_paths = {
+          :top_level     => common_base_path + device + name + "db",
+          :singular_form => common_base_path + device + "database" + name + "db",
+        }
+        old_db_paths.each do |type, old_db_path|
+          if old_db_path.exist?
+            FileUtils.mkdir_p(current_db_path.parent.parent)
+            FileUtils.move(old_db_path.parent, current_db_path.parent)
+            return
+          end
+        end
+      end
     end
   end
 end

data/lib/droonga/catalog/dataset.rb

@@ -85,6 +85,14 @@ module Droonga
         routes
       end
 
+      def single_slice?
+        # TODO: Support slice key
+        replicas.all? do |volume|
+          volume.is_a?(SingleVolume) or
+            volume.slices.size == 1
+        end
+      end
+
       private
       def create_volumes(raw_volumes)
         raw_volumes.collect do |raw_volume|

data/lib/droonga/catalog/version1.rb

@@ -54,10 +54,9 @@ module Droonga
               partitions.each do |partition|
                 if partition =~ pattern
                   database_name = $POSTMATCH
-                  path = File.join([device, Path.databases.basename.to_s, database_name, "db"])
-                  path = Pathname(path).expand_path(base_path)
-                  migrate_database_location(path, :device => device,
-                                                  :name   => database_name)
+                  path = Path.databases(base_path) +
+                           device + database_name + "db"
+                  migrate_database_location(path, device, database_name)
                   options = {
                     :dataset => dataset_name,
                     :database => path.to_s,
@@ -389,14 +388,6 @@ module Droonga
         end
       end
 
-      def migrate_database_location(path, params)
-        old_path = File.join([params[:device], params[:name], "db"])
-        old_path = Pathname(old_path).expand_path(base_path)
-        if old_path.exist? and not path.exist?
-          FileUtils.move(old_path.to_s, path.to_s)
-        end
-      end
-
       class Dataset < Catalog::Dataset
         def compute_routes(args, live_nodes=nil)
           routes = []

data/lib/droonga/catalog/version2.rb

@@ -44,10 +44,11 @@ module Droonga
             volume.slices.each do |slice|
               volume_address = slice.volume.address
               if volume_address.node == node
-                path = File.join([device, Path.databases.basename.to_s, volume_address.name, "db"])
-                path = Pathname(path).expand_path(base_path)
-                migrate_database_location(path, :device => device,
-                                                :name   => volume_address.name)
+                name = volume_address.name
+                path = Path.databases(base_path) +
+                         device + name + "db"
+                migrate_database_location(path, device, name)
+
                 options = {
                   :dataset => dataset_name,
                   :database => path.to_s,
@@ -86,14 +87,6 @@ module Droonga
         end
         nodes.sort.uniq
       end
-
-      def migrate_database_location(path, params)
-        old_path = File.join([params[:device], params[:name], "db"])
-        old_path = Pathname(old_path).expand_path(base_path)
-        if old_path.exist? and not path.exist?
-          FileUtils.move(old_path.to_s, path.to_s)
-        end
-      end
     end
   end
 end

data/lib/droonga/command/droonga_engine.rb

@@ -20,7 +20,7 @@ require "fileutils"
 require "yaml"
 
 require "coolio"
-require "sigdump"
+require "sigdump/setup"
 
 require "droonga/engine/version"
 require "droonga/path"
@@ -114,7 +114,7 @@ module Droonga
 
       class Configuration
         attr_reader :host, :port, :tag, :log_file, :pid_file_path
-        attr_reader :ready_notify_fd, :orchestrated
+        attr_reader :ready_notify_fd
         def initialize
           config = load_config
 
@@ -122,23 +122,13 @@ module Droonga
           @port = config["port"] || Address::DEFAULT_PORT
           @tag  = config["tag"]  || Address::DEFAULT_TAG
 
-          @given_values = {}
-
           @log_file        = nil
           @daemon          = false
           @pid_file_path   = nil
           @ready_notify_fd = nil
-          @orchestrated = true
 
           if have_config_file?
-            if config.include?("daemon")
-              @daemon = config["daemon"]
-            else
-              @daemon = true
-            end
-            if @daemon
-              self.pid_file_path = config["pid_file"] || Path.default_pid_file
-            end
+            self.pid_file_path = config["pid_file"] if config["pid_file"]
             self.log_file  = config["log_file"] || Path.default_log_file
             self.log_level = config["log_level"] if config.include?("log_level")
           end
@@ -148,34 +138,6 @@ module Droonga
           File.exist?(Path.config)
         end
 
-        def have_given_host?
-          @given_values.include?(:host)
-        end
-
-        def have_given_port?
-          @given_values.include?(:port)
-        end
-
-        def have_given_tag?
-          @given_values.include?(:tag)
-        end
-
-        def have_given_daemon?
-          @given_values.include?(:daemon)
-        end
-
-        def have_given_log_file?
-          @given_values.include?(:log_file)
-        end
-
-        def have_given_log_level?
-          @given_values.include?(:log_level)
-        end
-
-        def have_given_pid_file?
-          @given_values.include?(:pid_file)
-        end
-
         def load_config
           if have_config_file?
             YAML.load_file(Path.config)
@@ -225,7 +187,6 @@ module Droonga
           add_log_options(parser)
           add_process_options(parser)
           add_path_options(parser)
-          add_orchestration_options(parser)
           add_notification_options(parser)
         end
 
@@ -245,19 +206,16 @@ module Droonga
                     "The host name of the Droonga engine",
                     "(#{@host})") do |host|
             @host = host
-            @given_values[:host] = host
           end
           parser.on("--port=PORT", Integer,
                     "The port number of the Droonga engine",
                     "(#{@port})") do |port|
             @port = port
-            @given_values[:port] = port
           end
           parser.on("--tag=TAG",
                     "The tag of the Droonga engine",
                     "(#{@tag})") do |tag|
             @tag = tag
-            @given_values[:tag] = tag
           end
         end
 
@@ -271,12 +229,10 @@ module Droonga
                     "[#{levels_label}]",
                     "(#{log_level})") do |level|
             self.log_level = level
-            @given_values[:log_level] = log_level
           end
           parser.on("--log-file=FILE",
                     "Output logs to FILE") do |file|
             self.log_file = file
-            @given_values[:log_file] = file
           end
         end
 
@@ -286,17 +242,14 @@ module Droonga
           parser.on("--daemon",
                     "Run as a daemon") do
             @daemon = true
-            @given_values[:daemon] = true
           end
           parser.on("--no-daemon",
                     "Run as a regular process") do
             @daemon = false
-            @given_values[:daemon] = false
           end
           parser.on("--pid-file=PATH",
                     "Put PID to PATH") do |path|
             self.pid_file_path = path
-            @given_values[:pid_file] = path
           end
         end
 
@@ -310,13 +263,6 @@ module Droonga
           end
         end
 
-        def add_orchestration_options(parser)
-          parser.on("--no-orchestration",
-                    "Disable orchestration with other nodes") do
-            @orchestrated = false
-          end
-        end
-
         def add_notification_options(parser)
           parser.separator("")
           parser.separator("Notification:")
@@ -348,7 +294,7 @@ module Droonga
 
           trap_signals
           @loop.run
-          @serf.stop if @serf and @serf.running?
+          @serf.stop if @serf.running?
 
           @service_runner.success?
         end
@@ -389,14 +335,14 @@ module Droonga
 
         def stop_gracefully
           @command_runner.stop
-          @serf.stop if @serf
+          @serf.stop
           @catalog_observer.stop
           @service_runner.stop_gracefully
         end
 
         def stop_immediately
           @command_runner.stop
-          @serf.stop if @serf
+          @serf.stop
           @catalog_observer.stop
           @service_runner.stop_immediately
         end
@@ -427,7 +373,6 @@ module Droonga
         end
 
         def run_serf
-          return nil unless @configuration.orchestrated
           serf = Serf.new(@loop, @configuration.engine_name)
           serf.start
           serf

data/lib/droonga/command/droonga_engine_service.rb

@@ -16,6 +16,7 @@
 require "optparse"
 
 require "coolio"
+require "sigdump/setup"
 
 require "droonga/worker_process_agent"
 require "droonga/engine"

data/lib/droonga/command/droonga_engine_worker.rb

@@ -17,6 +17,7 @@ require "optparse"
 require "fileutils"
 
 require "coolio"
+require "sigdump/setup"
 
 require "droonga/job_receiver"
 require "droonga/plugin_loader"

data/lib/droonga/command/remote.rb

@@ -89,6 +89,13 @@ module Droonga
         end
       end
 
+      class SetStatus < Base
+        def process
+          status = NodeStatus.new
+          status.set(@params["key"], @params["value"])
+        end
+      end
+
       class Join < Base
         def process
           log("type = #{type}")
@@ -115,6 +122,10 @@ module Droonga
           @params["dataset"]
         end
 
+        def messages_per_second
+          @params["messages_per_second"]
+        end
+
         def valid_params?
           have_required_params? and
             valid_node?(source_node) and
@@ -220,7 +231,8 @@ module Droonga
                               :source_host      => source_host,
                               :destination_host => joining_host,
                               :port             => port,
-                              :tag              => tag)
+                              :tag              => tag,
+                              :messages_per_second => messages_per_second)
           status.delete(:absorbing)
           sleep(1)
         end
@@ -258,6 +270,7 @@ module Droonga
                               :destination_host => host,
                               :port             => port,
                               :tag              => tag,
+                              :messages_per_second => messages_per_second,
                               :client           => "droonga-send")
           status.delete(:absorbing)
         end
@@ -278,6 +291,10 @@ module Droonga
         def tag
           @tag ||= @params["tag"]
         end
+
+        def messages_per_second
+          @messages_per_second ||= @params["messages_per_second"]
+        end
       end
 
       class ModifyReplicasBase < Base

data/lib/droonga/command/serf_event_handler.rb

@@ -46,6 +46,7 @@ module Droonga
         FileUtils.mkdir_p(Path.serf_event_handler_errors)
         File.open(Path.serf_event_handler_error_file, "w") do |file|
           file.write(exception.inspect)
+          file.write(exception.backtrace)
         end
         true
       end
@@ -72,6 +73,8 @@ module Droonga
           Remote::ChangeRole
         when "report_status"
           Remote::ReportStatus
+        when "set_status"
+          Remote::SetStatus
         when "join"
           Remote::Join
         when "set_replicas"

data/lib/droonga/data_absorber.rb

@@ -15,59 +15,249 @@
 
 require "open3"
 
+require "droonga/loggable"
+require "droonga/client"
+require "droonga/catalog_generator"
+require "droonga/catalog_fetcher"
+
 module Droonga
   class DataAbsorber
+    include Loggable
+
+    DEFAULT_MESSAGES_PER_SECOND = 100
+
+    TIME_UNKNOWN = -1
+    PROGRESS_UNKNOWN = -1
+
     class << self
       def absorb(params)
-        drndump = params[:drndump] || "drndump"
-        drndump_options = []
-        drndump_options += ["--host", params[:source_host]] if params[:source_host]
-        drndump_options += ["--port", params[:port].to_s] if params[:port]
-        drndump_options += ["--tag", params[:tag]] if params[:tag]
-        drndump_options += ["--dataset", params[:dataset]] if params[:dataset]
-        drndump_options += ["--receiver-host", params[:destination_host]]
-        drndump_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
-
-        #TODO: We should use droonga-send instead of droonga-request,
-        #      because droonga-request is too slow.
-        #      However, to do it, we have to implement an API to know
-        #      that all messages sent by droonga-send are completely
-        #      processed.
-        client = params[:client] || "droonga-request"
-        client_options = []
-        if client.include?("droonga-request")
-          client_options += ["--host", params[:destination_host]]
-          client_options += ["--port", params[:port].to_s] if params[:port]
-          client_options += ["--tag", params[:tag]] if params[:tag]
-          client_options += ["--receiver-host", params[:destination_host]]
-          client_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
-        elsif client.include?("droonga-send")
-          #XXX Don't use round-robin with multiple endpoints
-          #    even if there are too much data.
-          #    Schema and indexes must be sent to just one endpoint
-          #    to keep their order, but currently there is no way to
-          #    extract only schema and indexes via drndump.
-          #    So, we always use just one endpoint for now,
-          #    even if there are too much data.
-          server = "droonga:#{params[:destination_host]}"
-          server = "#{server}:#{params[:port].to_s}" if params[:port]
-          server = "#{server}/#{params[:tag].to_s}" if params[:tag]
-          client_options += ["--server", server]
-        else
-          raise ArgumentError.new("Unknwon type client: #{client}")
-        end
+        new(params).absorb
+      end
+    end
+
+    attr_reader :params
+    def initialize(params)
+      @params = params
 
-        drndump_command_line = [drndump] + drndump_options
-        client_command_line = [client] + client_options
+      @messages_per_second = @params[:messages_per_second] || DEFAULT_MESSAGES_PER_SECOND
 
-        env = {}
-        Open3.pipeline_r([env, *drndump_command_line],
-                         [env, *client_command_line]) do |last_stdout, thread|
-          last_stdout.each do |output|
-            yield output if block_given?
+      @drndump = @params[:drndump] || "drndump"
+      # We should use droonga-send instead of droonga-request,
+      # because droonga-request is too slow.
+      @client = @params[:client] || "droonga-send"
+
+      @dataset = @params[:dataset] || CatalogGenerator::DEFAULT_DATASET
+      @port    = @params[:port]    || CatalogGenerator::DEFAULT_PORT
+      @tag     = @params[:tag]     || CatalogGenerator::DEFAULT_TAG
+
+      @source_host      = @params[:source_host]
+      @destination_host = @params[:destination_host]
+
+      @receiver_port = @params[:receiver_port]
+    end
+
+    MESSAGES_PER_SECOND_MATCHER = /(\d+(\.\d+)?) messages\/second/
+
+    def absorb
+      drndump_command_line = [@drndump] + drndump_options
+      client_command_line  = [@client] + client_options(@client)
+
+      start_time_in_seconds = Time.new.to_i
+      env = {}
+      Open3.pipeline_r([env, *drndump_command_line],
+                       [env, *client_command_line]) do |last_stdout, thread|
+        last_stdout.each do |output|
+          if block_given?
+            messages_per_second = nil
+            if output =~ MESSAGES_PER_SECOND_MATCHER
+              messages_per_second = $1.to_f
+            end
+            yield(:progress => report_progress(start_time_in_seconds),
+                  :output   => output,
+                  :messages_per_second => messages_per_second)
           end
         end
       end
     end
+
+    def can_report_remaining_time?
+      required_time_in_seconds != Droonga::DataAbsorber::TIME_UNKNOWN and
+        required_time_in_seconds > 0
+    end
+
+    def required_time_in_seconds
+      @required_time_in_seconds ||= calculate_required_time_in_seconds
+    end
+
+    ONE_MINUTE_IN_SECONDS = 60
+    ONE_HOUR_IN_SECONDS = ONE_MINUTE_IN_SECONDS * 60
+
+    def report_progress(start_time_in_seconds)
+      return nil unless can_report_remaining_time?
+
+      elapsed_time = Time.new.to_i - start_time_in_seconds
+      progress = elapsed_time.to_f / required_time_in_seconds
+      progress = [(progress * 100).to_i, 100].min
+
+      remaining_seconds  = [required_time_in_seconds - elapsed_time, 0].max
+      remaining_hours    = (remaining_seconds / ONE_HOUR_IN_SECONDS).floor
+      remaining_seconds -= remaining_hours * ONE_HOUR_IN_SECONDS
+      remaining_minutes  = (remaining_seconds / ONE_MINUTE_IN_SECONDS).floor
+      remaining_seconds -= remaining_minutes * ONE_MINUTE_IN_SECONDS
+      remaining_time     = sprintf("%02i:%02i:%02i", remaining_hours, remaining_minutes, remaining_seconds)
+
+      "#{progress}% done (maybe #{remaining_time} remaining)"
+    end
+
+    def source_client
+      options = {
+        :host          => @source_host,
+        :port          => @port,
+        :tag           => @tag,
+        :progocol      => :droonga,
+        :receiver_host => @destination_host,
+        :receiver_port => 0,
+      }
+      @source_client ||= Droonga::Client.new(options)
+    end
+
+    def destination_client
+      options = {
+        :host          => @destination_host,
+        :port          => @port,
+        :tag           => @tag,
+        :progocol      => :droonga,
+        :receiver_host => @destination_host,
+        :receiver_port => 0,
+      }
+      @destination_client ||= Droonga::Client.new(options)
+    end
+
+    def source_node_suspendable?
+      (source_replica_hosts - [@source_host]).size > 1
+    end
+
+    private
+    def calculate_required_time_in_seconds
+      if @client.include?("droonga-send")
+        total_n_source_records / @messages_per_second
+      else
+        TIME_UNKNOWN
+      end
+    end
+
+    def drndump_options
+      options = []
+      options += ["--host", @source_host] if @source_host
+      options += ["--port", @port]
+      options += ["--tag", @tag]
+      options += ["--dataset", @dataset]
+      options += ["--receiver-host", @destination_host]
+      options += ["--receiver-port", @receiver_port] if @receiver_port
+      options.collect(&:to_s)
+    end
+
+    def droonga_request_options
+      options = []
+      options += ["--host", @destination_host]
+      options += ["--port", @port]
+      options += ["--tag", @tag]
+      options += ["--receiver-host", @destination_host]
+      options += ["--receiver-port", @receiver_port] if @receiver_port
+      options.collect(&:to_s)
+    end
+
+    def droonga_send_options
+      options = []
+  
+      #XXX Don't use round-robin with multiple endpoints
+      #    even if there are too much data.
+      #    Schema and indexes must be sent to just one endpoint
+      #    to keep their order, but currently there is no way to
+      #    extract only schema and indexes via drndump.
+      #    So, we always use just one endpoint for now,
+      #    even if there are too much data.
+      server = "droonga:#{params[:destination_host]}"
+      server = "#{server}:#{params[:port].to_s}"
+      server = "#{server}/#{params[:tag].to_s}"
+      options += ["--server", server]
+  
+      #XXX We should restrict the traffic to avoid overflowing!
+      options += ["--messages-per-second", @messages_per_second]
+
+      options += ["--report-throughput"]
+  
+      options.collect(&:to_s)
+    end
+
+    def client_options(client)
+      if client.include?("droonga-request")
+        droonga_request_options
+      elsif client.include?("droonga-send")
+        droonga_send_options
+      else
+        raise ArgumentError.new("Unknwon type client: #{client}")
+      end
+    end
+
+    def source_tables
+      response = source_client.request("dataset" => @dataset,
+                                       "type"    => "table_list")
+      body = response["body"][1]
+      tables = body[1..-1]
+      tables.collect do |table|
+        table[1]
+      end
+    end
+
+    def total_n_source_records
+      queries = {}
+      source_tables.each do |table|
+        queries["n_records_of_#{table}"] = {
+          "source" => table,
+          "output" => {
+            "elements" => ["count"],
+          },
+        }
+      end
+      response = source_client.request("dataset" => @dataset,
+                                       "type"    => "search",
+                                       "body"    => {
+                                         "queries" => queries,
+                                       })
+      n_records = 0
+      response["body"].each do |query_name, result|
+        n_records += result["count"]
+      end
+      n_records
+    end
+
+    def source_replica_hosts
+      @source_replica_hosts ||= get_source_replica_hosts
+    end
+
+    def get_source_replica_hosts
+      generator = CatalogGenerator.new
+      generator.load(source_catalog)
+      dataset = generator.dataset_for_host(@source_host)
+      return [] unless dataset
+      dataset.replicas.hosts
+    end
+
+    def source_catalog
+      @source_catalog ||= fetch_source_catalog
+    end
+
+    def fetch_source_catalog
+      fetcher = CatalogFetcher.new(:host          => @source_host,
+                                   :port          => @port,
+                                   :tag           => @tag,
+                                   :receiver_host => @destination_host)
+      fetcher.fetch(:dataset => @dataset)
+    end
+
+    def log_tag
+      "data-absorber"
+    end
   end
 end