droonga-engine 1.0.3 → 1.0.4

data/lib/droonga/catalog_generator.rb

@@ -20,10 +20,23 @@ module Droonga
     DEFAULT_DATASET = "Default"
     DEFAULT_HOSTS = ["127.0.0.1"]
     DEFAULT_N_WORKERS = 4
-    DEFAULT_PLUGINS = ["groonga", "search", "crud", "dump"]
+    DEFAULT_N_SLICES = 1
+    DEFAULT_PLUGINS = ["groonga", "search", "crud", "dump", "system"]
     DEFAULT_PORT = 10031
     DEFAULT_TAG = "droonga"
 
+    attr_reader :datasets
+
+    class << self
+      def generate(datasets_params)
+        generator = new
+        datasets_params.each do |name, params|
+          generator.add_dataset(name, params)
+        end
+        generator.generate
+      end
+    end
+
     def initialize
       @version = 2
       @effective_date = Time.now
@@ -42,6 +55,44 @@ module Droonga
       }
     end
 
+    def load(catalog)
+      catalog["datasets"].each do |name, dataset|
+        add_dataset(name, dataset_to_params(dataset))
+      end
+      self
+    end
+
+    def dataset_for_host(host)
+      @datasets.each do |name, dataset|
+        if dataset.replicas.hosts.include?(host)
+          return dataset
+        end
+      end
+      nil
+    end
+
+    def modify(dataset_modifications)
+      dataset_modifications.each do |name, modification|
+        dataset = @datasets[name]
+        next unless dataset
+
+        replicas = dataset.replicas
+
+        if modification[:replica_hosts]
+          replicas.hosts = modification[:replica_hosts]
+        end
+
+        if modification[:add_replica_hosts]
+          replicas.hosts += modification[:add_replica_hosts]
+          replicas.hosts.uniq!
+        end
+
+        if modification[:remove_replica_hosts]
+          replicas.hosts -= modification[:remove_replica_hosts]
+        end
+      end
+    end
+
     private
     def catalog_datasets
       catalog_datasets = {}
@@ -52,6 +103,8 @@ module Droonga
     end
 
     class Dataset
+      attr_reader :name
+
       def initialize(name, options)
         @name = name
         @options = options
@@ -75,7 +128,7 @@ module Droonga
 
       def replicas
         return @options[:replicas] if @options[:replicas]
-        @generated_replicas ||= Replicas.new(@options).to_json
+        @generated_replicas ||= Replicas.new(@options)
       end
 
       def to_catalog
@@ -83,7 +136,7 @@ module Droonga
           "nWorkers" => n_workers,
           "plugins"  => plugins,
           "schema"   => schema,
-          "replicas" => replicas,
+          "replicas" => replicas.to_json,
         }
         catalog["fact"] = fact if fact
         catalog
@@ -93,6 +146,9 @@ module Droonga
     end
 
     class Replicas
+      attr_accessor :hosts
+      attr_reader :port, :tag, :n_slices
+
       def initialize(options={})
         @hosts      = options[:hosts] || DEFAULT_HOSTS
         @port       = options[:port]
@@ -122,7 +178,7 @@ module Droonga
         @host       = host
         @port       = options[:port]     || DEFAULT_PORT
         @tag        = options[:tag]      || DEFAULT_TAG
-        @n_slices   = options[:n_slices] || 1
+        @n_slices   = options[:n_slices] || DEFAULT_N_SLICES
 
         @n_volumes = 0
       end
@@ -159,5 +215,32 @@ module Droonga
         @weight ||= 100 / @n_slices
       end
     end
+
+    ADDRESS_MATCHER = /\A(.*):(\d+)\/([^\.]+)\.(.+)\z/
+
+    def dataset_to_params(dataset)
+      params = {}
+      params[:n_workers] = dataset["nWorkers"]
+      params[:n_slices]  = dataset["replicas"].first["slices"].size
+      params[:plugins]   = dataset["plugins"]
+      params[:schema]    = dataset["schema"] if dataset["schema"]
+      params[:fact]      = dataset["fact"] if dataset["fact"]
+
+      nodes = dataset["replicas"].collect do |replica|
+        ADDRESS_MATCHER =~ replica["slices"].first["volume"]["address"]
+        {
+          :host => $1,
+          :port => $2.to_i,
+          :tag  => $3,
+          :path => $4,
+        }
+      end
+      params[:tag]   = nodes.first[:tag]
+      params[:port]  = nodes.first[:port].to_i
+      params[:hosts] = nodes.collect do |node|
+        node[:host]
+      end
+      params
+    end
   end
 end

data/lib/droonga/command/droonga_engine.rb

@@ -19,11 +19,13 @@ require "ipaddr"
 require "fileutils"
 
 require "coolio"
+require "sigdump"
 
 require "droonga/path"
 require "droonga/serf"
-require "droonga/catalog_observer"
+require "droonga/file_observer"
 require "droonga/service_control_protocol"
+require "droonga/line_buffer"
 
 module Droonga
   module Command
@@ -130,7 +132,7 @@ module Droonga
         end
 
         def log_level
-          ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default_label
+          ENV["DROONGA_LOG_LEVEL"] || Logger::Level.default
         end
 
         def daemon?
@@ -235,6 +237,7 @@ module Droonga
 
         def run
           @serf = run_serf
+          @serf_status_observer = run_serf_status_observer
           @service_runner = run_service
           @catalog_observer = run_catalog_observer
           @loop_breaker = Coolio::AsyncWatcher.new
@@ -267,12 +270,16 @@ module Droonga
           trap(:HUP) do
             restart_immediately
           end
+          trap(:USR2) do
+            Sigdump.dump
+          end
         end
 
         def stop_gracefully
           @loop_breaker.signal
           @loop_breaker.detach
           @serf.shutdown
+          @serf_status_observer.stop
           @catalog_observer.stop
           @service_runner.stop_gracefully
         end
@@ -281,6 +288,7 @@ module Droonga
           @loop_breaker.signal
           @loop_breaker.detach
           @serf.shutdown
+          @serf_status_observer.stop
           @catalog_observer.stop
           @service_runner.stop_immediately
         end
@@ -290,7 +298,6 @@ module Droonga
           old_service_runner = @service_runner
           @service_runner = run_service
           @service_runner.on_ready = lambda do
-            @serf.restart if @serf.running?
             @service_runner.on_failure = nil
             old_service_runner.stop_gracefully
           end
@@ -304,7 +311,6 @@ module Droonga
           @loop_breaker.signal
           old_service_runner = @service_runner
           @service_runner = run_service
-          @serf.restart if @serf.running?
           old_service_runner.stop_immediately
         end
 
@@ -320,8 +326,22 @@ module Droonga
           serf
         end
 
+        def restart_serf
+          @serf.shutdown if @serf
+          @serf = run_serf
+        end
+
+        def run_serf_status_observer
+          serf_status_observer = FileObserver.new(@loop, Serf.status_file)
+          serf_status_observer.on_change = lambda do
+            restart_serf
+          end
+          serf_status_observer.start
+          serf_status_observer
+        end
+
         def run_catalog_observer
-          catalog_observer = CatalogObserver.new(@loop)
+          catalog_observer = FileObserver.new(@loop, Path.catalog)
           catalog_observer.on_change = lambda do
             restart_graceful
           end
@@ -414,9 +434,9 @@ module Droonga
 
         def attach_control_read_in(control_read_in)
           @control_read_in = Coolio::IO.new(control_read_in)
+          line_buffer = LineBuffer.new
           on_read = lambda do |data|
-            # TODO: should buffer data to handle half line received case
-            data.each_line do |line|
+            line_buffer.feed(data) do |line|
               case line
               when Messages::READY
                 on_ready

data/lib/droonga/command/droonga_engine_service.rb

@@ -18,6 +18,7 @@ require "optparse"
 require "coolio"
 
 require "droonga/service_control_protocol"
+require "droonga/line_buffer"
 require "droonga/engine"
 require "droonga/fluent_message_receiver"
 require "droonga/internal_fluent_message_receiver"
@@ -162,8 +163,8 @@ module Droonga
         @control_read = Coolio::IO.new(IO.new(@control_read_fd))
         @control_read_fd = nil
         on_read = lambda do |data|
-          # TODO: should buffer data to handle half line received case
-          data.each_line do |line|
+          line_buffer = LineBuffer.new
+          line_buffer.feed(data) do |line|
             case line
             when Messages::STOP_GRACEFUL
               stop_gracefully

data/lib/droonga/command/serf_event_handler.rb

@@ -13,15 +13,13 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "optparse"
-require "pathname"
 require "json"
-require "fileutils"
-require "tempfile"
 
 require "droonga/path"
 require "droonga/serf"
-require "droonga/live_nodes_list_observer"
+require "droonga/catalog_generator"
+require "droonga/data_absorber"
+require "droonga/safe_file_writer"
 
 module Droonga
   module Command
@@ -35,11 +33,14 @@ module Droonga
       def initialize
         @serf = ENV["SERF"] || Serf.path
         @serf_rpc_address = ENV["SERF_RPC_ADDRESS"] || "127.0.0.1:7373"
+        @serf_name = ENV["SERF_SELF_NAME"]
       end
 
       def run
         parse_event
+        return true unless event_for_me?
 
+        process_event
         output_live_nodes
         true
       end
@@ -47,14 +48,209 @@ module Droonga
       private
       def parse_event
         @event_name = ENV["SERF_EVENT"]
+        @payload = nil
         case @event_name
         when "user"
-          @event_name += ":#{ENV["SERF_USER_EVENT"]}"
+          @event_sub_name = ENV["SERF_USER_EVENT"]
+          @payload = JSON.parse($stdin.gets)
+          puts "event sub name = #{@event_sub_name}"
         when "query"
-          @event_name += ":#{ENV["SERF_USER_QUERY"]}"
+          @event_sub_name = ENV["SERF_QUERY_NAME"]
+          @payload = JSON.parse($stdin.gets)
+          puts "event sub name = #{@event_sub_name}"
         end
       end
 
+      def event_for_me?
+        return true unless @payload
+        return true unless @payload["node"]
+
+        @payload["node"] == @serf_name
+      end
+
+      def process_event
+        case @event_sub_name
+        when "change_role"
+          save_status(:role, @payload["role"])
+        when "join"
+          join
+        when "set_replicas"
+          set_replicas
+        when "add_replicas"
+          add_replicas
+        when "remove_replicas"
+          remove_replicas
+        when "absorb_data"
+          absorb_data
+        end
+      end
+
+      def host
+        @serf_name.split(":").first
+      end
+
+      def given_hosts
+        hosts = @payload["hosts"]
+        return nil unless hosts
+        hosts = [hosts] if hosts.is_a?(String)
+        hosts
+      end
+
+      def join
+        type = @payload["type"]
+        puts "type = #{type}"
+        case type
+        when "replica"
+          join_as_replica
+        end
+      end
+
+      def join_as_replica
+        source = @payload["source"]
+        return unless source
+
+        puts "source  = #{source}"
+
+        generator = create_current_catalog_generator
+        dataset = generator.dataset_for_host(source) ||
+                    generator.dataset_for_host(host)
+        return unless dataset
+
+        dataset_name = dataset.name
+        tag          = dataset.replicas.tag
+        port         = dataset.replicas.port
+        other_hosts  = dataset.replicas.hosts
+
+        puts "dataset = #{dataset_name}"
+        puts "port    = #{port}"
+        puts "tag     = #{tag}"
+
+        if @payload["copy"]
+          puts "starting to copy data from #{source}"
+
+          modify_catalog do |modifier|
+            modifier.datasets[dataset_name].replicas.hosts = [host]
+          end
+          sleep(1) # wait for restart
+
+          DataAbsorber.absorb(:dataset          => dataset_name,
+                              :source_host      => source,
+                              :destination_host => host,
+                              :port             => port,
+                              :tag              => tag)
+          sleep(1)
+        end
+
+        puts "joining to the cluster: update myself"
+
+        modify_catalog do |modifier|
+          modifier.datasets[dataset_name].replicas.hosts += other_hosts
+          modifier.datasets[dataset_name].replicas.hosts.uniq!
+        end
+        sleep(1) # wait for restart
+
+        puts "joining to the cluster: update others"
+
+        source_node  = "#{source}:#{port}/#{tag}"
+        Serf.send_query(source_node, "add_replicas",
+                        "dataset" => dataset_name,
+                        "hosts"   => [host])
+      end
+
+      def set_replicas
+        dataset = @payload["dataset"]
+        return unless dataset
+
+        hosts = given_hosts
+        return unless hosts
+
+        puts "new replicas: #{hosts.join(",")}"
+
+        modify_catalog do |modifier|
+          modifier.datasets[dataset].replicas.hosts = hosts
+        end
+      end
+
+      def add_replicas
+        dataset = @payload["dataset"]
+        return unless dataset
+
+        hosts = given_hosts
+        return unless hosts
+
+        hosts -= [host]
+        return if hosts.empty?
+
+        puts "adding replicas: #{hosts.join(",")}"
+
+        modify_catalog do |modifier|
+          modifier.datasets[dataset].replicas.hosts += hosts
+          modifier.datasets[dataset].replicas.hosts.uniq!
+        end
+      end
+
+      def remove_replicas
+        dataset = @payload["dataset"]
+        return unless dataset
+
+        hosts = given_hosts
+        return unless hosts
+
+        puts "removing replicas: #{hosts.join(",")}"
+
+        modify_catalog do |modifier|
+          modifier.datasets[dataset].replicas.hosts -= hosts
+        end
+      end
+
+      def modify_catalog
+        generator = create_current_catalog_generator
+        yield(generator)
+        SafeFileWriter.write(Path.catalog, JSON.pretty_generate(generator.generate))
+      end
+
+      def create_current_catalog_generator
+        current_catalog = JSON.parse(Path.catalog.read)
+        generator = CatalogGenerator.new
+        generator.load(current_catalog)
+      end
+
+      def absorb_data
+        return unless event_for_me?
+
+        source = @payload["source"]
+        return unless source
+
+        puts "start to absorb data from #{source}"
+
+        dataset_name = @payload["dataset"]
+        port         = @payload["port"]
+        tag          = @payload["tag"]
+
+        if dataset_name.nil? or port.nil? or tag.nil?
+          current_catalog = JSON.parse(Path.catalog.read)
+          generator = CatalogGenerator.new
+          generator.load(current_catalog)
+
+          dataset = generator.dataset_for_host(source)
+          return unless dataset
+
+          dataset_name = dataset.name
+          port = dataset.replicas.port
+          tag  = dataset.replicas.tag
+        end
+
+        puts "dataset = #{dataset_name}"
+        puts "port    = #{port}"
+        puts "tag     = #{tag}"
+
+        DataAbsorber.absorb(:dataset          => dataset_name,
+                            :source_host      => source,
+                            :destination_host => host,
+                            :port             => port,
+                            :tag              => tag)
+      end
+
       def live_nodes
         nodes = {}
         members = `#{@serf} members -rpc-addr #{@serf_rpc_address}`
@@ -70,15 +266,16 @@ module Droonga
       end
 
       def output_live_nodes
-        list_path = LiveNodesListObserver.path
+        path = Path.live_nodes
         nodes = live_nodes
         file_contents = JSON.pretty_generate(nodes)
-        # Don't output the file directly to prevent loading of incomplete file!
-        Tempfile.open(list_path.basename.to_s, list_path.parent.to_s, "w") do |output|
-          output.write(file_contents)
-          output.flush
-          File.rename(output.path, list_path.to_s)
-        end
+        SafeFileWriter.write(path, file_contents)
+      end
+
+      def save_status(key, value)
+        status = Serf.load_status
+        status[key] = value
+        SafeFileWriter.write(Serf.status_file, JSON.pretty_generate(status))
       end
     end
   end

data/lib/droonga/data_absorber.rb

@@ -0,0 +1,55 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "open3"
+
+module Droonga
+  class DataAbsorber
+    class << self
+      def absorb(params)
+        drndump = params[:drndump] || "drndump"
+        drndump_options = []
+        drndump_options += ["--host", params[:source_host]] if params[:source_host]
+        drndump_options += ["--port", params[:port].to_s] if params[:port]
+        drndump_options += ["--tag", params[:tag]] if params[:tag]
+        drndump_options += ["--dataset", params[:dataset]] if params[:dataset]
+        drndump_options += ["--receiver-host", params[:destination_host]]
+        drndump_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
+
+        client = params[:client] || "droonga-request"
+        client_options = []
+        client_options += ["--host", params[:destination_host]]
+        client_options += ["--port", params[:port].to_s] if params[:port]
+        client_options += ["--tag", params[:tag]] if params[:tag]
+        client_options += ["--receiver-host", params[:destination_host]]
+        client_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
+
+        drndump_command_line = [drndump] + drndump_options
+        client_command_line = [client] + client_options
+
+        Open3.popen3(*drndump_command_line) do |dump_in, dump_out, dump_error, dump_thread|
+          dump_in.close
+          Open3.popen3(*client_command_line) do |client_in, client_out, client_error, client_thread|
+            client_out.close
+            dump_out.each do |dump|
+              yield dump if block_given?
+              client_in.puts(dump)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/droonga/dispatcher.rb

@@ -46,19 +46,20 @@ module Droonga
       end
     end
 
-    attr_accessor :live_nodes
+    attr_reader :engine_state
 
     def initialize(engine_state, catalog)
       @engine_state = engine_state
+      @forwarder = @engine_state.forwarder
+      @replier = @engine_state.replier
       @catalog = catalog
-      @live_nodes = catalog.all_nodes
       @adapter_runners = create_adapter_runners
       @farm = Farm.new(@engine_state.name, @catalog, @engine_state.loop,
-                       :dispatcher => self)
+                       :engine_state => @engine_state,
+                       :dispatcher => self,
+                       :forwarder  => @forwarder)
       @collector_runners = create_collector_runners
       @step_runners = create_step_runners
-      @forwarder = @engine_state.forwarder
-      @replier = @engine_state.replier
     end
 
     def start
@@ -169,22 +170,23 @@ module Droonga
 
     def dispatch_steps(steps)
       id = @engine_state.generate_id
-      destinations = {}
+
+      destinations = []
       steps.each do |step|
         dataset = @catalog.dataset(step["dataset"])
         if dataset
-          routes = dataset.get_routes(step, @live_nodes)
+          routes = dataset.get_routes(step, @engine_state.live_nodes)
           step["routes"] = routes
         else
           step["routes"] ||= [id]
         end
-        routes = step["routes"]
-        routes.each do |route|
-          destinations[farm_path(route)] = true
+        destinations += step["routes"].collect do |route|
+          farm_path(route)
         end
       end
+
       dispatch_message = { "id" => id, "steps" => steps }
-      destinations.each_key do |destination|
+      destinations.uniq.each do |destination|
         dispatch(dispatch_message, destination)
       end
     end
@@ -210,6 +212,18 @@ module Droonga
       @engine_state.local_route?(route)
     end
 
+    def write_step?(step)
+      return false unless step["dataset"]
+
+      step_runner = @step_runners[step["dataset"]]
+      return false unless step_runner
+
+      step_definition = step_runner.find(step["command"])
+      return false unless step_definition
+
+      step_definition.write?
+    end
+
     private
     def farm_path(route)
       @engine_state.farm_path(route)

data/lib/droonga/engine/version.rb

@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.0.3"
+    VERSION = "1.0.4"
   end
 end