droonga-engine 1.0.4 → 1.0.5

data/lib/droonga/command/serf_event_handler.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "json"
 
@@ -34,14 +34,22 @@ module Droonga
         @serf = ENV["SERF"] || Serf.path
         @serf_rpc_address = ENV["SERF_RPC_ADDRESS"] || "127.0.0.1:7373"
         @serf_name = ENV["SERF_SELF_NAME"]
+        @response = {
+          "log" => []
+        }
       end
 
       def run
         parse_event
-        return true unless event_for_me?
+        unless event_for_me?
+          log(" => ignoring event not for me")
+          output_response
+          return true
+        end
 
         process_event
         output_live_nodes
+        output_response
         true
       end
 
@@ -53,11 +61,13 @@ module Droonga
         when "user"
           @event_sub_name = ENV["SERF_USER_EVENT"]
           @payload = JSON.parse($stdin.gets)
-          puts "event sub name = #{@event_sub_name}"
+          log("event sub name = #{@event_sub_name}")
         when "query"
           @event_sub_name = ENV["SERF_QUERY_NAME"]
           @payload = JSON.parse($stdin.gets)
-          puts "event sub name = #{@event_sub_name}"
+          log("event sub name = #{@event_sub_name}")
+        when "member-join", "member-leave", "member-update", "member-reap"
+          output_live_nodes
         end
       end
 
@@ -72,6 +82,8 @@ module Droonga
         case @event_sub_name
         when "change_role"
           save_status(:role, @payload["role"])
+        when "report_status"
+          report_status
         when "join"
           join
         when "set_replicas"
@@ -82,9 +94,17 @@ module Droonga
           remove_replicas
         when "absorb_data"
           absorb_data
+        when "publish_catalog"
+          publish_catalog
+        when "unpublish_catalog"
+          unpublish_catalog
         end
       end
 
+      def output_response
+        puts JSON.generate(@response)
+      end
+
       def host
         @serf_name.split(":").first
       end
@@ -96,9 +116,13 @@ module Droonga
         hosts
       end
 
+      def report_status
+        @response["value"] = status(@payload["key"].to_sym)
+      end
+
       def join
         type = @payload["type"]
-        puts "type = #{type}"
+        log("type = #{type}")
         case type
         when "replica"
           join_as_replica
@@ -106,55 +130,106 @@ module Droonga
       end
 
       def join_as_replica
-        source = @payload["source"]
-        return unless source
+        source_node = @payload["source"]
+        return unless source_node
 
-        puts "source  = #{source}"
+        log("source_node  = #{source_node}")
 
-        generator = create_current_catalog_generator
-        dataset = generator.dataset_for_host(source) ||
+        source_host = source_node.split(":").first
+
+        # fetch_port = @payload["fetch_port"]
+        # catalog = fetch_catalog(source_node, fetch_port)
+        catalog = JSON.parse(Path.catalog.read) #XXX workaround until "fetch" become available
+
+        generator = create_current_catalog_generator(catalog)
+        dataset = generator.dataset_for_host(source_host) ||
                     generator.dataset_for_host(host)
         return unless dataset
 
+        # restart self with the fetched catalog.
+        SafeFileWriter.write(Path.catalog, JSON.pretty_generate(catalog))
+
         dataset_name = dataset.name
         tag          = dataset.replicas.tag
         port         = dataset.replicas.port
         other_hosts  = dataset.replicas.hosts
 
-        puts "dataset = #{dataset_name}"
-        puts "port    = #{port}"
-        puts "tag     = #{tag}"
+        log("dataset = #{dataset_name}")
+        log("port    = #{port}")
+        log("tag     = #{tag}")
 
         if @payload["copy"]
-          puts "starting to copy data from #{source}"
+          log("starting to copy data from #{source_host}")
 
           modify_catalog do |modifier|
             modifier.datasets[dataset_name].replicas.hosts = [host]
           end
-          sleep(1) # wait for restart
+          sleep(5) #TODO: wait for restart. this should be done more safely, to avoid starting of absorbing with old catalog.json.
 
+          save_status(:absorbing, true)
           DataAbsorber.absorb(:dataset          => dataset_name,
-                              :source_host      => source,
+                              :source_host      => source_host,
                               :destination_host => host,
                               :port             => port,
                               :tag              => tag)
+          delete_status(:absorbing)
           sleep(1)
         end
 
-        puts "joining to the cluster: update myself"
+        log("joining to the cluster: update myself")
 
         modify_catalog do |modifier|
           modifier.datasets[dataset_name].replicas.hosts += other_hosts
           modifier.datasets[dataset_name].replicas.hosts.uniq!
         end
-        sleep(1) # wait for restart
+      end
+
+      def fetch_catalog(source_node, port)
+        source_host = source_node.split(":").first
+
+        url = "http://#{source_host}:#{port}"
+        connection = Faraday.new(url) do |builder|
+          builder.response(:follow_redirects)
+          builder.adapter(Faraday.default_adapter)
+        end
+        response = connection.get("/catalog.json")
+        catalog = response.body
+
+        JSON.parse(catalog)
+      end
+
+      def publish_catalog
+        port = @payload["port"]
+        return unless port
+
+        env = {}
+        publisher_command_line = [
+          "droonga-engine-data-publisher",
+            "--base-dir", Path.base.to_s,
+            "--port", port.to_s,
+            "--published-file", Path.catalog.to_s
+        ]
+        pid = spawn(env, *publisher_command_line)
+        Process.detach(pid)
+        sleep(1) # wait until the directory is published
+
+        published_dir = Path.published(port)
+        pid_file = published_dir + ".pid"
+
+        File.open(pid_file.to_s, "w") do |file|
+          file.puts(pid)
+        end
+      end
 
-        puts "joining to the cluster: update others"
+      def unpublish_catalog
+        port = @payload["port"]
+        return unless port
 
-        source_node  = "#{source}:#{port}/#{tag}"
-        Serf.send_query(source_node, "add_replicas",
-                        "dataset" => dataset_name,
-                        "hosts"   => [host])
+        published_dir = Path.published(port)
+        pid_file = published_dir + ".pid"
+        pid = pid_file.read.to_i
+
+        Process.kill("INT", pid)
       end
 
       def set_replicas
@@ -164,7 +239,7 @@ module Droonga
         hosts = given_hosts
         return unless hosts
 
-        puts "new replicas: #{hosts.join(",")}"
+        log("new replicas: #{hosts.join(",")}")
 
         modify_catalog do |modifier|
           modifier.datasets[dataset].replicas.hosts = hosts
@@ -181,7 +256,7 @@ module Droonga
         hosts -= [host]
         return if hosts.empty?
 
-        puts "adding replicas: #{hosts.join(",")}"
+        log("adding replicas: #{hosts.join(",")}")
 
         modify_catalog do |modifier|
           modifier.datasets[dataset].replicas.hosts += hosts
@@ -196,7 +271,7 @@ module Droonga
         hosts = given_hosts
         return unless hosts
 
-        puts "removing replicas: #{hosts.join(",")}"
+        log("removing replicas: #{hosts.join(",")}")
 
         modify_catalog do |modifier|
           modifier.datasets[dataset].replicas.hosts -= hosts
@@ -209,19 +284,17 @@ module Droonga
         SafeFileWriter.write(Path.catalog, JSON.pretty_generate(generator.generate))
       end
 
-      def create_current_catalog_generator
-        current_catalog = JSON.parse(Path.catalog.read)
+      def create_current_catalog_generator(current_catalog=nil)
+        current_catalog ||= JSON.parse(Path.catalog.read)
         generator = CatalogGenerator.new
         generator.load(current_catalog)
       end
 
       def absorb_data
-        return unless event_for_me?
-
         source = @payload["source"]
         return unless source
 
-        puts "start to absorb data from #{source}"
+        log("start to absorb data from #{source}")
 
         dataset_name = @payload["dataset"]
         port         = @payload["port"]
@@ -240,29 +313,22 @@ module Droonga
           tag  = dataset.replicas.tag
         end
 
-        puts "dataset = #{dataset_name}"
-        puts "port    = #{port}"
-        puts "tag     = #{tag}"
+        log("dataset = #{dataset_name}")
+        log("port    = #{port}")
+        log("tag     = #{tag}")
 
+        save_status(:absorbing, true)
         DataAbsorber.absorb(:dataset          => dataset_name,
                             :source_host      => source,
                             :destination_host => host,
                             :port             => port,
-                            :tag              => tag)
+                            :tag              => tag,
+                            :client           => "droonga-send")
+        delete_status(:absorbing)
       end
 
       def live_nodes
-        nodes = {}
-        members = `#{@serf} members -rpc-addr #{@serf_rpc_address}`
-        members.each_line do |member|
-          name, address, status, = member.strip.split(/\s+/)
-          if status == "alive"
-            nodes[name] = {
-              "serfAddress" => address,
-            }
-          end
-        end
-        nodes
+        Serf.live_nodes(@serf_name)
       end
 
       def output_live_nodes
@@ -272,11 +338,25 @@ module Droonga
         SafeFileWriter.write(path, file_contents)
       end
 
+      def status(key)
+        Serf.status(key)
+      end
+
       def save_status(key, value)
         status = Serf.load_status
         status[key] = value
         SafeFileWriter.write(Serf.status_file, JSON.pretty_generate(status))
       end
+
+      def delete_status(key)
+        status = Serf.load_status
+        status.delete(key)
+        SafeFileWriter.write(Serf.status_file, JSON.pretty_generate(status))
+      end
+
+      def log(message)
+        @response["log"] << message
+      end
     end
   end
 end

data/lib/droonga/data_absorber.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "open3"
 
@@ -28,25 +28,43 @@ module Droonga
         drndump_options += ["--receiver-host", params[:destination_host]]
         drndump_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
 
+        #TODO: We should use droonga-send instead of droonga-request,
+        #      because droonga-request is too slow.
+        #      However, to do it, we have to implement an API to know
+        #      that all messages sent by droonga-send are completely
+        #      processed.
         client = params[:client] || "droonga-request"
         client_options = []
-        client_options += ["--host", params[:destination_host]]
-        client_options += ["--port", params[:port].to_s] if params[:port]
-        client_options += ["--tag", params[:tag]] if params[:tag]
-        client_options += ["--receiver-host", params[:destination_host]]
-        client_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
+        if client.include?("droonga-request")
+          client_options += ["--host", params[:destination_host]]
+          client_options += ["--port", params[:port].to_s] if params[:port]
+          client_options += ["--tag", params[:tag]] if params[:tag]
+          client_options += ["--receiver-host", params[:destination_host]]
+          client_options += ["--receiver-port", params[:receiver_port].to_s] if params[:receiver_port]
+        elsif client.include?("droonga-send")
+          #XXX Don't use round-robin with multiple endpoints
+          #    even if there are too much data.
+          #    Schema and indexes must be sent to just one endpoint
+          #    to keep their order, but currently there is no way to
+          #    extract only schema and indexes via drndump.
+          #    So, we always use just one endpoint for now,
+          #    even if there are too much data.
+          server = "droonga:#{params[:destination_host]}"
+          server = "#{server}:#{params[:port].to_s}" if params[:port]
+          server = "#{server}/#{params[:tag].to_s}" if params[:tag]
+          client_options += ["--server", server]
+        else
+          raise ArgumentError.new("Unknwon type client: #{client}")
+        end
 
         drndump_command_line = [drndump] + drndump_options
         client_command_line = [client] + client_options
 
-        Open3.popen3(*drndump_command_line) do |dump_in, dump_out, dump_error, dump_thread|
-          dump_in.close
-          Open3.popen3(*client_command_line) do |client_in, client_out, client_error, client_thread|
-            client_out.close
-            dump_out.each do |dump|
-              yield dump if block_given?
-              client_in.puts(dump)
-            end
+        env = {}
+        Open3.pipeline_r([env, *drndump_command_line],
+                         [env, *client_command_line]) do |last_stdout, thread|
+          last_stdout.each do |output|
+            yield output if block_given?
           end
         end
       end

data/lib/droonga/dispatcher.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "tsort"
 
@@ -63,6 +63,9 @@ module Droonga
     end
 
     def start
+      @farm.on_ready = lambda do
+        @engine_state.on_ready
+      end
       @farm.start
     end
 
@@ -143,10 +146,10 @@ module Droonga
       else
         steps = message["steps"]
         if steps
-          session_planner = SessionPlanner.new(self, steps)
+          session_planner = SessionPlanner.new(@engine_state, steps)
           dataset = message["dataset"] || @message["dataset"]
           collector_runner = @collector_runners[dataset]
-          session = session_planner.create_session(id, collector_runner)
+          session = session_planner.create_session(id, self, collector_runner)
           @engine_state.register_session(id, session)
         else
           logger.error("no steps error: id=#{id}, message=#{message}")
@@ -175,7 +178,7 @@ module Droonga
       steps.each do |step|
         dataset = @catalog.dataset(step["dataset"])
         if dataset
-          routes = dataset.get_routes(step, @engine_state.live_nodes)
+          routes = dataset.compute_routes(step, @engine_state.live_nodes)
           step["routes"] = routes
         else
           step["routes"] ||= [id]
@@ -279,24 +282,24 @@ module Droonga
     end
 
     def log_tag
-      "[#{Process.ppid}][#{Process.pid}] dispatcher"
+      "dispatcher"
     end
 
     class SessionPlanner
       attr_reader :steps
 
-      def initialize(dispatcher, steps)
-        @dispatcher = dispatcher
+      def initialize(engine_state, steps)
+        @engine_state = engine_state
         @steps = steps
       end
 
-      def create_session(id, collector_runner)
+      def create_session(id, dispatcher, collector_runner)
         resolve_descendants
         tasks = []
         inputs = {}
         @steps.each do |step|
           step["routes"].each do |route|
-            next unless @dispatcher.local?(route)
+            next unless @engine_state.local_route?(route)
             task = {
               "route" => route,
               "step" => step,
@@ -310,7 +313,7 @@ module Droonga
             end
           end
         end
-        Session.new(id, @dispatcher, collector_runner, tasks, inputs)
+        Session.new(id, dispatcher, collector_runner, tasks, inputs)
       end
 
       def resolve_descendants
@@ -328,7 +331,8 @@ module Droonga
           (step["outputs"] || []).each do |output|
             descendants[output] = []
             @descendants[output].each do |index|
-              @steps[index]["n_of_expects"] += step["routes"].size
+              live_routes = @engine_state.remove_dead_routes(step["routes"])
+              @steps[index]["n_of_expects"] += live_routes.size
               descendants[output].concat(@steps[index]["routes"])
             end
           end

data/lib/droonga/distributed_command_planner.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class DistributedCommandPlanner

data/lib/droonga/distributor.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "tsort"
 

data/lib/droonga/engine/version.rb

@@ -11,10 +11,10 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class Engine
-    VERSION = "1.0.4"
+    VERSION = "1.0.5"
   end
 end

data/lib/droonga/engine.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "time"
 require "fileutils"
@@ -29,6 +29,7 @@ module Droonga
   class Engine
     include Loggable
 
+    attr_writer :on_ready
     def initialize(loop, name, internal_name)
       @state = EngineState.new(loop, name, internal_name)
       @catalog = load_catalog
@@ -38,10 +39,14 @@ module Droonga
       @live_nodes_list_observer.on_change = lambda do
         @state.live_nodes = load_live_nodes
       end
+      @on_ready = nil
     end
 
     def start
       logger.trace("start: start")
+      @state.on_ready = lambda do
+        @on_ready.call if @on_ready
+      end
       @state.start
       @live_nodes_list_observer.start
       @dispatcher.start
@@ -87,7 +92,7 @@ module Droonga
       loader = CatalogLoader.new(catalog_path.to_s)
       catalog = loader.load
       logger.info("catalog loaded",
-                  :path  => catalog_path,
+                  :path  => catalog_path.to_s,
                   :mtime => catalog_path.mtime)
       catalog
     end
@@ -97,7 +102,7 @@ module Droonga
       loader = LiveNodesListLoader.new(path)
       live_nodes = loader.load
       logger.info("live-nodes loaded",
-                  :path  => path,
+                  :path  => path.to_s,
                   :mtime => path.mtime)
       live_nodes
     end

data/lib/droonga/engine_state.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "English"
 
@@ -31,21 +31,22 @@ module Droonga
     attr_reader :internal_name
     attr_reader :forwarder
     attr_reader :replier
+    attr_writer :on_ready
     attr_accessor :on_finish
     attr_accessor :catalog
-    attr_writer :dead_nodes
     def initialize(loop, name, internal_name)
       @loop = loop
       @name = name
       @internal_name = internal_name
       @sessions = {}
       @current_id = 0
-      @forwarder = Forwarder.new(@loop)
-      @forwarder.resume
+      @forwarder = Forwarder.new(@loop, :buffering => true)
       @replier = Replier.new(@forwarder)
+      @on_ready = nil
       @on_finish = nil
       @catalog = nil
       @live_nodes = nil
+      @dead_nodes = []
     end
 
     def start
@@ -111,13 +112,21 @@ module Droonga
     end
 
     def live_nodes=(nodes)
+      old_live_nodes = @live_nodes
       @live_nodes = nodes
       @dead_nodes = all_nodes - @live_nodes
+      @forwarder.resume if old_live_nodes != @live_nodes
       @live_nodes
     end
 
-    def dead_nodes
-      @dead_nodes || []
+    def remove_dead_routes(routes)
+      routes.reject do |route|
+        @dead_nodes.include?(farm_path(route))
+      end
+    end
+
+    def on_ready
+      @on_ready.call if @on_ready
     end
 
     private

data/lib/droonga/error.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Droonga
   class Error < StandardError

data/lib/droonga/error_messages.rb

@@ -11,7 +11,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "droonga/error"
 require "droonga/status_code"

data/lib/droonga/event_loop.rb

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "coolio"