droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/database_scanner.rb

@@ -0,0 +1,106 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga"
+
+module Droonga
+  module DatabaseScanner
+    def n_tables
+      n_tables  = 0
+      each_table do |table|
+        n_tables += 1
+      end
+      n_tables
+    end
+
+    def n_columns
+      n_columns = 0
+      each_table do |table|
+        n_columns += table.columns.size
+      end
+      n_columns
+    end
+
+    def n_records
+      n_records = 0
+      each_table do |table|
+        unless index_only_table?(table)
+          n_records += table.size
+        end
+      end
+      n_records
+    end
+
+    def total_n_objects
+      n_tables  = 0
+      n_columns = 0
+      n_records = 0
+      each_table do |table|
+        n_tables += 1
+        n_columns += table.columns.size
+        unless index_only_table?(table)
+          n_records += table.size
+        end
+      end
+      n_tables + n_columns + n_records
+    end
+
+    def each_table(&block)
+      options = {
+        :ignore_missing_object => true,
+        :order_by => :key,
+      }
+      reference_tables = []
+      @context.database.each(options) do |object|
+        next unless table?(object)
+        if reference_table?(object)
+          reference_tables << object
+          next
+        end
+        yield(object)
+      end
+      reference_tables.each do |reference_table|
+        yield(object)
+      end
+    end
+
+    def table?(object)
+      object.is_a?(::Groonga::Table)
+    end
+
+    def reference_table?(table)
+      table.support_key? and table?(table.domain)
+    end
+
+    def index_only_table?(table)
+      return false if table.columns.empty?
+      table.columns.all? do |column|
+        index_column?(column)
+      end
+    end
+
+    def index_column?(column)
+      column.is_a?(::Groonga::IndexColumn)
+    end
+
+    def each_index_columns(&block)
+      each_table do |table|
+        table.columns.each do |column|
+          yield(column) if index_column?(column)
+        end
+      end
+    end
+  end
+end

data/lib/droonga/{live_nodes_list_loader.rb → deferrable.rb}

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,36 +13,23 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "pathname"
-require "json"
-
 module Droonga
-  class LiveNodesListLoader
-    def initialize(path)
-      @path = path
-    end
+  module Deferrable
+    attr_writer :on_ready, :on_failure
 
-    def load
-      list = parse
-      list.keys
+    def wait_until_ready(target)
+      target.on_ready = lambda do
+        on_ready
+      end
     end
 
     private
-    def parse
-      return default_list unless @path.exist?
-
-      contents = @path.read
-      return default_list if contents.empty?
-
-      begin
-        JSON.parse(contents)
-      rescue JSON::ParserError
-        default_list
-      end
+    def on_ready
+      @on_ready.call if @on_ready
     end
 
-    def default_list
-      {}
+    def on_failure
+      @on_failure.call if @on_failure
     end
   end
 end

data/lib/droonga/differ.rb

@@ -0,0 +1,58 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Droonga
+  module Differ
+    class << self
+      def diff(a, b)
+        unless a.class == b.class
+          return "#{a.inspect} <=> #{b.inspect}"
+        end
+        case a
+        when Hash
+          diff_hashes(a, b)
+        when Array
+          diff_arrays(a, b)
+        else
+          if a == b
+            nil
+          else
+            "#{a.inspect} <=> #{b.inspect}"
+          end
+        end
+      end
+
+      def diff_hashes(a, b)
+        difference = {}
+        (a.keys + b.keys).uniq.each do |key|
+          unless a[key] == b[key]
+            difference[key] = diff(a[key], b[key])
+          end
+        end
+        difference
+      end
+
+      def diff_arrays(a, b)
+        difference = {}
+        [a.size, b.size].max.times do |index|
+          unless a[index] == b[index]
+            difference[index] = diff(a[index], b[index])
+          end
+        end
+        difference
+      end
+    end
+  end
+end

data/lib/droonga/dispatcher.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014 Droonga Project
+# Copyright (C) 2013-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -23,6 +23,7 @@ require "droonga/farm"
 require "droonga/session"
 require "droonga/error_messages"
 require "droonga/distributor"
+require "droonga/node_role"
 
 module Droonga
   class Dispatcher
@@ -46,30 +47,32 @@ module Droonga
       end
     end
 
-    attr_reader :engine_state
+    attr_reader :engine_state, :cluster
 
-    def initialize(engine_state, catalog)
+    def initialize(engine_state, cluster, catalog)
       @engine_state = engine_state
+      @cluster = cluster
       @forwarder = @engine_state.forwarder
       @replier = @engine_state.replier
       @catalog = catalog
       @adapter_runners = create_adapter_runners
       @farm = Farm.new(@engine_state.name, @catalog, @engine_state.loop,
                        :engine_state => @engine_state,
+                       :cluster => @cluster,
                        :dispatcher => self,
-                       :forwarder  => @forwarder)
+                       :forwarder  => @forwarder,
+                       :internal_connection_lifetime =>
+                         @engine_state.internal_connection_lifetime)
+      @engine_state.wait_until_ready(@farm)
       @collector_runners = create_collector_runners
       @step_runners = create_step_runners
     end
 
     def start
-      @farm.on_ready = lambda do
-        @engine_state.on_ready
-      end
       @farm.start
     end
 
-    def stop_gracefully(&on_stop)
+    def stop_gracefully(&block)
       logger.trace("stop_gracefully: start")
       @collector_runners.each_value do |collector_runner|
         collector_runner.shutdown
@@ -77,8 +80,10 @@ module Droonga
       @adapter_runners.each_value do |adapter_runner|
         adapter_runner.shutdown
       end
-      @farm.stop_gracefully(&on_stop)
-      logger.trace("stop_gracefully: done")
+      @farm.stop_gracefully do
+        yield
+        logger.trace("stop_gracefully: done")
+      end
     end
 
     def stop_immediately
@@ -89,11 +94,16 @@ module Droonga
       @adapter_runners.each_value do |adapter_runner|
         adapter_runner.shutdown
       end
-      @farm.shutdown
+      @farm.stop_immediately
       logger.trace("stop_immediately: done")
     end
 
+    def refresh_node_reference
+      @farm.refresh_node_reference
+    end
+
     def process_message(message)
+      logger.trace("process_message: start", :message => message)
       @message = message
       if message["type"] == "dispatcher"
         process_internal_message(message["body"])
@@ -111,11 +121,16 @@ module Droonga
                 "body"       => formatted_error.response_body)
         end
       end
+      logger.trace("process_message: done")
     end
 
     def forward(message, destination)
       logger.trace("forward start")
-      @forwarder.forward(message, destination)
+      if local_route?(destination) or direct_route?(destination)
+        @forwarder.forward(message, destination)
+      else
+        @cluster.forward(message, destination)
+      end
       logger.trace("forward done")
     end
 
@@ -153,82 +168,147 @@ module Droonga
     end
 
     def process_internal_message(message)
+      logger.trace("process_internal_message: start", :message => message)
       id = message["id"]
       session = @engine_state.find_session(id)
       if session
         session.receive(message["input"], message["value"])
       else
+        logger.trace("process_internal_message: no session")
         steps = message["steps"]
         if steps
-          session_planner = SessionPlanner.new(@engine_state, steps)
+          session_planner = SessionPlanner.new(@engine_state, @cluster, steps)
           dataset = message["dataset"] || @message["dataset"]
           collector_runner = @collector_runners[dataset]
           session = session_planner.create_session(id, self, collector_runner)
-          @engine_state.register_session(id, session)
+          if session.need_result?
+            timeout = message["timeout"] ||
+                        @engine_state.internal_connection_lifetime
+            @engine_state.register_session(id, session,
+                                           :timeout => timeout)
+            session.start
+            logger.trace("process_internal_message: waiting for results")
+          else
+            session.start
+            session.finish
+            session = nil
+            logger.trace("process_internal_message: no need to wait for results")
+          end
         else
-          logger.error("no steps error: id=#{id}, message=#{message}")
+          logger.error("no steps error", :id => id, :message => message)
           return
           #todo: take cases receiving result before its query into account
         end
-        session.start
       end
-      @engine_state.unregister_session(id) if session.done?
+      @engine_state.unregister_session(id) if session and session.done?
+      logger.trace("process_internal_message: done")
     end
 
     def dispatch(message, destination)
-      if local?(destination)
+      logger.trace("dispatch: start", :message => message, :destination => destination)
+      if local_route?(destination)
         process_internal_message(message)
       else
-        @forwarder.forward(@message.merge("body" => message),
-                           "type" => "dispatcher",
-                           "to"   => destination)
+        forward_message = @message.merge("body" => message)
+        forward_destination = {
+          "type" => "dispatcher",
+          "to"   => destination,
+        }
+        if direct_route?(forward_destination)
+          @forwarder.forward(forward_message, forward_destination)
+        else
+          @cluster.forward(forward_message, forward_destination)
+        end
       end
+      logger.trace("dispatch: done")
     end
 
     def dispatch_steps(steps)
+      logger.trace("dispatch_steps: start", :steps => steps)
       id = @engine_state.generate_id
 
       destinations = []
+      timeout = nil
       steps.each do |step|
+        calculated_timeout = timeout_from_step(step)
+        if calculated_timeout
+          timeout = calculated_timeout
+        end
+
         dataset = @catalog.dataset(step["dataset"])
         if dataset
-          routes = dataset.compute_routes(step, @engine_state.live_nodes)
-          step["routes"] = routes
+          if write_step?(step)
+            step["write"] = true
+            target_nodes = @cluster.writable_nodes
+            if target_nodes.empty?
+              logger.error("there is no node to dispath a write step!",
+                           :my_role   => NodeRole.mine,
+                           :all_nodes => @cluster.engine_nodes.collect(&:to_json),
+                           :step      => step)
+            end
+          else
+            target_nodes = @cluster.readable_nodes
+            if target_nodes.empty?
+              logger.error("there is no node to dispath a read step!",
+                           :my_role   => NodeRole.mine,
+                           :all_nodes => @cluster.engine_nodes.collect(&:to_json),
+                           :step      => step)
+            end
+          end
+          routes = dataset.compute_routes(step, target_nodes)
+          step["routes"] = routes.collect do |route|
+            internal_route(route)
+          end
         else
           step["routes"] ||= [id]
         end
+
         destinations += step["routes"].collect do |route|
-          farm_path(route)
+          internal_farm_path(route)
         end
       end
 
-      dispatch_message = { "id" => id, "steps" => steps }
+      dispatch_message = {
+        "id"    => id,
+        "steps" => steps,
+        "timeout" => timeout,
+      }
       destinations.uniq.each do |destination|
         dispatch(dispatch_message, destination)
       end
+
+      logger.trace("dispatch_steps: done")
     end
 
     def process_local_message(local_message)
+      logger.trace("process_local_message: start", :steps => local_message)
       task = local_message["task"]
       slice_name = task["route"]
+      slice_name = public_route(slice_name)
       step = task["step"]
       command = step["command"]
       descendants = {}
       step["descendants"].each do |name, routes|
         descendants[name] = routes.collect do |route|
-          farm_path(route)
+          internal_farm_path(route)
         end
       end
       local_message["descendants"] = descendants
       farm_message = @message.merge("body" => local_message,
                                     "type" => command)
       @farm.process(slice_name, farm_message)
+      logger.trace("process_local_message: done")
     end
 
-    def local?(route)
+    def local_route?(route)
       @engine_state.local_route?(route)
     end
 
+    def direct_route?(route)
+      receiver = route["to"]
+      not @cluster.engine_node_names.include?(receiver)
+    end
+
     def write_step?(step)
       return false unless step["dataset"]
 
@@ -241,12 +321,46 @@ module Droonga
       step_definition.write?
     end
 
+    def timeout_from_step(step)
+      return nil unless step["dataset"]
+
+      step_runner = @step_runners[step["dataset"]]
+      return nil unless step_runner
+
+      step_definition = step_runner.find(step["command"])
+      return nil unless step_definition
+
+      step_definition.timeout_for_step(step)
+    end
+
     private
-    def farm_path(route)
-      @engine_state.farm_path(route)
+    def internal_route(route)
+      @engine_state.internal_route(route)
+    end
+
+    def public_route(route)
+      @engine_state.public_route(route)
+    end
+
+    def internal_farm_path(route)
+      @engine_state.internal_farm_path(route)
+    end
+
+    def public_farm_path(route)
+      @engine_state.public_farm_path(route)
+    end
+
+    def acceptable_role?(message)
+      return true unless message["targetRole"]
+      role = message["targetRole"].downcase
+      role == NodeRole::ANY or role == NodeRole.mine
     end
 
     def process_input_message(message)
+      unless acceptable_role?(message)
+        @cluster.bounce(message)
+        return
+      end
       dataset = message["dataset"]
       adapter_runner = @adapter_runners[dataset]
       adapted_message = adapter_runner.adapt_input(message)
@@ -255,6 +369,7 @@ module Droonga
       distributor = Distributor.new(self, plan)
       distributor.distribute
     rescue Droonga::UnsupportedMessageError => error
+      logger.trace("process_input_message: rescue", :error => error)
       target_message = error.message
       raise UnknownType.new(target_message["type"], target_message["dataset"])
     end
@@ -302,8 +417,9 @@ module Droonga
     class SessionPlanner
       attr_reader :steps
 
-      def initialize(engine_state, steps)
+      def initialize(engine_state, cluster, steps)
         @engine_state = engine_state
+        @cluster = cluster
         @steps = steps
       end
 
@@ -345,14 +461,21 @@ module Droonga
           (step["outputs"] || []).each do |output|
             descendants[output] = []
             @descendants[output].each do |index|
-              live_routes = @engine_state.remove_dead_routes(step["routes"])
-              @steps[index]["n_of_expects"] += live_routes.size
+              responsive_routes = select_responsive_routes(step["routes"])
+              @steps[index]["n_of_expects"] += responsive_routes.size
               descendants[output].concat(@steps[index]["routes"])
             end
           end
           step["descendants"] = descendants
         end
       end
+
+      def select_responsive_routes(routes)
+        selected_nodes = @cluster.readable_nodes
+        routes.select do |route|
+          selected_nodes.include?(@engine_state.public_farm_path(route))
+        end
+      end
     end
   end
 end