droonga-engine 1.1.0 → 1.1.1

data/lib/droonga/distributed_command_planner.rb

@@ -15,8 +15,12 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "droonga/loggable"
+
 module Droonga
   class DistributedCommandPlanner
+    include Loggable
+
     attr_accessor :key
 
     REDUCE_SUM = "sum"
@@ -38,7 +42,9 @@ module Droonga
     end
 
     def plan
-      unified_reducers + unified_gatherers + [fixed_processor]
+      steps = unified_reducers + unified_gatherers + [fixed_processor]
+      logger.debug("distribution plan", :steps => steps)
+      steps
     end
 
     def reduce(params=nil)
@@ -173,5 +179,9 @@ module Droonga
     def plan_errors_handling
       reduce("errors"=> REDUCE_SUM)
     end
+
+    def log_tag
+      "distributed_command_planner"
+    end
   end
 end

data/lib/droonga/engine.rb

@@ -36,20 +36,21 @@ module Droonga
 
     attr_reader :cluster
 
-    def initialize(loop, name, internal_name, options={})
-      @name = name
-      @internal_name = internal_name
-      @loop = loop
+    def initialize(params={})
+      @name = params[:name]
+      @internal_name = params[:internal_name]
+      @loop = params[:loop]
       @catalog = load_catalog
-      @state = EngineState.new(loop, name,
-                               internal_name,
-                               :catalog  => @catalog,
+      @state = EngineState.new(:loop          => @loop,
+                               :name          => @name,
+                               :internal_name => @internal_name,
+                               :catalog       => @catalog,
                                :internal_connection_lifetime =>
-                                 options[:internal_connection_lifetime])
-      @cluster = Cluster.new(loop,
-                             :catalog  => @catalog,
+                                 params[:internal_connection_lifetime])
+      @cluster = Cluster.new(:loop    => @loop,
+                             :catalog => @catalog,
                              :internal_connection_lifetime =>
-                               options[:internal_connection_lifetime])
+                               params[:internal_connection_lifetime])
 
       @dispatcher = create_dispatcher
       @cluster.on_change = lambda do

data/lib/droonga/engine/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,6 +15,6 @@
 
 module Droonga
   class Engine
-    VERSION = "1.1.0"
+    VERSION = "1.1.1"
   end
 end

data/lib/droonga/engine_node.rb

@@ -30,24 +30,19 @@ module Droonga
 
     attr_reader :name
 
-    def initialize(loop, name, state, options={})
-      @loop = loop
-      @name  = name
-      @state = state
+    def initialize(params)
+      @loop  = params[:loop]
+      @name  = params[:name]
+      @state = params[:state]
       logger.trace("initialize: start")
 
-      @buffer = ForwardBuffer.new(name)
-      boundary_timestamp = accept_messages_newer_than_timestamp
-      @buffer.process_messages_newer_than(boundary_timestamp)
-      @buffer.on_forward = lambda do |message, destination|
-        output(message, destination)
-      end
+      @buffer = create_buffer
 
       @node_name = NodeName.parse(@name)
 
       @sender = nil
       @auto_close_timer = nil
-      @auto_close_timeout = options[:auto_close_timeout] ||
+      @auto_close_timeout = params[:auto_close_timeout] ||
                               DEFAULT_AUTO_CLOSE_TIMEOUT_SECONDS
 
       logger.trace("initialize: done")
@@ -112,19 +107,20 @@ module Droonga
 
     def role
       if @state
-        @state["role"]
+        NodeRole.normalize(@state["role"])
       else
         NodeRole::SERVICE_PROVIDER
       end
     end
 
     def live?
-      @state.nil? or @state["live"]
+      @state.nil? or
+        @state["live"] == true
     end
 
     def forwardable?
       return false unless live?
-      role == NodeRole.mine
+      role == sender_role
     end
 
     def readable?
@@ -133,7 +129,7 @@ module Droonga
     end
 
     def writable?
-      case NodeRole.mine
+      case sender_role
       when NodeRole::SERVICE_PROVIDER
         true
       when NodeRole::ABSORB_SOURCE
@@ -181,18 +177,21 @@ module Droonga
     end
 
     private
-    def parse_node_name(name)
-      unless name =~ /\A(.*):(\d+)\/([^.]+)\z/
-        raise "name format: hostname:port/tag"
+    def sender_role
+      NodeRole.mine
+    end
+
+    def create_buffer
+      buffer = ForwardBuffer.new(@name)
+      boundary_timestamp = accept_messages_newer_than_timestamp
+      buffer.process_messages_newer_than(boundary_timestamp)
+      buffer.on_forward = lambda do |message, destination|
+        output(message, destination)
       end
-      {
-        :host => $1,
-        :port => $2,
-        :tag  => $3,
-      }
+      buffer
     end
 
-    def have_unprocessed_messages?
+    def have_unprocessed_messages_in_other_node?
       @state and @state["have_unprocessed_messages"]
     end
 
@@ -222,12 +221,13 @@ module Droonga
     end
 
     def complete_service_provider?
-      service_provider? and not have_unprocessed_messages?
+      service_provider? and
+        not have_unprocessed_messages_in_other_node?
     end
 
     def really_writable?
       return false unless writable?
-      case NodeRole.mine
+      case sender_role
       when NodeRole::SERVICE_PROVIDER
         service_provider?
       when NodeRole::ABSORB_SOURCE
@@ -249,14 +249,14 @@ module Droonga
       command = destination["type"]
       receiver = destination["to"]
       arguments = destination["arguments"]
-      parsed_receiver = parse_node_name(receiver)
+      parsed_receiver = NodeName.parse(receiver)
 
       override_message = {
         "type" => command,
       }
       override_message["arguments"] = arguments if arguments
       message = message.merge(override_message)
-      output_tag = "#{parsed_receiver[:tag]}.message"
+      output_tag = "#{parsed_receiver.tag}.message"
       log_info = "<#{receiver}>:<#{output_tag}>"
       logger.trace("forward: start: #{log_info}")
       sender.send(output_tag, message)

data/lib/droonga/engine_state.rb

@@ -17,6 +17,7 @@ require "English"
 
 require "droonga/loggable"
 require "droonga/deferrable"
+require "droonga/address"
 require "droonga/event_loop"
 require "droonga/forwarder"
 require "droonga/replier"
@@ -37,17 +38,15 @@ module Droonga
     attr_accessor :catalog
     attr_accessor :on_finish
 
-    def initialize(loop, name, internal_name, params)
-      @loop = loop
-      @name = name
-      @internal_name = internal_name
+    def initialize(params)
+      @loop = params[:loop]
+      @name = params[:name]
+      @internal_name = params[:internal_name]
       @internal_connection_lifetime = params[:internal_connection_lifetime]
       @sessions = {}
       @current_id = 0
-      @forwarder = Forwarder.new(@loop,
-                                 :auto_close_timeout =>
-                                   @internal_connection_lifetime)
-      @replier = Replier.new(@forwarder)
+      @forwarder = create_forwarder
+      @replier = create_replier
       @on_finish = nil
       @catalog = params[:catalog]
     end
@@ -68,52 +67,48 @@ module Droonga
       route.start_with?(@name) or route.start_with?(@internal_name)
     end
 
-    FARM_PATH_MATCHER = /\A[^:]+:\d+\/[^.]+/
-
     def internal_route(route)
-      if FARM_PATH_MATCHER =~ route
-        name = $MATCH
-        if name == @name or name == @internal_name
-          return route.sub(name, @internal_name)
-        end
+      name = Address.parse(route).node
+      if name == @name
+        route.sub(name, @internal_name)
+      else
+        route
       end
+    rescue ArgumentError
       route
     end
 
     def public_route(route)
-      if FARM_PATH_MATCHER =~ route
-        name = $MATCH
-        if name == @internal_name
-          return route.sub(name, @name)
-        end
+      name = Address.parse(route).node
+      if name == @internal_name
+        route.sub(name, @name)
+      else
+        route
       end
+    rescue ArgumentError
       route
     end
 
     def internal_farm_path(route)
-      if FARM_PATH_MATCHER =~ route
-        name = $MATCH
-        if name == @name or name == @internal_name
-          @internal_name
-        else
-          name
-        end
+      name = Address.parse(route).node
+      if name == @name
+        @internal_name
       else
-        route
+        name
       end
+    rescue ArgumentError
+      route
     end
 
     def public_farm_path(route)
-      if FARM_PATH_MATCHER =~ route
-        name = $MATCH
-        if name == @internal_name
-          @name
-        else
-          name
-        end
+      name = Address.parse(route).node
+      if name == @internal_name
+        @name
       else
-        route
+        name
       end
+    rescue ArgumentError
+      route
     end
 
     def generate_id
@@ -154,6 +149,16 @@ module Droonga
     end
 
     private
+    def create_forwarder
+      Forwarder.new(@loop,
+                    :auto_close_timeout =>
+                      @internal_connection_lifetime)
+    end
+
+    def create_replier
+      Replier.new(@forwarder)
+    end
+
     def log_tag
       "engine_state"
     end

data/lib/droonga/forward_buffer.rb

@@ -41,16 +41,12 @@ module Droonga
       @unpacker = MessagePack::Unpacker.new
 
       @target = node_name
-      @serf = Serf.new(ENV["DROONGA_ENGINE_NAME"])
-
-      dirname = node_name.gsub("/", ":")
-      @data_directory = Path.intentional_buffer + dirname
-      FileUtils.mkdir_p(@data_directory.to_s)
+      @process_messages_newer_than_timestamp = nil
     end
 
     def add(message, destination)
       logger.trace("add: start")
-      @serf.set_have_unprocessed_messages_for(@target)
+      serf.set_have_unprocessed_messages_for(@target)
       buffered_message = {
         "message"     => message,
         "destination" => destination,
@@ -81,18 +77,18 @@ module Droonga
         # (ex. messages generated by Dispatcher)
         @process_messages_newer_than_timestamp = nil
       end
-      @serf.reset_have_unprocessed_messages_for(@target)
+      serf.reset_have_unprocessed_messages_for(@target)
       logger.trace("start_forward: done")
     end
 
     def buffered_messages
-      Pathname.glob("#{@data_directory}/*#{SUFFIX}").sort_by do |path|
+      Pathname.glob("#{data_directory}/*#{SUFFIX}").sort_by do |path|
         path
       end
     end
 
     def empty?
-      @data_directory.children.empty?
+      data_directory.children.empty?
     end
 
     def process_messages_newer_than(timestamp)
@@ -100,6 +96,21 @@ module Droonga
     end
 
     private
+    def data_directory
+      @data_directory ||= resolve_data_directory
+    end
+
+    def resolve_data_directory
+      dirname = @target.gsub("/", ":")
+      data_directory = Path.intentional_buffer + dirname
+      FileUtils.mkdir_p(data_directory.to_s)
+      data_directory
+    end
+
+    def serf
+      @serf ||= Serf.new(ENV["DROONGA_ENGINE_NAME"] || @target)
+    end
+
     def forward(buffered_message_path)
       logger.trace("forward: start (#{buffered_message_path})")
 
@@ -146,7 +157,7 @@ module Droonga
 
     def create_buffered_message_path(time_stamp=Time.now)
       basename = Timestamp.stringify(time_stamp)
-      Path.unique_file_path(@data_directory, basename, SUFFIX)
+      Path.unique_file_path(data_directory, basename, SUFFIX)
     end
 
     def on_forward(message, destination)

data/lib/droonga/node_role.rb

@@ -26,6 +26,8 @@ module Droonga
 
     ANY = "any".downcase
 
+    #XXX ANY is not a valid role for a node. It is used
+    #    just for checking acceptability of messages.
     ROLES = [
       SERVICE_PROVIDER,
       ABSORB_SOURCE,

data/lib/droonga/plugins/groonga/select.rb

@@ -256,6 +256,9 @@ module Droonga
             converter = RequestConverter.new
             select_request = input_message.body
             search_request = converter.convert(select_request)
+            logger.debug("Conversion of select:",
+                         :select => select_request,
+                         :search => search_request)
             input_message.type = "search"
             input_message.body = search_request
           end

data/lib/droonga/plugins/search.rb

@@ -69,7 +69,9 @@ module Droonga
           # I have to apply it at first, before "limit" and "offset" are applied.
           count_mapper = elements["count"]
           if count_mapper
-            if count_mapper["no_output"]
+            if count_mapper.is_a?(String)
+              value["count"] = value[count_mapper]
+            elsif count_mapper["no_output"]
               value.delete("count")
             else
               value["count"] = value[count_mapper["target"]].size

data/lib/droonga/plugins/search/distributed_search_planner.rb

@@ -39,9 +39,11 @@ module Droonga
 
           ensure_unifiable!
 
+          logger.debug("from", :queries => @request)
           @queries.each do |input_name, query|
             transform_query(input_name, query)
           end
+          logger.debug("to", :queries => @request)
 
           broadcast(:body => @request)
 
@@ -99,6 +101,10 @@ module Droonga
           end
         end
 
+        def log_tag
+          "distributed_search_planner"
+        end
+
         class QueryTransformer
           attr_reader :reducers, :mappers
 
@@ -154,6 +160,8 @@ module Droonga
             @records_offset = final_offset
             @records_limit = final_limit
 
+            return unless @dataset.sliced?
+
             updated_sort_limit = nil
             updated_output_limit = nil
             if final_limit == UNLIMITED
@@ -174,6 +182,7 @@ module Droonga
           end
 
           def calculate_sort_offset!
+            return unless @dataset.sliced?
             # Offset for workers must be zero, because we have to apply "limit" and
             # "offset" on the last gathering phase instead of each reducing phase.
             if rich_sort?
@@ -206,17 +215,18 @@ module Droonga
           end
 
           def calculate_output_offset!
+            return unless @dataset.sliced?
             @output["offset"] = 0 if have_records? and @output["offset"]
           end
 
           def final_offset
-            return @original_output_offset unless @dataset.sliced?
+            return 0 unless @dataset.sliced?
 
             @original_sort_offset + @original_output_offset
           end
 
           def final_limit
-            return @original_output_limit unless @dataset.sliced?
+            return UNLIMITED unless @dataset.sliced?
 
             if @original_sort_limit == UNLIMITED and
                 @original_output_limit == UNLIMITED
@@ -256,10 +266,12 @@ module Droonga
                   @query["sortBy"]["limit"] = UNLIMITED
                 end
                 @output["limit"] = UNLIMITED
+                mapper = {
+                  "target" => "records",
+                }
+              else
+                mapper = "count"
               end
-              mapper = {
-                "target" => "records",
-              }
               unless @output["elements"].include?("records")
                 @records_limit = UNLIMITED if @dataset.sliced?
                 @output["elements"] << "records"