fluent-plugin-droonga 0.7.0 → 0.8.0

data/lib/droonga/pluggable.rb

@@ -17,6 +17,14 @@
 
 module Droonga
   module Pluggable
+    class UnknownPlugin < StandardError
+      attr_reader :command
+
+      def initialize(command)
+        @command = command
+      end
+    end
+
     def shutdown
       $log.trace("#{log_tag}: shutdown: plugin: start")
       @plugins.each do |plugin|
@@ -33,10 +41,7 @@ module Droonga
       plugin = find_plugin(command)
       $log.trace("#{log_tag}: process: start: <#{command}>",
                  :plugin => plugin.class)
-      if plugin.nil?
-        raise "unknown plugin: <#{command}>: " +
-                "TODO: improve error handling"
-      end
+      raise UnknownPlugin.new(command) if plugin.nil?
       plugin.process(command, *arguments)
       $log.trace("#{log_tag}: process: done: <#{command}>",
                  :plugin => plugin.class)

data/lib/droonga/plugin.rb

@@ -33,11 +33,20 @@ module Droonga
     end
 
     def process(command, *arguments)
-      __send__(self.class.method_name(command), *arguments)
+      run_command(command, *arguments)
     rescue => exception
+      process_error(command, exception, arguments)
+    end
+    
+    private
+    def run_command(command, *arguments)
+      __send__(self.class.method_name(command), *arguments)
+    end
+
+    def process_error(command, error, arguments)
       Logger.error("error while processing #{command}",
-                   arguments: arguments,
-                   exception: exception)
+                   error,
+                   :arguments => arguments)
     end
   end
 end

data/lib/droonga/plugin/collector/basic.rb

@@ -27,21 +27,47 @@ module Droonga
     def collector_gather(result)
       output = body ? body[input_name] : input_name
       if output.is_a?(Hash)
-        element = output["element"]
-        if element
-          result[element] = apply_output_range(result[element], output)
-          result[element] = apply_output_attributes_and_format(result[element], output)
+        elements = output["elements"]
+        if elements && elements.is_a?(Hash)
+          # phase 1: pre-process
+          elements.each do |element, mapper|
+            case mapper["type"]
+            when "count"
+              result[element] = result[mapper["target"]].size
+            when "sort"
+              # do nothing on this phase!
+            end
+          end
+          # phase 2: post-process
+          elements.each do |element, mapper|
+            if mapper["no_output"]
+              result.delete(element)
+              next
+            end
+
+            case mapper["type"]
+            when "count"
+              # do nothing on this phase!
+            when "sort"
+              # because "count" type mapper requires all items of the array,
+              # I have to apply "sort" type mapper later.
+              if result[element]
+                result[element] = apply_output_range(result[element], mapper)
+                result[element] = apply_output_attributes_and_format(result[element], mapper)
+              end
+            end
+          end
         end
         output = output["output"]
       end
-      emit(result, output)
+      emit(output, result)
     end
 
     def apply_output_range(items, output)
       if items && items.is_a?(Array)
         offset = output["offset"] || 0
         unless offset.zero?
-          items = items[offset..-1]
+          items = items[offset..-1] || []
         end
 
         limit = output["limit"] || 0
@@ -80,7 +106,7 @@ module Droonga
         value = request
         old_value = output_values[output]
         value = reduce(elements, old_value, request) if old_value
-        emit(value, output)
+        emit(output, value)
       end
     end
 
@@ -93,7 +119,10 @@ module Droonga
         when "sum"
           reduced_values = values[0][key] + values[1][key]
         when "sort"
-          reduced_values = merge(values[0][key], values[1][key], deal["operators"])
+          reduced_values = merge(values[0][key],
+                                 values[1][key],
+                                 :operators => deal["operators"],
+                                 :key_column => deal["key_column"])
         end
 
         reduced_values = apply_output_range(reduced_values, "limit" => deal["limit"])
@@ -103,16 +132,10 @@ module Droonga
       return result
     end
 
-    def merge(x, y, operators)
-      # Normalize operators at first for optimization.
-      operators ||= []
-      operators = operators.collect do |operator|
-        if operator.is_a?(String)
-          { "operator" => operator }
-        else
-          operator
-        end
-      end
+    def merge(x, y, options={})
+      operators = options[:operators] = normalize_operators(options[:operators])
+
+      unify_by_key!(x, y, options)
 
       index = 0
       y.each do |_y|
@@ -128,6 +151,17 @@ module Droonga
       return x
     end
 
+    def normalize_operators(operators)
+      operators ||= []
+      operators.collect do |operator|
+        if operator.is_a?(String)
+          { "operator" => operator }
+        else
+          operator
+        end
+      end
+    end
+
     def compare(x, y, operators)
       operators.each_with_index do |operator, index|
         column = operator["column"] || index
@@ -138,5 +172,44 @@ module Droonga
       end
       return false
     end
+
+    def unify_by_key!(base_items, unified_items, options={})
+      key_column_index = options[:key_column]
+      return unless key_column_index
+
+      # The unified records must be smaller than the base, because
+      # I sort unified records at last. I want to sort only smaller array.
+      if base_items.size < unified_items.size
+        base_items, unified_items = unified_items, base_items
+      end
+
+      rest_unified_items = unified_items.dup
+
+      base_items.reject! do |base_item|
+        key = base_item[key_column_index]
+        rest_unified_items.any? do |unified_item|
+          if unified_item[key_column_index] == key
+            base_item.each_with_index do |value, column|
+              next if column == key_column_index
+              unified_item[column] += value
+            end
+            rest_unified_items -= [unified_item]
+            true
+          else
+            false
+          end
+        end
+      end
+
+      unless rest_unified_items.size == unified_items.size
+        unified_items.sort! do |a, b|
+          if compare(a, b, options[:operators])
+            -1
+          else
+            1
+          end
+        end
+      end
+    end
   end
 end

data/lib/droonga/plugin/distributor/crud.rb

@@ -22,22 +22,22 @@ module Droonga
     repository.register("crud", self)
 
     command :add
-    def add(envelope)
-      key = envelope["body"]["key"] || rand.to_s
-      scatter_all(envelope, key)
+    def add(message)
+      key = message["body"]["key"] || rand.to_s
+      scatter_all(message, key)
     end
 
     command :update
-    def update(envelope)
-      key = envelope["body"]["key"] || rand.to_s
-      scatter_all(envelope, key)
+    def update(message)
+      key = message["body"]["key"] || rand.to_s
+      scatter_all(message, key)
     end
 
     # TODO: What is this?
     command :reset
-    def reset(envelope)
-      key = envelope["body"]["key"] || rand.to_s
-      scatter_all(envelope, key)
+    def reset(message)
+      key = message["body"]["key"] || rand.to_s
+      scatter_all(message, key)
     end
   end
 end

data/lib/droonga/plugin/distributor/distributed_search_planner.rb

@@ -0,0 +1,401 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Droonga
+  class DistributedSearchPlanner
+    attr_reader :messages
+
+    def initialize(search_request_message)
+      @source_message = search_request_message
+      @request = @source_message["body"]
+      @queries = @request["queries"]
+
+      @input_names = []
+      @output_names = []
+      @output_mappers = {}
+      @messages = []
+
+      build_messages
+    end
+
+    private
+    UNLIMITED = -1
+
+    def build_messages
+      Searcher::QuerySorter.validate_dependencies(@queries)
+
+      ensure_unifiable!
+
+      @queries.each do |input_name, query|
+        transform_query(input_name, query)
+      end
+
+      gatherer = {
+        "type" => "gather",
+        "body" => @output_mappers,
+        "inputs" => @output_names, # XXX should be placed in the "body"?
+        "post" => true, # XXX should be placed in the "body"?
+      }
+      @messages << gatherer
+      searcher = {
+        "type" => "broadcast",
+        "command" => "search", # XXX should be placed in the "body"?
+        "dataset" => @source_message["dataset"] || @request["dataset"],
+        "body" => @request,
+        "outputs" => @input_names, # XXX should be placed in the "body"?
+        "replica" => "random", # XXX should be placed in the "body"?
+      }
+      @messages.push(searcher)
+    end
+
+    def ensure_unifiable!
+      @queries.each do |name, query|
+        if unifiable?(name) && query["output"]
+          query["output"]["unifiable"] = true
+        end
+      end
+    end
+
+    def unifiable?(name)
+      query = @queries[name]
+      return true if query["groupBy"]
+      name = query["source"]
+      return false unless @queries.keys.include?(name)
+      unifiable?(name)
+    end
+
+    def transform_query(input_name, query)
+      output = query["output"]
+
+      # Skip reducing phase for a result with no output.
+      if output.nil? or
+           output["elements"].nil? or
+           (!output["elements"].include?("count") &&
+            !output["elements"].include?("records"))
+        return
+      end
+
+      @input_names << input_name
+      output_name = input_name + "_reduced"
+      @output_names << output_name
+
+      transformer = QueryTransformer.new(query)
+
+      reducer = {
+        "type" => "reduce",
+        "body" => {
+          input_name => {
+            output_name => transformer.reducers,
+          },
+        },
+        "inputs" => [input_name], # XXX should be placed in the "body"?
+        "outputs" => [output_name], # XXX should be placed in the "body"?
+      }
+      @messages << reducer
+
+      @output_mappers[output_name] = {
+        "output" => input_name,
+        "elements" => transformer.mappers,
+      }
+    end
+
+    class QueryTransformer
+      attr_reader :reducers, :mappers
+
+      def initialize(query)
+        @query = query
+        @output = @query["output"]
+        @reducers = {}
+        @mappers = {}
+        @output_records = true
+        transform!
+      end
+
+      def transform!
+        # The collector module supports only "simple" format search results.
+        # So we have to override the format and restore it on the gathering
+        # phase.
+        @records_format = @output["format"] || "simple"
+        @output["format"] = "simple"
+
+        @sort_keys = @query["sortBy"] || []
+        @sort_keys = @sort_keys["keys"] || [] if @sort_keys.is_a?(Hash)
+
+        calculate_offset_and_limit!
+        build_count_mapper_and_reducer!
+        build_records_mapper_and_reducer!
+      end
+
+      def calculate_offset_and_limit!
+        @original_sort_offset = sort_offset
+        @original_output_offset = output_offset
+        @original_sort_limit = sort_limit
+        @original_output_limit = output_limit
+
+        calculate_sort_offset!
+        calculate_output_offset!
+
+        # We have to calculate limit based on offset.
+        # <A, B = limited integer (0...MAXINT)>
+        # | sort limit | output limit | => | worker's sort limit      | worker's output limit   | final limit |
+        # =============================    ====================================================================
+        # | UNLIMITED  | UNLIMITED    | => | UNLIMITED                | UNLIMITED               | UNLIMITED   |
+        # | UNLIMITED  | B            | => | final_offset + B         | final_offset + B        | B           |
+        # | A          | UNLIMITED    | => | final_offset + A         | final_offset + A        | A           |
+        # | A          | B            | => | final_offset + max(A, B) | final_offset + min(A, B)| min(A, B)   |
+
+        # XXX final_limit and final_offset calculated in many times
+
+        @records_offset = final_offset
+        @records_limit = final_limit
+
+        if final_limit == UNLIMITED
+          @output["limit"] = UNLIMITED
+        else
+          if rich_sort?
+            @query["sortBy"]["limit"] = final_offset + [sort_limit, output_limit].max
+          end
+          @output["limit"] = final_offset + final_limit
+        end
+      end
+
+      def calculate_sort_offset!
+        # Offset for workers must be zero, because we have to apply "limit" and
+        # "offset" on the last gathering phase instead of each reducing phase.
+        if rich_sort?
+          @query["sortBy"]["offset"] = 0
+        end
+      end
+
+      def sort_offset
+        if rich_sort?
+          @query["sortBy"]["offset"] || 0
+        else
+          0
+        end
+      end
+
+      def output_offset
+        @output["offset"] || 0
+      end
+
+      def sort_limit
+        if rich_sort?
+          @query["sortBy"]["limit"] || UNLIMITED
+        else
+          UNLIMITED
+        end
+      end
+
+      def output_limit
+        @output["limit"] || 0
+      end
+
+      def calculate_output_offset!
+        @output["offset"] = 0 if have_records?
+      end
+
+      def final_offset
+        @original_sort_offset + @original_output_offset
+      end
+
+      def final_limit
+        if @original_sort_limit == UNLIMITED && @original_output_limit == UNLIMITED
+          UNLIMITED
+        else
+          if @original_sort_limit == UNLIMITED
+            @original_output_limit
+          elsif @original_output_limit == UNLIMITED
+            @original_sort_limit
+          else
+            [@original_sort_limit, @original_output_limit].min
+          end
+        end
+      end
+
+      def have_records?
+        @output["elements"].include?("records")
+      end
+
+      def rich_sort?
+        @query["sortBy"].is_a?(Hash)
+      end
+
+      def unifiable?
+        @output["unifiable"]
+      end
+
+      def build_count_mapper_and_reducer!
+        return unless @output["elements"].include?("count")
+
+        @reducers["count"] = {
+          "type" => "sum",
+        }
+        if unifiable?
+          @query["sortBy"]["limit"] = -1 if @query["sortBy"].is_a?(Hash)
+          @output["limit"] = -1
+          mapper = {
+            "type" => "count",
+            "target" => "records",
+          }
+          unless @output["elements"].include?("records")
+            @records_limit = -1
+            @output["elements"] << "records"
+            @output["attributes"] ||= ["_key"]
+            @output_records = false
+          end
+          @mappers["count"] = mapper
+        end
+      end
+
+      def build_records_mapper_and_reducer!
+        # Skip reducing phase for a result with no record output.
+        return if !@output["elements"].include?("records") || @records_limit.zero?
+
+        # Append sort key attributes to the list of output attributes
+        # temporarily, for the reducing phase. After all extra columns
+        # are removed on the gathering phase.
+        final_attributes = output_attribute_names
+        update_output_attributes!
+
+        @reducers["records"] = build_records_reducer
+
+        mapper = {
+          "type" => "sort",
+          "offset" => @records_offset,
+          "limit" => @records_limit,
+          "format" => @records_format,
+          "attributes" => final_attributes,
+        }
+        mapper["no_output"] = true unless @output_records
+        @mappers["records"] = mapper
+      end
+
+      def output_attribute_names
+        attributes = @output["attributes"] || []
+        if attributes.is_a?(Hash)
+          attributes.keys
+        else
+          attributes.collect do |attribute|
+            if attribute.is_a?(Hash)
+              attribute["label"] || attribute["source"]
+            else
+              attribute
+            end
+          end
+        end
+      end
+
+      def update_output_attributes!
+        @output["attributes"] = array_style_attributes
+        @output["attributes"] += sort_attribute_names
+        if unifiable? && !@output["attributes"].include?("_key")
+          @output["attributes"] << "_key"
+        end
+      end
+
+      def array_style_attributes
+        attributes = @output["attributes"] || []
+        if attributes.is_a?(Hash)
+          attributes.keys.collect do |key|
+            attribute = attributes[key]
+            case attribute
+            when String
+              {
+                "label" => key,
+                "source" => attribute,
+              }
+            when Hash
+              attribute["label"] = key
+              attribute
+            end
+          end
+        else
+          attributes
+        end
+      end
+
+      def source_column_names
+        attributes = @output["attributes"] || []
+        if attributes.is_a?(Hash)
+          attributes_hash = attributes
+          attributes = []
+          attributes_hash.each do |key, attribute|
+            attributes << attribute["source"] || key
+          end
+          attributes
+        else
+          attributes.collect do |attribute|
+            if attribute.is_a?(Hash)
+              attribute["source"] || attribute["label"]
+            else
+              attribute
+            end
+          end
+        end
+      end
+
+      def sort_attribute_names
+        sort_attributes = @sort_keys.collect do |key|
+          key = key[1..-1] if key[0] == "-"
+          key
+        end
+        attributes = source_column_names
+        sort_attributes.reject! do |attribute|
+          attributes.include?(attribute)
+        end
+        sort_attributes
+      end
+
+      ASCENDING_OPERATOR = "<".freeze
+      DESCENDING_OPERATOR = ">".freeze
+
+      def build_records_reducer
+        attributes = @output["attributes"]
+        key_column_index = attributes.index("_key")
+
+        operators = @sort_keys.collect do |sort_key|
+          operator = ASCENDING_OPERATOR
+          if sort_key[0] == "-"
+            operator = DESCENDING_OPERATOR
+            sort_key = sort_key[1..-1]
+          end
+          {
+            "operator" => operator,
+            "column" => attributes.index(sort_key),
+          }
+        end
+
+        reducer = {
+          "type" => "sort",
+          "operators" => operators,
+        }
+        if unifiable? && !key_column_index.nil?
+          reducer["key_column"] = key_column_index
+        end
+
+        # On the reducing phase, we apply only "limit". We cannot apply
+        # "offset" on this phase because the collector merges a pair of
+        # results step by step even if there are three or more results.
+        # Instead, we apply "offset" on the gathering phase.
+        reducer["limit"] = @output["limit"]
+
+        reducer
+      end
+    end
+  end
+end