rroonga 5.0.4-x64-mingw32 → 5.0.5-x64-mingw32

data/vendor/local/lib/groonga/plugins/sharding/logical_parameters.rb

@@ -0,0 +1,44 @@
+module Groonga
+  module Sharding
+    class LogicalParametersCommand < Command
+      register("logical_parameters",
+               [
+                 "range_index",
+               ])
+
+      def run_body(input)
+        range_index = parse_range_index(input[:range_index])
+
+        parameters = [
+          :range_index,
+        ]
+        writer.map("parameters", parameters.size) do
+          parameters.each do |name|
+            writer.write(name.to_s)
+            writer.write(Parameters.__send__(name))
+          end
+        end
+
+        Parameters.range_index = range_index if range_index
+      end
+
+      private
+      def parse_range_index(value)
+        case value
+        when nil
+          nil
+        when "auto"
+          :auto
+        when "always"
+          :always
+        when "never"
+          :never
+        else
+          message = "[logical_parameters][range_index] "
+          message << "must be auto, always or never: <#{value}>"
+          raise InvalidArgument, message
+        end
+      end
+    end
+  end
+end

data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb

@@ -14,6 +14,7 @@ module Groonga
                  "offset",
                  "limit",
                  "output_columns",
+                 "use_range_index",
                ])
 
       def run_body(input)
@@ -55,7 +56,26 @@ module Groonga
         end
       end
 
+      private
+      def cache_key(input)
+        key = "logical_range_filter\0"
+        key << "#{input[:logical_table]}\0"
+        key << "#{input[:shard_key]}\0"
+        key << "#{input[:min]}\0"
+        key << "#{input[:min_border]}\0"
+        key << "#{input[:max]}\0"
+        key << "#{input[:max_border]}\0"
+        key << "#{input[:order]}\0"
+        key << "#{input[:filter]}\0"
+        key << "#{input[:offset]}\0"
+        key << "#{input[:limit]}\0"
+        key << "#{input[:output_columns]}\0"
+        key << "#{input[:use_range_index]}\0"
+        key
+      end
+
       class ExecuteContext
+        attr_reader :use_range_index
         attr_reader :enumerator
         attr_reader :order
         attr_reader :filter
@@ -68,6 +88,7 @@ module Groonga
         attr_reader :threshold
         def initialize(input)
           @input = input
+          @use_range_index = parse_use_range_index(@input[:use_range_index])
           @enumerator = LogicalEnumerator.new("logical_range_filter", @input)
           @order = parse_order(@input, :order)
           @filter = @input[:filter]
@@ -93,6 +114,17 @@ module Groonga
         end
 
         private
+        def parse_use_range_index(use_range_index)
+          case use_range_index
+          when "yes"
+            true
+          when "no"
+            false
+          else
+            nil
+          end
+        end
+
         def parse_order(input, name)
           order = input[name]
           return :ascending if order.nil?
@@ -123,23 +155,21 @@ module Groonga
         end
 
         def execute
-          first_table = nil
+          first_shard = nil
           enumerator = @context.enumerator
+          target_range = enumerator.target_range
           if @context.order == :descending
             each_method = :reverse_each
           else
             each_method = :each
           end
-          enumerator.send(each_method) do |table, shard_key, shard_range|
-            first_table ||= table
-            next if table.empty?
-
-            shard_executor = ShardExecutor.new(@context,
-                                               table, shard_key, shard_range)
+          enumerator.send(each_method) do |shard, shard_range|
+            first_shard ||= shard
+            shard_executor = ShardExecutor.new(@context, shard, shard_range)
             shard_executor.execute
             break if @context.current_limit == 0
           end
-          if first_table.nil?
+          if first_shard.nil?
             message =
               "[logical_range_filter] no shard exists: " +
               "logical_table: <#{enumerator.logical_table}>: " +
@@ -148,17 +178,16 @@ module Groonga
           end
           if @context.result_sets.empty?
             result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC,
-                                          :key_type => first_table)
+                                          :key_type => first_shard.table)
             @context.result_sets << result_set
           end
         end
       end
 
       class ShardExecutor
-        def initialize(context, table, shard_key, shard_range)
+        def initialize(context, shard, shard_range)
           @context = context
-          @table = table
-          @shard_key = shard_key
+          @shard = shard
           @shard_range = shard_range
 
           @filter = @context.filter
@@ -168,26 +197,36 @@ module Groonga
           @target_range = @context.enumerator.target_range
 
           @cover_type = @target_range.cover_type(@shard_range)
-
-          @expression_builder = RangeExpressionBuilder.new(@shard_key,
-                                                           @target_range,
-                                                           @filter)
         end
 
         def execute
           return if @cover_type == :none
+          return if @shard.table.empty?
+
+          shard_key = @shard.key
+          if shard_key.nil?
+            message = "[logical_range_filter] shard_key doesn't exist: " +
+                      "<#{@shard.key_name}>"
+            raise InvalidArgument, message
+          end
+
+          expression_builder = RangeExpressionBuilder.new(shard_key,
+                                                          @target_range,
+                                                          @filter)
 
-          index_info = @shard_key.find_index(Operator::LESS)
+          index_info = shard_key.find_index(Operator::LESS)
           if index_info
             range_index = index_info.index
-            range_index = nil unless use_range_index?(range_index)
+            unless use_range_index?(range_index, expression_builder)
+              range_index = nil
+            end
           else
             range_index = nil
           end
 
           case @cover_type
           when :all
-            filter_shard_all(range_index)
+            filter_shard_all(range_index, expression_builder)
           when :partial_min
             if range_index
               filter_by_range(range_index,
@@ -195,7 +234,7 @@ module Groonga
                               nil, nil)
             else
               filter_table do |expression|
-                @expression_builder.build_partial_min(expression)
+                expression_builder.build_partial_min(expression)
               end
             end
           when :partial_max
@@ -205,7 +244,7 @@ module Groonga
                               @target_range.max, @target_range.max_border)
             else
               filter_table do |expression|
-                @expression_builder.build_partial_max(expression)
+                expression_builder.build_partial_max(expression)
               end
             end
           when :partial_min_and_max
@@ -215,73 +254,201 @@ module Groonga
                               @target_range.max, @target_range.max_border)
             else
               filter_table do |expression|
-                @expression_builder.build_partial_min_and_max(expression)
+                expression_builder.build_partial_min_and_max(expression)
               end
             end
           end
         end
 
         private
-        def use_range_index?(range_index)
+        def decide_use_range_index(use, reason, line, method)
+          message = "[logical_range_filter]"
+          if use
+            message << "[range-index] "
+          else
+            message << "[select] "
+          end
+          message << "<#{@shard.table_name}>: "
+          message << reason
+          Context.instance.logger.log(Logger::Level::DEBUG,
+                                      __FILE__,
+                                      line,
+                                      method.to_s,
+                                      message)
+
+          use
+        end
+
+        def use_range_index?(range_index, expression_builder)
+          use_range_index_parameter_message =
+            "force by use_range_index parameter"
+          case @context.use_range_index
+          when true
+            return decide_use_range_index(true,
+                                          use_range_index_parameter_message,
+                                          __LINE__, __method__)
+          when false
+            return decide_use_range_index(false,
+                                          use_range_index_parameter_message,
+                                          __LINE__, __method__)
+          end
+
+          range_index_logical_parameter_message =
+            "force by range_index logical parameter"
+          case Parameters.range_index
+          when :always
+            return decide_use_range_index(true,
+                                          range_index_logical_parameter_message,
+                                          __LINE__, __method__)
+          when :never
+            return decide_use_range_index(false,
+                                          range_index_logical_parameter_message,
+                                          __LINE__, __method__)
+          end
+
           current_limit = @context.current_limit
           if current_limit < 0
-            return false
+            reason = "limit is negative: <#{current_limit}>"
+            return decide_use_range_index(false, reason,
+                                          __LINE__, __method__)
           end
 
           required_n_records = @context.current_offset + current_limit
-          max_n_records = @table.size
+          max_n_records = @shard.table.size
           if max_n_records <= required_n_records
-            return false
+            reason = "the number of required records (#{required_n_records}) "
+            reason << ">= "
+            reason << "the number of records in shard (#{max_n_records})"
+            return decide_use_range_index(false, reason,
+                                          __LINE__, __method__)
           end
 
           threshold = @context.threshold
           if threshold <= 0.0
-            return true
+            reason = "threshold is negative: <#{threshold}>"
+            return decide_use_range_index(true, reason,
+                                          __LINE__, __method__)
           end
           if threshold >= 1.0
-            return false
+            reason = "threshold (#{threshold}) >= 1.0"
+            return decide_use_range_index(false, reason,
+                                          __LINE__, __method__)
           end
 
+          table = @shard.table
           estimated_n_records = 0
           case @cover_type
           when :all
             if @filter
-              create_expression(@table) do |expression|
-                @expression_builder.build_all(expression)
-                estimated_n_records = expression.estimate_size(@table)
+              create_expression(table) do |expression|
+                expression_builder.build_all(expression)
+                unless range_index_available_expression?(expression,
+                                                         __LINE__, __method__)
+                  return false
+                end
+                estimated_n_records = expression.estimate_size(table)
               end
             else
               estimated_n_records = max_n_records
             end
           when :partial_min
-            create_expression(@table) do |expression|
-              @expression_builder.build_partial_min(expression)
-              estimated_n_records = expression.estimate_size(@table)
+            create_expression(table) do |expression|
+              expression_builder.build_partial_min(expression)
+              unless range_index_available_expression?(expression,
+                                                       __LINE__, __method__)
+                return false
+              end
+              estimated_n_records = expression.estimate_size(table)
             end
           when :partial_max
-            create_expression(@table) do |expression|
-              @expression_builder.build_partial_max(expression)
-              estimated_n_records = expression.estimate_size(@table)
+            create_expression(table) do |expression|
+              expression_builder.build_partial_max(expression)
+              unless range_index_available_expression?(expression,
+                                                       __LINE__, __method__)
+                return false
+              end
+              estimated_n_records = expression.estimate_size(table)
             end
           when :partial_min_and_max
-            create_expression(@table) do |expression|
-              @expression_builder.build_partial_min_and_max(expression)
-              estimated_n_records = expression.estimate_size(@table)
+            create_expression(table) do |expression|
+              expression_builder.build_partial_min_and_max(expression)
+              unless range_index_available_expression?(expression,
+                                                       __LINE__, __method__)
+                return false
+              end
+              estimated_n_records = expression.estimate_size(table)
             end
           end
 
           if estimated_n_records <= required_n_records
-            return false
+            reason = "the number of required records (#{required_n_records}) "
+            reason << ">= "
+            reason << "the number of estimated records (#{estimated_n_records})"
+            return decide_use_range_index(false, reason,
+                                          __LINE__, __method__)
           end
 
           hit_ratio = estimated_n_records / max_n_records.to_f
-          hit_ratio >= threshold
+          use_range_index_by_hit_ratio = (hit_ratio >= threshold)
+          if use_range_index_by_hit_ratio
+            relation = ">="
+          else
+            relation = "<"
+          end
+          reason = "hit ratio "
+          reason << "(#{hit_ratio}=#{estimated_n_records}/#{max_n_records})"
+          reason << "#{relation} threshold (#{threshold})"
+          decide_use_range_index(use_range_index_by_hit_ratio, reason,
+                                 __LINE__, __method__)
+        end
+
+        def range_index_available_expression?(expression, line, method_name)
+          nested_reference_vector_column_accessor =
+            find_nested_reference_vector_column_accessor(expression)
+          if nested_reference_vector_column_accessor
+            reason = "nested reference vector column accessor can't be used: "
+            reason << "<#{nested_reference_vector_column_accessor.name}>"
+            return decide_use_range_index(false, reason, line, method_name)
+          end
+
+          selector_only_procedure = find_selector_only_procedure(expression)
+          if selector_only_procedure
+            reason = "selector only procedure can't be used: "
+            reason << "<#{selector_only_procedure.name}>"
+            return decide_use_range_index(false, reason, line, method_name)
+          end
+
+          true
+        end
+
+        def find_nested_reference_vector_column_accessor(expression)
+          expression.codes.each do |code|
+            value = code.value
+            next unless value.is_a?(Accessor)
+
+            sub_accessor = value
+            while sub_accessor.have_next?
+              object = sub_accessor.object
+              return value if object.is_a?(Column) and object.vector?
+              sub_accessor = sub_accessor.next
+            end
+          end
+          nil
+        end
+
+        def find_selector_only_procedure(expression)
+          expression.codes.each do |code|
+            value = code.value
+            return value if value.is_a?(Procedure) and value.selector_only?
+          end
+          nil
         end
 
-        def filter_shard_all(range_index)
+        def filter_shard_all(range_index, expression_builder)
+          table = @shard.table
           if @filter.nil?
-            if @table.size <= @context.current_offset
-              @context.current_offset -= @table.size
+            if table.size <= @context.current_offset
+              @context.current_offset -= table.size
               return
             end
             if range_index
@@ -289,7 +456,7 @@ module Groonga
                               nil, nil,
                               nil, nil)
             else
-              sort_result_set(@table)
+              sort_result_set(table)
             end
           else
             if range_index
@@ -298,7 +465,7 @@ module Groonga
                               nil, nil)
             else
               filter_table do |expression|
-                @expression_builder.build_all(expression)
+                expression_builder.build_all(expression)
               end
             end
           end
@@ -394,9 +561,10 @@ module Groonga
         end
 
         def filter_table
-          create_expression(@table) do |expression|
+          table = @shard.table
+          create_expression(table) do |expression|
             yield(expression)
-            result_set = @table.select(expression)
+            result_set = table.select(expression)
             sort_result_set(result_set)
           end
         end