rroonga 7.0.2-x64-mingw32 → 7.1.1-x64-mingw32

data/vendor/local/lib/groonga/plugins/sharding/logical_enumerator.rb

@@ -162,7 +162,11 @@ module Groonga
         end
 
         def least_over_time
-          Time.local(@year, @month, @day + 1)
+          next_day = Time.local(@year, @month, @day) + (60 * 60 * 24)
+          while next_day.day == @day # For leap second
+            next_day += 1
+          end
+          next_day
         end
 
         def min_time
@@ -186,9 +190,13 @@ module Groonga
 
         def least_over_time
           if @max_day.nil?
-            Time.local(@year, @month + 1, 1)
+            if @month == 12
+              Time.local(@year + 1, 1, 1)
+            else
+              Time.local(@year, @month + 1, 1)
+            end
           else
-            Time.local(@year, @month, @max_day + 1)
+            Time.local(@year, @month, @max_day)
           end
         end
 

data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb

@@ -15,6 +15,7 @@ module Groonga
                  "limit",
                  "output_columns",
                  "use_range_index",
+                 "post_filter",
                ])
 
       def run_body(input)
@@ -71,6 +72,9 @@ module Groonga
         key << "#{input[:limit]}\0"
         key << "#{input[:output_columns]}\0"
         key << "#{input[:use_range_index]}\0"
+        key << "#{input[:post_filter]}\0"
+        dynamic_columns = DynamicColumns.parse(input)
+        key << dynamic_columns.cache_key
         key
       end
 
@@ -81,11 +85,14 @@ module Groonga
         attr_reader :filter
         attr_reader :offset
         attr_reader :limit
+        attr_reader :dynamic_columns
         attr_accessor :current_offset
         attr_accessor :current_limit
         attr_reader :result_sets
         attr_reader :unsorted_result_sets
+        attr_reader :temporary_tables
         attr_reader :threshold
+        attr_reader :post_filter
         def initialize(input)
           @input = input
           @use_range_index = parse_use_range_index(@input[:use_range_index])
@@ -94,6 +101,7 @@ module Groonga
           @filter = @input[:filter]
           @offset = (@input[:offset] || 0).to_i
           @limit = (@input[:limit] || 10).to_i
+          @dynamic_columns = DynamicColumns.parse(@input)
 
           @current_offset = @offset
           @current_limit = @limit
@@ -101,7 +109,11 @@ module Groonga
           @result_sets = []
           @unsorted_result_sets = []
 
+          @temporary_tables = []
+
           @threshold = compute_threshold
+
+          @post_filter = @input[:post_filter]
         end
 
         def close
@@ -111,6 +123,10 @@ module Groonga
           @result_sets.each do |result_set|
             result_set.close if result_set.temporary?
           end
+
+          @temporary_tables.each do |table|
+            table.close
+          end
         end
 
         private
@@ -179,6 +195,12 @@ module Groonga
           if @context.result_sets.empty?
             result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC,
                                           :key_type => first_shard.table)
+            @context.dynamic_columns.each_initial do |dynamic_column|
+              dynamic_column.apply(result_set)
+            end
+            @context.dynamic_columns.each_filtered do |dynamic_column|
+              dynamic_column.apply(result_set)
+            end
             @context.result_sets << result_set
           end
         end
@@ -190,7 +212,10 @@ module Groonga
           @shard = shard
           @shard_range = shard_range
 
+          @target_table = @shard.table
+
           @filter = @context.filter
+          @post_filter = @context.post_filter
           @result_sets = @context.result_sets
           @unsorted_result_sets = @context.unsorted_result_sets
 
@@ -201,7 +226,7 @@ module Groonga
 
         def execute
           return if @cover_type == :none
-          return if @shard.table.empty?
+          return if @target_table.empty?
 
           shard_key = @shard.key
           if shard_key.nil?
@@ -224,6 +249,24 @@ module Groonga
             range_index = nil
           end
 
+          @context.dynamic_columns.each_initial do |dynamic_column|
+            if @target_table == @shard.table
+              if @cover_type == :all
+                @target_table = @target_table.select_all
+              else
+                expression_builder.filter = nil
+                @target_table = create_expression(@target_table) do |expression|
+                  expression_builder.build(expression, @shard_range)
+                  @target_table.select(expression)
+                end
+                @cover_type = :all
+                expression_builder.filter = @filter
+              end
+              @context.temporary_tables << @target_table
+            end
+            dynamic_column.apply(@target_table)
+          end
+
           execute_filter(range_index, expression_builder)
         end
 
@@ -273,11 +316,20 @@ module Groonga
                                           __LINE__, __method__)
           end
 
+          unless @context.dynamic_columns.empty?
+            reason = "dynamic columns are used"
+            return decide_use_range_index(false, reason, __LINE__, __method__)
+          end
+
+          unless @context.post_filter.nil?
+            reason = "post_filter is used"
+            return decide_use_range_index(false, reason, __LINE__, __method__)
+          end
+
           current_limit = @context.current_limit
           if current_limit < 0
             reason = "limit is negative: <#{current_limit}>"
-            return decide_use_range_index(false, reason,
-                                          __LINE__, __method__)
+            return decide_use_range_index(false, reason, __LINE__, __method__)
           end
 
           required_n_records = @context.current_offset + current_limit
@@ -449,9 +501,9 @@ module Groonga
         end
 
         def filter_shard_all(range_index, expression_builder)
-          table = @shard.table
+          table = @target_table
           if @filter.nil?
-            if table.size <= @context.current_offset
+            if @post_filter.nil? and table.size <= @context.current_offset
               @context.current_offset -= table.size
               return
             end
@@ -593,7 +645,7 @@ module Groonga
         end
 
         def filter_table
-          table = @shard.table
+          table = @target_table
           create_expression(table) do |expression|
             yield(expression)
             result_set = table.select(expression)
@@ -601,12 +653,34 @@ module Groonga
           end
         end
 
+        def apply_post_filter(table)
+          create_expression(table) do |expression|
+            expression.parse(@post_filter)
+            table.select(expression)
+          end
+        end
+
         def sort_result_set(result_set)
+          @context.temporary_tables.delete(result_set)
+
           if result_set.empty?
             result_set.close if result_set.temporary?
             return
           end
 
+          @context.dynamic_columns.each_filtered do |dynamic_column|
+            if result_set == @shard.table
+              result_set = result_set.select_all
+            end
+            dynamic_column.apply(result_set)
+          end
+
+          unless @post_filter.nil?
+            filtered_table = result_set
+            result_set = apply_post_filter(filtered_table)
+            @context.temporary_tables << filtered_table
+          end
+
           if result_set.size <= @context.current_offset
             @context.current_offset -= result_set.size
             result_set.close if result_set.temporary?

data/vendor/local/lib/groonga/plugins/sharding/logical_select.rb

@@ -97,32 +97,13 @@ module Groonga
           key << "#{drilldown.calc_types}\0"
           key << "#{drilldown.calc_target_name}\0"
           key << "#{drilldown.filter}\0"
-          cache_key_dynamic_columns(key, drilldown.dynamic_columns)
+          key << drilldown.dynamic_columns.cache_key
         end
         dynamic_columns = DynamicColumns.parse(input)
-        cache_key_dynamic_columns(key, dynamic_columns)
+        key << dynamic_columns.cache_key
         key
       end
 
-      def cache_key_dynamic_columns(key, dynamic_columns)
-        [
-          :initial,
-          :filtered,
-          :output
-        ].each do |stage|
-          target_dynamic_columns = dynamic_columns.__send__("each_#{stage}").to_a
-          target_dynamic_columns.sort_by(&:label).each do |dynamic_column|
-            key << "#{dynamic_column.label}\0"
-            key << "#{dynamic_column.stage}\0"
-            key << "#{dynamic_column.type}\0"
-            key << "#{dynamic_column.flags}\0"
-            key << "#{dynamic_column.value}\0"
-            key << "#{dynamic_column.window_sort_keys.join(',')}\0"
-            key << "#{dynamic_column.window_group_keys.join(',')}\0"
-          end
-        end
-      end
-
       def write_records(writer, context)
         result_sets = context.result_sets
 
@@ -144,6 +125,7 @@ module Groonga
             writer.write_table_columns(first_result_set, output_columns)
           end
 
+          n_outputs = 0
           current_offset = context.offset
           current_offset += n_hits if current_offset < 0
           current_limit = context.limit
@@ -152,18 +134,24 @@ module Groonga
             :offset => current_offset,
             :limit => current_limit,
           }
+          if context.sort_keys.any? {|sort_key| sort_key.start_with?("-")}
+            result_sets = result_sets.reverse
+          end
           result_sets.each do |result_set|
             if result_set.size > current_offset
               writer.write_table_records(result_set, output_columns, options)
+              n_written = [result_set.size - current_offset, current_limit].min
+              current_limit -= n_written
+              n_outputs += n_written
             end
             if current_offset > 0
               current_offset = [current_offset - result_set.size, 0].max
             end
-            current_limit -= result_set.size
             break if current_limit <= 0
             options[:offset] = current_offset
             options[:limit] = current_limit
           end
+          query_logger.log(:size, ":", "output(#{n_outputs})")
         end
       end
 
@@ -178,8 +166,13 @@ module Groonga
         }
 
         drilldowns.each do |drilldown|
+          limit = options[:limit]
+          limit += drilldown.size + 1 if limit < 0
+          offset = options[:offset]
+          offset += drilldown.size if offset < 0
+          n_written = [drilldown.size - offset, limit].min
           n_elements = 2 # for N hits and columns
-          n_elements += drilldown.size
+          n_elements += n_written
           writer.array("RESULTSET", n_elements) do
             writer.array("NHITS", 1) do
               writer.write(drilldown.size)
@@ -188,6 +181,7 @@ module Groonga
             writer.write_table_records(drilldown, output_columns,
                                        options)
           end
+          query_logger.log(:size, ":", "output.drilldown(#{n_written})")
         end
       end
 
@@ -208,6 +202,11 @@ module Groonga
               :limit  => drilldown.limit,
             }
 
+            limit = options[:limit]
+            limit += result_set.size + 1 if limit < 0
+            offset = options[:offset]
+            offset += result_set.size if offset < 0
+            n_written = [result_set.size - offset, limit].min
             writer.array("RESULTSET", n_elements) do
               writer.array("NHITS", 1) do
                 writer.write(result_set.size)
@@ -223,37 +222,11 @@ module Groonga
                 writer.write_table_records(result_set, output_columns, options)
               end
             end
+            query_logger.log(:size, ":", "output.drilldowns(#{n_written})")
           end
         end
       end
 
-      class LabeledArgumentParser
-        def initialize(parameters)
-          @parameters = parameters
-        end
-
-        def parse(prefix_pattern)
-          pattern = /\A#{prefix_pattern}\[(.+?)\]\.(.+)\z/
-          labeled_arguments = {}
-          @parameters.each do |key, value|
-            match_data = pattern.match(key)
-            next if match_data.nil?
-            labeled_argument = (labeled_arguments[match_data[1]] ||= {})
-            labeled_argument[match_data[2]] = value
-          end
-          labeled_arguments
-        end
-      end
-
-      module KeysParsable
-        private
-        def parse_keys(raw_keys)
-          return [] if raw_keys.nil?
-
-          raw_keys.strip.split(/ *, */)
-        end
-      end
-
       module Calculatable
         def calc_target(table)
           return nil if @calc_target_name.nil?
@@ -340,8 +313,6 @@ module Groonga
           @plain_drilldown.close
           @labeled_drilldowns.close
 
-          @dynamic_columns.close
-
           @temporary_tables.each do |table|
             table.close
           end
@@ -352,140 +323,6 @@ module Groonga
         end
       end
 
-      class DynamicColumns
-        class << self
-          def parse(input)
-            parser = LabeledArgumentParser.new(input)
-            columns = parser.parse(/columns?/)
-
-            initial_contexts = []
-            filtered_contexts = []
-            output_contexts = []
-            columns.each do |label, parameters|
-              contexts = nil
-              case parameters["stage"]
-              when "initial"
-                contexts = initial_contexts
-              when "filtered"
-                contexts = filtered_contexts
-              when "output"
-                contexts = output_contexts
-              else
-                next
-              end
-              contexts << DynamicColumnExecuteContext.new(label, parameters)
-            end
-
-            new(initial_contexts,
-                filtered_contexts,
-                output_contexts)
-          end
-        end
-
-        def initialize(initial_contexts,
-                       filtered_contexts,
-                       output_contexts)
-          @initial_contexts = initial_contexts
-          @filtered_contexts = filtered_contexts
-          @output_contexts = output_contexts
-        end
-
-        def each_initial(&block)
-          @initial_contexts.each(&block)
-        end
-
-        def each_filtered(&block)
-          @filtered_contexts.each(&block)
-        end
-
-        def each_output(&block)
-          @output_contexts.each(&block)
-        end
-
-        def close
-          @initial_contexts.each do |context|
-            context.close
-          end
-          @filtered_contexts.each do |context|
-            context.close
-          end
-          @output_contexts.each do |context|
-            context.close
-          end
-        end
-      end
-
-      class DynamicColumnExecuteContext
-        include KeysParsable
-
-        attr_reader :label
-        attr_reader :stage
-        attr_reader :type
-        attr_reader :flags
-        attr_reader :value
-        attr_reader :window_sort_keys
-        attr_reader :window_group_keys
-        def initialize(label, parameters)
-          @label = label
-          @stage = parameters["stage"]
-          @type = parse_type(parameters["type"])
-          @flags = parse_flags(parameters["flags"] || "COLUMN_SCALAR")
-          @value = parameters["value"]
-          @window_sort_keys = parse_keys(parameters["window.sort_keys"])
-          @window_group_keys = parse_keys(parameters["window.group_keys"])
-        end
-
-        def close
-        end
-
-        def apply(table, condition=nil)
-          column = table.create_column(@label, @flags, @type)
-          return if table.empty?
-
-          expression = Expression.create(table)
-          begin
-            expression.parse(@value)
-            if @window_sort_keys.empty? and @window_group_keys.empty?
-              expression.condition = condition if condition
-              table.apply_expression(column, expression)
-            else
-              table.apply_window_function(column, expression,
-                                          :sort_keys => @window_sort_keys,
-                                          :group_keys => @window_group_keys)
-            end
-          ensure
-            expression.close
-          end
-        end
-
-        private
-        def parse_type(type_raw)
-          return nil if type_raw.nil?
-
-          type = Context.instance[type_raw]
-          if type.nil?
-            message = "#{error_message_tag} unknown type: <#{type_raw}>"
-            raise InvalidArgument, message
-          end
-
-          case type
-          when Type, Table
-            type
-          else
-            message = "#{error_message_tag} invalid type: #{type.grn_inspect}"
-            raise InvalidArgument, message
-          end
-        end
-
-        def parse_flags(flags_raw)
-          Column.parse_flags(error_message_tag, flags_raw)
-        end
-
-        def error_message_tag
-          "[logical_select][columns][#{@stage}][#{@label}]"
-        end
-      end
-
       class PlainDrilldownExecuteContext
         include KeysParsable
         include Calculatable
@@ -555,16 +392,13 @@ module Groonga
 
         class << self
           def parse(input)
-            parser = LabeledArgumentParser.new(input)
-            drilldowns = parser.parse(/drilldowns?/)
-
             contexts = {}
-            drilldowns.each do |label, parameters|
-              next if parameters["keys"].nil?
-              context = LabeledDrilldownExecuteContext.new(label, parameters)
+            labeled_arguments = LabeledArguments.new(input, /drilldowns?/)
+            labeled_arguments.each do |label, arguments|
+              next if arguments["keys"].nil?
+              context = LabeledDrilldownExecuteContext.new(label, arguments)
               contexts[label] = context
             end
-
             new(contexts)
           end
         end
@@ -663,8 +497,6 @@ module Groonga
           @result_set.close if @result_set
           @unsorted_result_set.close if @unsorted_result_set
 
-          @dynamic_columns.close
-
           @temporary_tables.each do |table|
             table.close
           end
@@ -872,7 +704,16 @@ module Groonga
 
           @context.dynamic_columns.each_initial do |dynamic_column|
             if @target_table == @shard.table
-              @target_table = create_all_match_table(@target_table)
+              if @cover_type == :all
+                @target_table = @target_table.select_all
+              else
+                expression_builder = RangeExpressionBuilder.new(shard_key,
+                                                                @target_range)
+                expression = create_expression(@target_table)
+                expression_builder.build(expression, @shard_range)
+                @target_table = @target_table.select(expression)
+                @cover_type = :all
+              end
               @context.temporary_tables << @target_table
             end
             dynamic_column.apply(@target_table)
@@ -938,6 +779,9 @@ module Groonga
         end
 
         def add_result_set(result_set, condition)
+          query_logger.log(:size, ":",
+                           "select(#{result_set.size})[#{@shard.table_name}]")
+
           if result_set.empty?
             result_set.close
             return
@@ -945,8 +789,7 @@ module Groonga
 
           @context.dynamic_columns.each_filtered do |dynamic_column|
             if result_set == @shard.table
-              @context.temporary_tables << result_set
-              result_set = create_all_match_table(result_set)
+              result_set = result_set.select_all
             end
             dynamic_column.apply(result_set, condition)
           end
@@ -960,14 +803,8 @@ module Groonga
           end
         end
 
-        def create_all_match_table(table)
-          expression = Expression.create(table)
-          begin
-            expression.append_constant(true, Operator::PUSH, 1)
-            table.select(expression)
-          ensure
-            expression.close
-          end
+        def query_logger
+          Context.instance.query_logger
         end
       end
     end