rroonga 9.0.3-x64-mingw32 → 9.0.7-x64-mingw32

data/vendor/local/lib/groonga/plugins/functions/index_column.la

@@ -1,5 +1,5 @@
 # index_column.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/functions/math.la

@@ -1,5 +1,5 @@
 # math.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/functions/number.la

@@ -1,5 +1,5 @@
 # number.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/functions/string.la

@@ -1,5 +1,5 @@
 # string.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/functions/time.la

@@ -1,5 +1,5 @@
 # time.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/functions/vector.la

@@ -1,5 +1,5 @@
 # vector.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/query_expanders/tsv.la

@@ -1,5 +1,5 @@
 # tsv.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/sharding/dynamic_columns.rb

@@ -4,7 +4,7 @@ module Groonga
       include Enumerable
 
       class << self
-        def parse(input)
+        def parse(tag, input)
           initial_contexts = {}
           filtered_contexts = {}
           output_contexts = {}
@@ -21,7 +21,9 @@ module Groonga
             else
               next
             end
-            contexts[label] = DynamicColumnExecuteContext.new(label, arguments)
+            contexts[label] = DynamicColumnExecuteContext.new(tag,
+                                                              label,
+                                                              arguments)
           end
 
           new(DynamicColumnExecuteContexts.new(initial_contexts),
@@ -125,15 +127,26 @@ module Groonga
         end
 
         if options.fetch(:normal, true)
-          targets.each do |result_set, options|
+          targets.each do |result_set, target_options|
             normal_contexts.each do |context|
-              context.apply(result_set, options)
+              global_options = nil
+              if options and options[:query_log_prefix]
+                global_options = {
+                  query_log_prefix: options[:query_log_prefix],
+                }
+              end
+              if target_options and global_options
+                target_options = global_options.merge(target_options)
+              elsif global_options
+                target_options = global_options
+              end
+              context.apply(result_set, target_options)
             end
           end
         end
         if options.fetch(:window_function, true)
           window_function_contexts.each do |context|
-            context.apply_window_function(targets)
+            context.apply_window_function(targets, options)
           end
         end
       end
@@ -183,6 +196,7 @@ module Groonga
     end
 
     class DynamicColumnExecuteContext
+      include QueryLoggable
       include KeysParsable
 
       attr_reader :label
@@ -192,7 +206,8 @@ module Groonga
       attr_reader :value
       attr_reader :window_sort_keys
       attr_reader :window_group_keys
-      def initialize(label, arguments)
+      def initialize(tag, label, arguments)
+        @tag = tag
         @label = label
         @stage = arguments["stage"]
         @type = parse_type(arguments["type"])
@@ -210,22 +225,28 @@ module Groonga
       def apply(table, options=nil)
         return if table.find_column(@label)
         column = table.create_column(@label, @flags, @type)
-        return if table.empty?
 
-        condition = nil
-        condition = options[:condition] if options
-        expression = Expression.create(table)
-        begin
-          expression.parse(@value)
-          expression.condition = condition if condition
-          table.apply_expression(column, expression)
-        ensure
-          expression.close
+        query_log_prefix = nil
+        query_log_prefix = options[:query_log_prefix] if options
+        unless table.empty?
+          condition = nil
+          condition = options[:condition] if options
+          expression = Expression.create(table)
+          begin
+            expression.parse(@value)
+            expression.condition = condition if condition
+            table.apply_expression(column, expression)
+          ensure
+            expression.close
+          end
         end
+        query_logger.log(:size, ":",
+                         "#{query_log_prefix}columns[#{@label}](#{table.size})")
       end
 
-      def apply_window_function(targets)
+      def apply_window_function(targets, options=nil)
         executor = WindowFunctionExecutor.new
+        n_records = 0
         begin
           executor.source = @value
           executor.sort_keys = @window_sort_keys.join(", ")
@@ -240,12 +261,17 @@ module Groonga
               executor.add_context_table(table)
             else
               executor.add_table(table)
+              n_records += table.size
             end
           end
           executor.execute
         ensure
           executor.close
         end
+        query_log_prefix = nil
+        query_log_prefix = options[:query_log_prefix] if options
+        query_logger.log(:size, ":",
+                         "#{query_log_prefix}columns[#{@label}](#{n_records})")
       end
 
       private
@@ -276,7 +302,7 @@ module Groonga
       end
 
       def error_message_tag
-        "[logical_select][columns][#{@stage}][#{@label}]"
+        "#{@tag}[columns][#{@stage}][#{@label}]"
       end
     end
   end

data/vendor/local/lib/groonga/plugins/sharding/logical_count.rb

@@ -50,7 +50,7 @@ module Groonga
         key << "#{input[:max_border]}\0"
         key << "#{input[:filter]}\0"
         key << "#{input[:post_filter]}\0"
-        dynamic_columns = DynamicColumns.parse(input)
+        dynamic_columns = DynamicColumns.parse("[logical_count]", input)
         key << dynamic_columns.cache_key
         key
       end
@@ -69,11 +69,12 @@ module Groonga
       end
 
       class Counter
+        include Loggable
+
         def initialize(input, target_range)
-          @logger = Context.instance.logger
           @filter = input[:filter]
           @post_filter = input[:post_filter]
-          @dynamic_columns = DynamicColumns.parse(input)
+          @dynamic_columns = DynamicColumns.parse("[logical_count]", input)
           @target_range = target_range
           @contexts = []
           @temporary_tables = []
@@ -136,11 +137,11 @@ module Groonga
             message << "[select]"
           end
           message << " <#{table_name}>: #{reason}"
-          @logger.log(Logger::Level::DEBUG,
-                      __FILE__,
-                      line,
-                      method.to_s,
-                      message)
+          logger.log(Logger::Level::DEBUG,
+                     __FILE__,
+                     line,
+                     method.to_s,
+                     message)
         end
 
         def prepare_contexts

data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb

@@ -53,6 +53,7 @@ module Groonga
               break if limit <= 0
             end
           end
+          query_logger.log(:size, ":", "output(#{n_elements - 1})")
         ensure
           context.close
         end
@@ -75,7 +76,8 @@ module Groonga
         key << "#{input[:use_range_index]}\0"
         key << "#{input[:post_filter]}\0"
         key << "#{input[:sort_keys]}\0"
-        dynamic_columns = DynamicColumns.parse(input)
+        dynamic_columns = DynamicColumns.parse("[logical_range_filter]",
+                                               input)
         key << dynamic_columns.cache_key
         key
       end
@@ -97,6 +99,7 @@ module Groonga
         attr_reader :result_sets
         attr_reader :temporary_tables
         attr_reader :threshold
+        attr_reader :large_shard_threshold
         attr_reader :time_classify_types
         def initialize(input)
           @input = input
@@ -109,7 +112,8 @@ module Groonga
           @post_filter = @input[:post_filter]
           @sort_keys = parse_keys(@input[:sort_keys])
 
-          @dynamic_columns = DynamicColumns.parse(@input)
+          @dynamic_columns = DynamicColumns.parse("[logical_range_filter]",
+                                                  @input)
 
           @current_offset = @offset
           @current_limit = @limit
@@ -118,6 +122,7 @@ module Groonga
           @temporary_tables = []
 
           @threshold = compute_threshold
+          @large_shard_threshold = compute_large_shard_threshold
 
           @time_classify_types = detect_time_classify_types
         end
@@ -169,6 +174,12 @@ module Groonga
           (threshold_env || default_threshold).to_f
         end
 
+        def compute_large_shard_threshold
+          threshold_env = ENV["GRN_LOGICAL_RANGE_FILTER_LARGE_SHARD_THRESHOLD"]
+          default_threshold = "10_000"
+          (threshold_env || default_threshold).to_i(10)
+        end
+
         def detect_time_classify_types
           window_group_keys = []
           @dynamic_columns.each_filtered do |dynamic_column|
@@ -261,6 +272,7 @@ module Groonga
 
       class Window
         include Comparable
+        include Loggable
 
         attr_reader :unit
         attr_reader :step
@@ -326,11 +338,11 @@ module Groonga
                 message = "[logical_range_filter][window] "
                 message << "<#{@shard.table_name}>: "
                 message << inspect_range(current_min, current_max)
-                Context.instance.logger.log(Logger::Level::DEBUG,
-                                            __FILE__,
-                                            __LINE__,
-                                            __method__.to_s,
-                                            message)
+                logger.log(Logger::Level::DEBUG,
+                           __FILE__,
+                           __LINE__,
+                           __method__.to_s,
+                           message)
                 yield(windowed_table)
               end
               current_min = next_min
@@ -494,6 +506,9 @@ module Groonga
       end
 
       class ShardExecutor
+        include Loggable
+        include QueryLoggable
+
         attr_reader :shard
         attr_writer :previous_executor
         attr_writer :next_executor
@@ -634,12 +649,11 @@ module Groonga
           end
           message << "<#{@shard.table_name}>: "
           message << reason
-          Context.instance.logger.log(Logger::Level::DEBUG,
-                                      __FILE__,
-                                      line,
-                                      method.to_s,
-                                      message)
-
+          logger.log(Logger::Level::DEBUG,
+                     __FILE__,
+                     line,
+                     method.to_s,
+                     message)
           use
         end
 
@@ -693,7 +707,8 @@ module Groonga
 
           required_n_records = @context.current_offset + current_limit
           max_n_records = @shard.table.size
-          if max_n_records <= required_n_records
+          is_large_shard = (max_n_records > @context.large_shard_threshold)
+          if is_large_shard and max_n_records <= required_n_records
             reason = "the number of required records (#{required_n_records}) "
             reason << ">= "
             reason << "the number of records in shard (#{max_n_records})"
@@ -758,7 +773,7 @@ module Groonga
             end
           end
 
-          if estimated_n_records <= required_n_records
+          if is_large_shard and estimated_n_records <= required_n_records
             reason = "the number of required records (#{required_n_records}) "
             reason << ">= "
             reason << "the number of estimated records (#{estimated_n_records})"
@@ -966,10 +981,18 @@ module Groonga
                     n_matched_records = index_cursor.select(result_set, options)
                   end
                 end
+                # TODO: Add range information
+                query_logger.log(:size, ":",
+                                 "filter(#{n_matched_records})" +
+                                 "[#{@shard.table_name}]: #{@filter}")
               else
                 IndexCursor.open(table_cursor, range_index) do |index_cursor|
                   n_matched_records = index_cursor.select(result_set, options)
                 end
+                # TODO: Add range information
+                query_logger.log(:size, ":",
+                                 "filter(#{n_matched_records})" +
+                                 "[#{@shard.table_name}]")
               end
               if n_matched_records == -1
                 result_set.close
@@ -1108,6 +1131,16 @@ module Groonga
           if @context.current_limit > 0
             @context.current_limit -= sorted_result_set.size
           end
+          unparsed_sort_keys = sort_keys.collect do |sort_key|
+            key = sort_key[:key]
+            key = "-#{key}" if sort_key[:order] == :descending
+            key
+          end
+          query_logger.log(:size,
+                           ":",
+                           "sort(#{sorted_result_set.size})" +
+                           "[#{@shard.table_name}]: " +
+                           unparsed_sort_keys.join(","))
         end
 
         def detect_window

data/vendor/local/lib/groonga/plugins/sharding/logical_select.rb

@@ -109,7 +109,7 @@ module Groonga
           key << "#{drilldown.filter}\0"
           key << drilldown.dynamic_columns.cache_key
         end
-        dynamic_columns = DynamicColumns.parse(input)
+        dynamic_columns = DynamicColumns.parse("[logical_select]", input)
         key << dynamic_columns.cache_key
         key
       end
@@ -200,7 +200,18 @@ module Groonga
           end
           result_sets.each do |result_set|
             if result_set.size > current_offset
-              writer.write_table_records(result_set, output_columns, options)
+              if context.sort_keys.empty?
+                writer.write_table_records(result_set, output_columns, options)
+              else
+                sorted_result_set = result_set.sort(context.sort_keys, options)
+                context.temporary_tables << sorted_result_set
+                message = "sort(#{sorted_result_set.size}): "
+                message << context.sort_keys.join(",")
+                query_logger.log(:size, ":", message)
+                writer.write_table_records(sorted_result_set,
+                                           output_columns,
+                                           offset: 0, limit: -1)
+              end
               n_written = [result_set.size - current_offset, current_limit].min
               current_limit -= n_written
               n_outputs += n_written
@@ -220,23 +231,33 @@ module Groonga
         plain_drilldown = execute_context.plain_drilldown
 
         drilldowns = plain_drilldown.result_sets
+        sort_keys = plain_drilldown.sort_keys
         output_columns = plain_drilldown.output_columns
-        options = {
-          :offset => plain_drilldown.offset,
-          :limit  => plain_drilldown.limit,
-        }
 
         drilldowns.each do |drilldown|
+          options = {
+            :offset => plain_drilldown.offset,
+            :limit  => plain_drilldown.limit,
+          }
+          n_records = drilldown.size
           limit = options[:limit]
-          limit += drilldown.size + 1 if limit < 0
+          limit += n_records + 1 if limit < 0
           offset = options[:offset]
-          offset += drilldown.size if offset < 0
-          n_written = [drilldown.size - offset, limit].min
+          offset += n_records if offset < 0
+          n_written = [n_records - offset, limit].min
           n_elements = 2 # for N hits and columns
           n_elements += n_written
+          unless sort_keys.empty?
+            drilldown = drilldown.sort(sort_keys, options)
+            plain_drilldown.temporary_tables << drilldown
+            message = "drilldown.sort(#{drilldown.size}): "
+            message << sort_keys.join(",")
+            query_logger.log(:size, ":", message)
+            options = {offset: 0, limit: -1}
+          end
           writer.array("RESULTSET", n_elements) do
             writer.array("NHITS", 1) do
-              writer.write(drilldown.size)
+              writer.write(n_records)
             end
             writer.write_table_columns(drilldown, output_columns)
             writer.write_table_records(drilldown, output_columns,
@@ -252,11 +273,14 @@ module Groonga
 
         writer.map("DRILLDOWNS", labeled_drilldowns.n_result_sets) do
           labeled_drilldowns.each do |drilldown|
+            query_log_tag = "drilldowns[#{drilldown.label}]"
+
             writer.write(drilldown.label)
 
             result_set = drilldown.result_set
+            n_records = result_set.size
             n_elements = 2 # for N hits and columns
-            n_elements += result_set.size
+            n_elements += n_records
             output_columns = drilldown.output_columns
             options = {
               :offset => drilldown.offset,
@@ -267,10 +291,20 @@ module Groonga
             limit += result_set.size + 1 if limit < 0
             offset = options[:offset]
             offset += result_set.size if offset < 0
-            n_written = [result_set.size - offset, limit].min
+            if drilldown.sort_keys.empty?
+              n_written = [n_records - offset, limit].min
+            else
+              result_set = result_set.sort(drilldown.sort_keys, options)
+              drilldown.temporary_tables << result_set
+              message = "#{query_log_tag}.sort(#{result_set.size}): "
+              message << drilldown.sort_keys.join(",")
+              query_logger.log(:size, ":", message)
+              options = {offset: 0, limit: -1}
+              n_written = result_set.size
+            end
             writer.array("RESULTSET", n_elements) do
               writer.array("NHITS", 1) do
-                writer.write(result_set.size)
+                writer.write(n_records)
               end
               writer.write_table_columns(result_set, output_columns)
               if is_command_version1 and drilldown.need_command_version2?
@@ -283,7 +317,9 @@ module Groonga
                 writer.write_table_records(result_set, output_columns, options)
               end
             end
-            query_logger.log(:size, ":", "output.drilldowns(#{n_written})")
+            query_logger.log(:size,
+                             ":",
+                             "output.#{query_log_tag}(#{n_written})")
           end
         end
       end
@@ -359,7 +395,7 @@ module Groonga
           @load_columns = @input[:load_columns]
           @load_values = @input[:load_values]
 
-          @dynamic_columns = DynamicColumns.parse(@input)
+          @dynamic_columns = DynamicColumns.parse("[logical_select]", @input)
 
           @result_sets = []
           @shard_targets = []
@@ -399,7 +435,6 @@ module Groonga
         attr_reader :calc_types
         attr_reader :filter
         attr_reader :result_sets
-        attr_reader :unsorted_result_sets
         attr_reader :temporary_tables
         attr_reader :expressions
         def initialize(input)
@@ -416,7 +451,6 @@ module Groonga
           @filter = @input[:drilldown_filter]
 
           @result_sets = []
-          @unsorted_result_sets = []
 
           @temporary_tables = []
 
@@ -424,13 +458,6 @@ module Groonga
         end
 
         def close
-          @result_sets.each do |result_set|
-            result_set.close
-          end
-          @unsorted_result_sets.each do |result_set|
-            result_set.close
-          end
-
           @temporary_tables.each do |table|
             table.close
           end
@@ -529,7 +556,6 @@ module Groonga
         attr_reader :table
         attr_reader :dynamic_columns
         attr_accessor :result_set
-        attr_accessor :unsorted_result_set
         attr_reader :temporary_tables
         attr_reader :expressions
         def initialize(label, parameters)
@@ -546,10 +572,10 @@ module Groonga
           @filter = parameters["filter"]
           @table = parameters["table"]
 
-          @dynamic_columns = DynamicColumns.parse(parameters)
+          tag = "[logical_select][drilldowns][#{@label}]"
+          @dynamic_columns = DynamicColumns.parse(tag, parameters)
 
           @result_set = nil
-          @unsorted_result_set = nil
 
           @temporary_tables = []
 
@@ -557,9 +583,6 @@ module Groonga
         end
 
         def close
-          @result_set.close if @result_set
-          @unsorted_result_set.close if @unsorted_result_set
-
           @temporary_tables.each do |table|
             table.close
           end
@@ -593,6 +616,8 @@ module Groonga
       end
 
       class Executor
+        include QueryLoggable
+
         def initialize(context)
           @context = context
         end
@@ -656,6 +681,7 @@ module Groonga
 
         def execute_plain_drilldown
           drilldown = @context.plain_drilldown
+          query_log_prefix = "drilldown"
           group_result = TableGroupResult.new
           begin
             group_result.key_begin = 0
@@ -670,14 +696,15 @@ module Groonga
                 end
               end
               result_set = group_result.table
-              result_set = apply_drilldown_filter(drilldown, result_set)
-              if drilldown.sort_keys.empty?
-                drilldown.result_sets << result_set
-              else
-                drilldown.result_sets << result_set.sort(drilldown.sort_keys)
-                drilldown.unsorted_result_sets << result_set
-              end
+              query_logger.log(:size,
+                               ":",
+                               "#{query_log_prefix}(#{result_set.size})")
+              result_set = apply_drilldown_filter(query_log_prefix,
+                                                  drilldown,
+                                                  result_set)
+              drilldown.temporary_tables << result_set
               group_result.table = nil
+              drilldown.result_sets << result_set
             end
           ensure
             group_result.close
@@ -688,6 +715,7 @@ module Groonga
           drilldowns = @context.labeled_drilldowns
 
           drilldowns.tsort_each do |drilldown|
+            query_log_prefix = "drilldowns[#{drilldown.label}]"
             group_result = TableGroupResult.new
             keys = drilldown.keys
             begin
@@ -713,15 +741,18 @@ module Groonga
                 end
               end
               result_set = group_result.table
-              drilldown.dynamic_columns.apply_initial([[result_set]])
-              result_set = apply_drilldown_filter(drilldown, result_set)
-              if drilldown.sort_keys.empty?
-                drilldown.result_set = result_set
-              else
-                drilldown.result_set = result_set.sort(drilldown.sort_keys)
-                drilldown.unsorted_result_set = result_set
-              end
+              query_logger.log(:size,
+                               ":",
+                               "#{query_log_prefix}(#{result_set.size})")
+              options = {query_log_prefix: "#{query_log_prefix}."}
+              drilldown.dynamic_columns.apply_initial([[result_set]],
+                                                      options)
+              result_set = apply_drilldown_filter(query_log_prefix,
+                                                  drilldown,
+                                                  result_set)
+              drilldown.temporary_tables << result_set
               group_result.table = nil
+              drilldown.result_set = result_set
             ensure
               group_result.close
             end
@@ -738,7 +769,7 @@ module Groonga
           end
         end
 
-        def apply_drilldown_filter(drilldown, result_set)
+        def apply_drilldown_filter(query_log_prefix, drilldown, result_set)
           filter = drilldown.filter
           return result_set if filter.nil?
 
@@ -747,11 +778,17 @@ module Groonga
           expression.parse(filter)
           filtered_result_set = result_set.select(expression)
           drilldown.temporary_tables << result_set
+          n_records = filtered_result_set.size
+          query_logger.log(:size,
+                           ":",
+                           "#{query_log_prefix}.filter(#{n_records})")
           filtered_result_set
         end
       end
 
       class ShardExecutor
+        include QueryLoggable
+
         def initialize(context, shard, shard_range)
           @context = context
           @shard = shard
@@ -829,14 +866,7 @@ module Groonga
             result_set = apply_post_filter(result_set)
             @temporary_tables << result_set
           end
-
-          if @sort_keys.empty?
-            @result_sets << result_set
-          else
-            sorted_result_set = result_set.sort(@sort_keys)
-            @temporary_tables << sorted_result_set
-            @result_sets << sorted_result_set
-          end
+          @result_sets << result_set
         end
 
         private
@@ -904,10 +934,6 @@ module Groonga
 
           @shard_results << [self, result_set, condition]
         end
-
-        def query_logger
-          Context.instance.query_logger
-        end
       end
     end
   end