rroonga 7.1.1-x64-mingw32 → 9.0.2-x64-mingw32

data/vendor/local/lib/groonga/plugins/sharding/logical_select.rb

@@ -28,6 +28,9 @@ module Groonga
                  "match_columns",
                  "query",
                  "drilldown_filter",
+                 "load_table",
+                 "load_columns",
+                 "load_values",
                ])
 
       def run_body(input)
@@ -36,6 +39,8 @@ module Groonga
           executor = Executor.new(context)
           executor.execute
 
+          load_records(context)
+
           n_results = 1
           n_plain_drilldowns = context.plain_drilldown.n_result_sets
           n_labeled_drilldowns = context.labeled_drilldowns.n_result_sets
@@ -87,9 +92,14 @@ module Groonga
         key << "#{input[:drilldown_calc_types]}\0"
         key << "#{input[:drilldown_calc_target]}\0"
         key << "#{input[:drilldown_filter]}\0"
+        key << "#{input[:post_filter]}\0"
+        key << "#{input[:load_table]}\0"
+        key << "#{input[:load_columns]}\0"
+        key << "#{input[:load_values]}\0"
         labeled_drilldowns = LabeledDrilldowns.parse(input).sort_by(&:label)
         labeled_drilldowns.each do |drilldown|
           key << "#{drilldown.label}\0"
+          key << "#{drilldown.table}\0"
           key << "#{drilldown.keys.join(',')}\0"
           key << "#{drilldown.output_columns}\0"
           key << "#{drilldown.offset}\0"
@@ -104,6 +114,57 @@ module Groonga
         key
       end
 
+      def resolve_target_table_name(result_set)
+        target_table = result_set
+        while target_table.domain_id
+          domain = Context.instance[target_table.domain_id]
+          break unless domain.is_a?(Table)
+          target_table = domain
+        end
+        target_table.name
+      end
+
+      def load_records(context)
+        load_table = context.load_table
+        load_columns = context.load_columns
+        load_values = context.load_values
+        return if load_table.nil?
+        return if load_columns.nil?
+        return if load_values.nil?
+
+        n_loaded_records = 0
+        to = Context.instance[load_table]
+        to_columns = []
+        begin
+          load_columns.split(/\s*,\s*/).each do |name|
+            to_columns << to.find_column(name)
+          end
+          context.result_sets.each do |result_set|
+            output_columns = result_set.parse_output_columns(load_values)
+            begin
+              output_columns.apply(to_columns)
+            ensure
+              output_columns.close
+            end
+            n_sub_loaded_records = result_set.size
+            query_logger.log(:size,
+                             ":",
+                             "load(#{n_sub_loaded_records})" +
+                             "[#{resolve_target_table_name(result_set)}]: " +
+                             "[#{load_table}][#{to.size}]")
+            n_loaded_records += n_sub_loaded_records
+          end
+        ensure
+          to_columns.each do |column|
+            column.close if column.is_a?(Accessor)
+          end
+        end
+        query_logger.log(:size,
+                         ":",
+                         "load(#{n_loaded_records}): " +
+                         "[#{load_table}][#{to.size}]")
+      end
+
       def write_records(writer, context)
         result_sets = context.result_sets
 
@@ -271,9 +332,14 @@ module Groonga
         attr_reader :limit
         attr_reader :sort_keys
         attr_reader :output_columns
+        attr_reader :post_filter
+        attr_reader :load_table
+        attr_reader :load_columns
+        attr_reader :load_values
         attr_reader :dynamic_columns
         attr_reader :result_sets
-        attr_reader :unsorted_result_sets
+        attr_reader :shard_targets
+        attr_reader :shard_results
         attr_reader :plain_drilldown
         attr_reader :labeled_drilldowns
         attr_reader :temporary_tables
@@ -288,12 +354,16 @@ module Groonga
           @limit = (@input[:limit] || 10).to_i
           @sort_keys = parse_keys(@input[:sort_keys] || @input[:sortby])
           @output_columns = @input[:output_columns] || "_id, _key, *"
+          @post_filter = @input[:post_filter]
+          @load_table = @input[:load_table]
+          @load_columns = @input[:load_columns]
+          @load_values = @input[:load_values]
 
           @dynamic_columns = DynamicColumns.parse(@input)
 
           @result_sets = []
-          @unsorted_result_sets = []
-
+          @shard_targets = []
+          @shard_results = []
           @plain_drilldown = PlainDrilldownExecuteContext.new(@input)
           @labeled_drilldowns = LabeledDrilldowns.parse(@input)
 
@@ -303,13 +373,6 @@ module Groonga
         end
 
         def close
-          @result_sets.each do |result_set|
-            result_set.close if result_set.temporary?
-          end
-          @unsorted_result_sets.each do |result_set|
-            result_set.close if result_set.temporary?
-          end
-
           @plain_drilldown.close
           @labeled_drilldowns.close
 
@@ -550,7 +613,7 @@ module Groonga
           enumerator.each do |shard, shard_range|
             first_shard ||= shard
             shard_executor = ShardExecutor.new(@context, shard, shard_range)
-            shard_executor.execute
+            shard_executor.execute_pre
           end
           if first_shard.nil?
             message =
@@ -559,16 +622,35 @@ module Groonga
               "shard_key: <#{enumerator.shard_key_name}>"
             raise InvalidArgument, message
           end
-          if @context.result_sets.empty?
+
+          if @context.dynamic_columns.have_initial?
+            targets = []
+            @context.shard_targets.each do |_, target_table|
+              targets << [target_table]
+            end
+            @context.dynamic_columns.apply_initial(targets)
+          end
+          @context.shard_targets.each do |shard_executor, target_table|
+            shard_executor.execute
+          end
+
+          if @context.shard_results.empty?
             result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC,
                                           :key_type => first_shard.table)
-            @context.dynamic_columns.each_initial do |dynamic_column|
-              dynamic_column.apply(result_set)
+            @context.temporary_tables << result_set
+            targets = [[result_set]]
+            @context.dynamic_columns.apply_initial(targets)
+            @context.dynamic_columns.apply_filtered(targets)
+            @context.result_sets << result_set
+          else
+            targets = []
+            @context.shard_results.each do |_, result_set, condition|
+              targets << [result_set, {condition: condition}]
             end
-            @context.dynamic_columns.each_filtered do |dynamic_column|
-              dynamic_column.apply(result_set)
+            @context.dynamic_columns.apply_filtered(targets)
+            @context.shard_results.each do |shard_executor, result_set, _|
+              shard_executor.execute_post(result_set)
             end
-            @context.result_sets << result_set
           end
         end
 
@@ -631,9 +713,7 @@ module Groonga
                 end
               end
               result_set = group_result.table
-              drilldown.dynamic_columns.each_initial do |dynamic_column|
-                dynamic_column.apply(result_set)
-              end
+              drilldown.dynamic_columns.apply_initial([[result_set]])
               result_set = apply_drilldown_filter(drilldown, result_set)
               if drilldown.sort_keys.empty?
                 drilldown.result_set = result_set
@@ -682,16 +762,19 @@ module Groonga
           @match_columns = @context.match_columns
           @query = @context.query
           @filter = @context.filter
+          @post_filter = @context.post_filter
           @sort_keys = @context.sort_keys
           @result_sets = @context.result_sets
-          @unsorted_result_sets = @context.unsorted_result_sets
+          @shard_targets = @context.shard_targets
+          @shard_results = @context.shard_results
+          @temporary_tables = @context.temporary_tables
 
           @target_range = @context.enumerator.target_range
 
           @cover_type = @target_range.cover_type(@shard_range)
         end
 
-        def execute
+        def execute_pre
           return if @cover_type == :none
           return if @target_table.empty?
 
@@ -702,7 +785,7 @@ module Groonga
             raise InvalidArgument, message
           end
 
-          @context.dynamic_columns.each_initial do |dynamic_column|
+          if @context.dynamic_columns.have_initial?
             if @target_table == @shard.table
               if @cover_type == :all
                 @target_table = @target_table.select_all
@@ -714,12 +797,14 @@ module Groonga
                 @target_table = @target_table.select(expression)
                 @cover_type = :all
               end
-              @context.temporary_tables << @target_table
+              @temporary_tables << @target_table
             end
-            dynamic_column.apply(@target_table)
           end
+          @shard_targets << [self, @target_table]
+        end
 
-          create_expression_builder(shard_key) do |expression_builder|
+        def execute
+          create_expression_builder(@shard.key) do |expression_builder|
             case @cover_type
             when :all
               filter_shard_all(expression_builder)
@@ -739,11 +824,26 @@ module Groonga
           end
         end
 
+        def execute_post(result_set)
+          if @post_filter
+            result_set = apply_post_filter(result_set)
+            @temporary_tables << result_set
+          end
+
+          if @sort_keys.empty?
+            @result_sets << result_set
+          else
+            sorted_result_set = result_set.sort(@sort_keys)
+            @temporary_tables << sorted_result_set
+            @result_sets << sorted_result_set
+          end
+        end
+
         private
         def filter_shard_all(expression_builder)
           if @query.nil? and @filter.nil?
-            add_result_set(@target_table, nil)
-            @context.temporary_tables.delete(@target_table)
+            @temporary_tables.delete(@target_table)
+            add_result(@target_table, nil)
           else
             filter_table do |expression|
               expression_builder.build_all(expression)
@@ -775,10 +875,16 @@ module Groonga
           table = @target_table
           expression = create_expression(table)
           yield(expression)
-          add_result_set(table.select(expression), expression)
+          add_result(table.select(expression), expression)
+        end
+
+        def apply_post_filter(table)
+          expression = create_expression(table)
+          expression.parse(@post_filter)
+          table.select(expression)
         end
 
-        def add_result_set(result_set, condition)
+        def add_result(result_set, condition)
           query_logger.log(:size, ":",
                            "select(#{result_set.size})[#{@shard.table_name}]")
 
@@ -787,20 +893,16 @@ module Groonga
             return
           end
 
-          @context.dynamic_columns.each_filtered do |dynamic_column|
-            if result_set == @shard.table
+          if result_set == @shard.table
+            if @context.dynamic_columns.have_filtered?
               result_set = result_set.select_all
+              @temporary_tables << result_set
             end
-            dynamic_column.apply(result_set, condition)
-          end
-
-          if @sort_keys.empty?
-            @result_sets << result_set
           else
-            @unsorted_result_sets << result_set
-            sorted_result_set = result_set.sort(@sort_keys)
-            @result_sets << sorted_result_set
+            @temporary_tables << result_set
           end
+
+          @shard_results << [self, result_set, condition]
         end
 
         def query_logger

data/vendor/local/lib/groonga/plugins/suggest/suggest.la

@@ -17,7 +17,7 @@ old_library='suggest.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libgroonga.la /build/dist-x64/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp -lmsgpackc'
+dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libgroonga.la /build/dist-x64/lib/libonigmo.la -llz4 -lws2_32 -ldbghelp -lpsapi -lz -lmsgpackc'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la

@@ -17,7 +17,7 @@ old_library='stop_word.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libgroonga.la -lmsgpackc /build/dist-x64/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp'
+dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libgroonga.la -lmsgpackc /build/dist-x64/lib/libonigmo.la -llz4 -lws2_32 -ldbghelp -lpsapi -lz'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la

@@ -17,7 +17,7 @@ old_library='mecab.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libmecab.la /build/dist-x64/lib/libgroonga.la -lmsgpackc /build/dist-x64/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp'
+dependency_libs=' -L/build/dist-x64/lib /build/dist-x64/lib/libmecab.la /build/dist-x64/lib/libgroonga.la -lmsgpackc /build/dist-x64/lib/libonigmo.la -llz4 -lws2_32 -ldbghelp -lpsapi -lz'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/scripts/ruby/backtrace_entry.rb

@@ -6,7 +6,7 @@ module Groonga
         file = match_data.pre_match
         line = match_data[1].to_i
         detail_match_data = /\A(in )?(\S+)\s*/.match(match_data.post_match)
-        if detail_match_data[1]
+        if detail_match_data and detail_match_data[1]
           method = detail_match_data[2]
           message = detail_match_data.post_match
         else

data/vendor/local/lib/groonga/scripts/ruby/command_line/grndb.rb

@@ -5,6 +5,7 @@ module Groonga
     class Grndb
       def initialize(argv)
         @program_path, *@arguments = argv
+        @output = LocaleOutput.new($stderr)
         @succeeded = true
         @database_path = nil
       end
@@ -15,9 +16,9 @@ module Groonga
         begin
           options = command_line_parser.parse(@arguments)
         rescue Slop::Error => error
-          $stderr.puts(error.message)
-          $stderr.puts
-          $stderr.puts(command_line_parser.help_message)
+          @output.puts(error.message)
+          @output.puts
+          @output.puts(command_line_parser.help_message)
           return false
         end
         @succeeded
@@ -68,9 +69,9 @@ module Groonga
       def open_database(command, options)
         arguments = options.arguments
         if arguments.empty?
-          $stderr.puts("Database path is missing")
-          $stderr.puts
-          $stderr.puts(command.help_message)
+          @output.puts("Database path is missing")
+          @output.puts
+          @output.puts(command.help_message)
           @succeesed = false
           return
         end
@@ -80,8 +81,8 @@ module Groonga
         begin
           database = Database.open(@database_path)
         rescue Error => error
-          $stderr.puts("Failed to open database: <#{@database_path}>")
-          $stderr.puts(error.message)
+          @output.puts("Failed to open database: <#{@database_path}>")
+          @output.puts(error.message)
           @succeeded = false
           return
         end
@@ -95,13 +96,13 @@ module Groonga
 
       def failed(*messages)
         messages.each do |message|
-          $stderr.puts(message)
+          @output.puts(message)
         end
         @succeeded = false
       end
 
       def recover(database, options, arguments)
-        recoverer = Recoverer.new
+        recoverer = Recoverer.new(@output)
         recoverer.database = database
         recoverer.force_truncate = options[:force_truncate]
         recoverer.force_lock_clear = options[:force_lock_clear]
@@ -114,7 +115,7 @@ module Groonga
       end
 
       def check(database, options, arguments)
-        checker = Checker.new
+        checker = Checker.new(@output)
         checker.program_path = @program_path
         checker.database_path = @database_path
         checker.database = database
@@ -142,7 +143,8 @@ module Groonga
         attr_writer :log_paths
         attr_writer :on_failure
 
-        def initialize
+        def initialize(output)
+          @output = output
           @context = Context.instance
           @checked = {}
         end
@@ -210,6 +212,8 @@ module Groonga
 
         private
         def check_log_file(log_file)
+          return # Disable for now
+
           parser = GroongaLog::Parser.new
           parser.parse(log_file) do |statistic|
             p statistic.to_h
@@ -451,13 +455,17 @@ module Groonga
         attr_writer :force_truncate
         attr_writer :force_lock_clear
 
-        def initialize
+        def initialize(output)
+          @output = output
           @context = Context.instance
         end
 
         def recover
           if @force_truncate
-            truncate_corrupt_objects
+            @database.each do |object|
+              next unless truncate_target?(object)
+              truncate_broken_object(object)
+            end
           end
           if @force_lock_clear
             clear_locks
@@ -465,27 +473,48 @@ module Groonga
           @database.recover
         end
 
-        def truncate_corrupt_objects
-          @database.each do |object|
-            next unless object.corrupt?
-            logger = @context.logger
-            object_path = object.path
-            object_dirname = File.dirname(object_path)
-            object_basename = File.basename(object_path)
-            object.truncate
-            Dir.foreach(object_dirname) do |path|
-              if path.start_with?("#{object_basename}.")
-                begin
-                  File.unlink("#{object_dirname}/#{path}")
-                  message = "Corrupted <#{object_path}> related file is removed: <#{path}>"
-                  $stdout.puts(message)
-                  logger.log(Logger::Level::INFO.to_i, __FILE__, __LINE__, "truncate_corrupt_objects", message)
-                rescue Error => error
-                  message = "Failed to remove file which is related to corrupted <#{object_path}>: <#{path}>"
-                  $stderr.puts(message)
-                  logger.log_error(message)
-                end
-              end
+        private
+        def truncate_target?(object)
+          return true if object.corrupt?
+
+          case object
+          when IndexColumn
+            # It'll be recovered later.
+            false
+          when Column, Table
+            object.locked?
+          else
+            false
+          end
+        end
+
+        def truncate_broken_object(object)
+          logger = @context.logger
+          name = object.name
+          object_path = object.path
+          object_dirname = File.dirname(object_path)
+          object_basename = File.basename(object_path)
+          object.truncate
+          message = "[#{name}] Truncated broken object: <#{object_path}>"
+          @output.puts(message)
+          logger.log(:info, message)
+
+          Dir.foreach(object_dirname) do |path|
+            next unless path.start_with?("#{object_basename}.")
+
+            full_path = "#{object_dirname}/#{path}"
+            begin
+              File.unlink(full_path)
+              message =
+                "[#{name}] Removed broken object related file: <#{full_path}>"
+              @output.puts(message)
+              logger.log(:info, message)
+            rescue Error => error
+              message =
+                "[#{name}] Failed to remove broken object related file: " +
+                "<#{full_path}>: #{error.class}: #{error.message}"
+              @output.puts(message)
+              logger.log_error(message)
             end
           end
         end