rroonga 9.0.3-x64-mingw32 → 9.0.7-x64-mingw32

data/vendor/local/lib/groonga/plugins/sharding/range_expression_builder.rb

@@ -79,7 +79,7 @@ module Groonga
         if @query
           is_empty = expression.empty?
           if @match_columns
-            table = Context.instance[expression[0].domain]
+            table = Context.instance[expression[0].domain_id]
             @match_columns_expression = Expression.create(table)
             @match_columns_expression.parse(@match_columns)
           end

data/vendor/local/lib/groonga/plugins/suggest/suggest.la

@@ -1,5 +1,5 @@
 # suggest.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la

@@ -1,5 +1,5 @@
 # stop_word.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la

@@ -1,5 +1,5 @@
 # mecab.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-2
+# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-10
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/scripts/ruby/command.rb

@@ -1,5 +1,8 @@
 module Groonga
   class Command
+    include Loggable
+    include QueryLoggable
+
     @@classes = {}
     class << self
       def register_class(name, klass)
@@ -20,10 +23,6 @@ module Groonga
       @writer ||= context.writer
     end
 
-    def query_logger
-      @query_logger ||= context.query_logger
-    end
-
     def cache_key(input)
       nil
     end

data/vendor/local/lib/groonga/scripts/ruby/command_line/grndb.rb

@@ -3,6 +3,8 @@ require "groonga-log"
 module Groonga
   module CommandLine
     class Grndb
+      include Loggable
+
       def initialize(argv)
         @program_path, *@arguments = argv
         @output = LocaleOutput.new($stderr)
@@ -36,8 +38,15 @@ module Groonga
           options.banner += " DB_PATH"
           options.string("--target", "Check only the target object.")
           options.array("--groonga-log-path",
-                        "Path to Groonga log file to be checked.",
-                        "You can specify multiple times to specify multiple log files.")
+                        "Path to Groonga log file to be checked. " +
+                        "You can specify multiple times to " +
+                        "specify multiple log files.")
+          options.time("--since",
+                       "Check only objects that are modified " +
+                       "since the specified time. " +
+                       "You can use ISO 8601 format or " +
+                       "-${N}${UNIT} format such as " +
+                       "-3days and -2.5weeks.")
 
           command.add_action do |options|
             open_database(command, options) do |database, rest_arguments|
@@ -97,6 +106,7 @@ module Groonga
       def failed(*messages)
         messages.each do |message|
           @output.puts(message)
+          logger.log(:error, message)
         end
         @succeeded = false
       end
@@ -115,9 +125,14 @@ module Groonga
       end
 
       def check(database, options, arguments)
+        message = "Checking database: <#{@database_path}>"
+        since = options[:since]
+        message << ": <#{since.iso8601}>" if since
+        logger.log(:info, message)
         checker = Checker.new(@output)
         checker.program_path = @program_path
         checker.database_path = @database_path
+        checker.since = since
         checker.database = database
         checker.log_paths = options[:groonga_log_path]
         checker.on_failure = lambda do |message|
@@ -130,15 +145,21 @@ module Groonga
 
         target_name = options[:target]
         if target_name
+          logger.log(:info, "Checking object: <#{target_name}>")
           checker.check_one(target_name)
+          logger.log(:info, "Checked object: <#{target_name}>")
         else
           checker.check_all
         end
+        logger.log(:info, "Checked database: <#{@database_path}>")
       end
 
       class Checker
+        include Loggable
+
         attr_writer :program_path
         attr_writer :database_path
+        attr_writer :since
         attr_writer :database
         attr_writer :log_paths
         attr_writer :on_failure
@@ -169,30 +190,19 @@ module Groonga
         end
 
         def check_database
+          return unless target_database?
+
           check_database_orphan_inspect
           check_database_locked
           check_database_corrupt
           check_database_dirty
+          check_database_empty_files
         end
 
         def check_one(target_name)
-          target = @context[target_name]
-          if target.nil?
-            exist_p = open_database_cursor do |cursor|
-              cursor.any? do
-                cursor.key == target_name
-              end
-            end
-            if exist_p
-              failed_to_open(target_name)
-            else
-              message = "[#{target_name}] Not exist."
-              failed(message)
-            end
-            return
+          open_object(target_name) do |target|
+            check_object_recursive(target)
           end
-
-          check_object_recursive(target)
         end
 
         def check_all
@@ -200,14 +210,11 @@ module Groonga
             cursor.each do |id|
               next if ID.builtin?(id)
               next if builtin_object_name?(cursor.key)
-              next if @context[id]
-              failed_to_open(cursor.key)
+              open_object(id) do |object|
+                check_object(object)
+              end
             end
           end
-
-          @database.each do |object|
-            check_object(object)
-          end
         end
 
         private
@@ -221,21 +228,31 @@ module Groonga
         end
 
         def check_database_orphan_inspect
+          found = false
           open_database_cursor do |cursor|
             cursor.each do |id|
               if cursor.key == "inspect" and @context[id].nil?
                 message =
                   "Database has orphan 'inspect' object. " +
                   "Remove it by '#{@program_path} recover #{@database_path}'."
+                found = true
                 failed(message)
                 break
               end
             end
           end
+          unless found
+            logger.log(:info,
+                       "Database doesn't have orphan 'inspect' object: " +
+                       "<#{@database_path}>")
+          end
         end
 
         def check_database_locked
-          return unless @database.locked?
+          unless @database.locked?
+            logger.log(:info, "Database is not locked: <#{@database_path}>")
+            return
+          end
 
           message =
             "Database is locked. " +
@@ -245,7 +262,10 @@ module Groonga
         end
 
         def check_database_corrupt
-          return unless @database.corrupt?
+          unless @database.corrupt?
+            logger.log(:info, "Database is not corrupted: <#{@database_path}>")
+            return
+          end
 
           message =
             "Database is corrupt. " +
@@ -254,7 +274,10 @@ module Groonga
         end
 
         def check_database_dirty
-          return unless @database.dirty?
+          unless @database.dirty?
+            logger.log(:info, "Database is not dirty: <#{@database_path}>")
+            return
+          end
 
           last_modified = @database.last_modified
           if File.stat(@database.path).mtime > last_modified
@@ -264,7 +287,7 @@ module Groonga
           open_database_cursor do |cursor|
             cursor.each do |id|
               next if ID.builtin?(id)
-              path = "%s.%07x" % [@database.path, id]
+              path = object_path(id)
               next unless File.exist?(path)
               return if File.stat(path).mtime > last_modified
             end
@@ -277,6 +300,20 @@ module Groonga
           failed(message)
         end
 
+        def check_database_empty_files
+          dirname = File.dirname(@database.path)
+          basename = File.basename(@database.path)
+
+          Dir.entries(dirname).sort.each do |path|
+            next unless path.start_with?(basename)
+
+            full_path = "#{dirname}/#{path}"
+            if File.stat(full_path).size.zero?
+              failed("Empty file exists: <#{full_path}>")
+            end
+          end
+        end
+
         def check_object(object)
           return if @checked.key?(object.id)
           @checked[object.id] = true
@@ -288,14 +325,20 @@ module Groonga
         def check_object_locked(object)
           case object
           when IndexColumn
-            return unless object.locked?
+            unless object.locked?
+              logger.log(:info, "[#{object.name}] Index column is not locked")
+              return
+            end
             message =
               "[#{object.name}] Index column is locked. " +
               "It may be broken. " +
               "Re-create index by '#{@program_path} recover #{@database_path}'."
             failed(message)
           when Column
-            return unless object.locked?
+            unless object.locked?
+              logger.log(:info, "[#{object.name}] Column is not locked")
+              return
+            end
             name = object.name
             message =
               "[#{name}] Data column is locked. " +
@@ -305,7 +348,10 @@ module Groonga
               "and (2) load data again."
             failed(message)
           when Table
-            return unless object.locked?
+            unless object.locked?
+              logger.log(:info, "[#{object.name}] Table is not locked")
+              return
+            end
             name = object.name
             message =
               "[#{name}] Table is locked. " +
@@ -320,13 +366,19 @@ module Groonga
         def check_object_corrupt(object)
           case object
           when IndexColumn
-            return unless object.corrupt?
+            unless object.corrupt?
+              logger.log(:info, "[#{object.name}] Index column is not corrupted")
+              return
+            end
             message =
               "[#{object.name}] Index column is corrupt. " +
               "Re-create index by '#{@program_path} recover #{@database_path}'."
             failed(message)
           when Column
-            return unless object.corrupt?
+            unless object.corrupt?
+              logger.log(:info, "[#{object.name}] Column is not corrupted")
+              return
+            end
             name = object.name
             message =
               "[#{name}] Data column is corrupt. " +
@@ -335,7 +387,10 @@ module Groonga
               "and (2) load data again."
             failed(message)
           when Table
-            return unless object.corrupt?
+            unless object.corrupt?
+              logger.log(:info, "[#{object.name}] Table is not corrupted")
+              return
+            end
             name = object.name
             message =
               "[#{name}] Table is corrupt. " +
@@ -353,67 +408,47 @@ module Groonga
           case target
           when Table
             unless target.is_a?(Groonga::Array)
-              domain_id = target.domain_id
-              domain = @context[domain_id]
-              if domain.nil?
-                record = Record.new(@database, domain_id)
-                failed_to_open(record.key)
-              elsif domain.is_a?(Table)
-                check_object_recursive(domain)
+              open_object(target.domain_id) do |domain|
+                if domain.is_a?(Table)
+                  check_object_recursive(domain)
+                end
               end
             end
 
             target.column_ids.each do |column_id|
-              column = @context[column_id]
-              if column.nil?
-                record = Record.new(@database, column_id)
-                failed_to_open(record.key)
-              else
+              open_object(column_id) do |column|
                 check_object_recursive(column)
               end
             end
           when FixedSizeColumn, VariableSizeColumn
-            range_id = target.range_id
-            range = @context[range_id]
-            if range.nil?
-              record = Record.new(@database, range_id)
-              failed_to_open(record.key)
-            elsif range.is_a?(Table)
-              check_object_recursive(range)
+            open_object(target.range_id) do |range|
+              if range.is_a?(Table)
+                check_object_recursive(range)
+              end
             end
 
             lexicon_ids = []
             target.indexes.each do |index_info|
               index = index_info.index
               lexicon_ids << index.domain_id
+              next unless target_id?(index.id)
               check_object(index)
             end
             lexicon_ids.uniq.each do |lexicon_id|
-              lexicon = @context[lexicon_id]
-              if lexicon.nil?
-                record = Record.new(@database, lexicon_id)
-                failed_to_open(record.key)
-              else
+              open_object(lexicon_id) do |lexicon|
                 check_object(lexicon)
               end
             end
           when IndexColumn
-            range_id = target.range_id
-            range = @context[range_id]
-            if range.nil?
-              record = Record.new(@database, range_id)
-              failed_to_open(record.key)
-              return
-            end
-            check_object(range)
-
-            target.source_ids.each do |source_id|
-              source = @context[source_id]
-              if source.nil?
-                record = Record.new(database, source_id)
-                failed_to_open(record.key)
-              elsif source.is_a?(Column)
-                check_object_recursive(source)
+            open_object(target.range_id) do |range|
+              check_object(range)
+
+              target.source_ids.each do |source_id|
+                open_object(source_id) do |source|
+                  if source.is_a?(Column)
+                    check_object_recursive(source)
+                  end
+                end
               end
             end
           end
@@ -437,6 +472,71 @@ module Groonga
           end
         end
 
+        def target_database?
+          return true if @since.nil?
+
+          suffixes = [
+            "",
+            ".001",
+            ".conf",
+            ".options",
+          ]
+          suffixes.each do |suffix|
+            path = "#{@database.path}#{suffix}"
+            next unless File.exist?(path)
+            return true if File.stat(path).mtime >= @since
+          end
+          false
+        end
+
+        def object_path(id)
+          "%s.%07X" % [@database.path, id]
+        end
+
+        def target_id?(id)
+          return true if @since.nil?
+
+          target_path = object_path(id)
+          return false unless File.exist?(target_path)
+          return true if File.stat(target_path).mtime >= @since
+
+          dir_path = File.dirname(@database.path)
+          Dir.open(dir_path) do |dir|
+            dir.each do |path|
+              full_path = File.join(dir_path, path)
+              next unless full_path.start_with?(target_path)
+              return true if File.stat(full_path).mtime >= @since
+            end
+          end
+          false
+        end
+
+        def open_object(name_or_id)
+          if name_or_id.is_a?(Integer)
+            id = name_or_id
+            return if id.nil?
+            name = @database[id]
+          else
+            name = name_or_id
+            id = @database[name_or_id]
+          end
+
+          if id.nil?
+            failed("[#{name}] Not exist.")
+            return
+          end
+
+          return unless target_id?(id)
+
+          object = @context[name]
+          if object.nil?
+            failed_to_open(name)
+            return
+          end
+
+          yield(object)
+        end
+
         def failed(message)
           @on_failure.call(message)
         end
@@ -451,6 +551,8 @@ module Groonga
       end
 
       class Recoverer
+        include Loggable
+
         attr_writer :database
         attr_writer :force_truncate
         attr_writer :force_lock_clear
@@ -461,6 +563,7 @@ module Groonga
         end
 
         def recover
+          logger.log(:info, "Recovering database: <#{@database.path}>")
           if @force_truncate
             @database.each do |object|
               next unless truncate_target?(object)
@@ -470,7 +573,9 @@ module Groonga
           if @force_lock_clear
             clear_locks
           end
+          remove_empty_files
           @database.recover
+          logger.log(:info, "Recovered database: <#{@database.path}>")
         end
 
         private
@@ -489,7 +594,6 @@ module Groonga
         end
 
         def truncate_broken_object(object)
-          logger = @context.logger
           name = object.name
           object_path = object.path
           object_dirname = File.dirname(object_path)
@@ -499,7 +603,7 @@ module Groonga
           @output.puts(message)
           logger.log(:info, message)
 
-          Dir.foreach(object_dirname) do |path|
+          Dir.entries(object_dirname).sort.each do |path|
             next unless path.start_with?("#{object_basename}.")
 
             full_path = "#{object_dirname}/#{path}"
@@ -514,7 +618,8 @@ module Groonga
                 "[#{name}] Failed to remove broken object related file: " +
                 "<#{full_path}>: #{error.class}: #{error.message}"
               @output.puts(message)
-              logger.log_error(message)
+              logger.log(:error, message)
+              logger.log_error(error)
             end
           end
         end
@@ -522,6 +627,7 @@ module Groonga
         def clear_locks
           if @database.locked?
             @database.clear_lock
+            logger.log(:info, "Clear locked database: <#{@database.path}>")
           end
           @database.each do |object|
             case object
@@ -530,6 +636,90 @@ module Groonga
             when Column, Table
               next unless object.locked?
               object.clear_lock
+              logger.log(:info,
+                         "[#{object.name}] Clear locked object: " +
+                         "<#{object.path}>")
+            end
+          end
+        end
+
+        def remove_empty_files
+          dirname = File.dirname(@database.path)
+          basename = File.basename(@database.path)
+
+          broken_ids = {}
+          Dir.entries(dirname).sort.each do |path|
+            next unless path.start_with?(basename)
+
+            full_path = "#{dirname}/#{path}"
+            next unless File.stat(full_path).size.zero?
+
+            match_data =
+              /\A#{Regexp.escape(basename)}\.([\da-fA-F]{7})/.match(path)
+            if match_data
+              id = Integer(match_data[1], 16)
+              name = @database[id]
+              if name
+                broken_ids[id] = true
+                next
+              end
+            end
+
+            begin
+              File.unlink(full_path)
+              message = "Removed empty file: <#{full_path}>"
+              @output.puts(message)
+              logger.log(:info, message)
+            rescue Error => error
+              message =
+                "Failed to remove empty file: <#{full_path}>: " +
+                "#{error.class}: #{error.message}"
+              logger.log(:error, message)
+              logger.log_error(error)
+            end
+          end
+
+          broken_ids.each_key do |id|
+            name = @database[id]
+            broken_basename = "%s.%07x" % [basename, id]
+
+            begin
+              Object.remove_force(name)
+              message =
+                "[#{name}] Remove a broken object that has empty file: " +
+                "<#{dirname}/#{broken_basename}>"
+              @output.puts(message)
+              logger.log(:info, message)
+            rescue Error => error
+              message =
+                "[#{name}] Failed to remove a broken object " +
+                "that has empty file: <#{dirname}/#{broken_basename}>: " +
+                "#{error.class}: #{error.message}"
+              logger.log(:error, message)
+              logger.log_error(error)
+              next
+            end
+
+            Dir.entries(dirname).sort.each do |path|
+              next unless path.start_with?(broken_basename)
+
+              full_path = "#{dirname}/#{path}"
+
+              begin
+                File.unlink(full_path)
+                message =
+                  "[#{name}] Removed a file for broken object " +
+                  "that has empty file: <#{full_path}>"
+                @output.puts(message)
+                logger.log(:info, message)
+              rescue Error => error
+                message =
+                  "[#{name}] Failed to removed a file for broken object " +
+                  "that has empty file: <#{full_path}>: " +
+                  "#{error.class}: #{error.message}"
+                logger.log(:error, message)
+                logger.log_error(error)
+              end
             end
           end
         end