rroonga 7.0.2-x86-mingw32 → 7.1.1-x86-mingw32

data/vendor/local/lib/groonga/plugins/sharding/range_expression_builder.rb

@@ -16,6 +16,21 @@ module Groonga
         @filter = nil
       end
 
+      def build(expression, shard_range)
+        cover_type = @target_range.cover_type(shard_range)
+        case cover_type
+        when :all
+          build_all(expression)
+        when :partial_min
+          build_partial_min(expression)
+        when :partial_max
+          build_partial_max(expression)
+        when :partial_min_and_max
+          build_partial_min_and_max(expression)
+        when :none
+        end
+      end
+
       def build_all(expression)
         build_condition(expression)
       end

data/vendor/local/lib/groonga/plugins/suggest/suggest.la

@@ -17,7 +17,7 @@ old_library='suggest.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libgroonga.la /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32 -lmsgpackc'
+dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libgroonga.la /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp -lmsgpackc'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la

@@ -17,7 +17,7 @@ old_library='stop_word.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libgroonga.la -lmsgpackc /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32'
+dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libgroonga.la -lmsgpackc /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la

@@ -17,7 +17,7 @@ old_library='mecab.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libmecab.la /build/dist-x86/lib/libgroonga.la -lmsgpackc /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32'
+dependency_libs=' -L/build/dist-x86/lib /build/dist-x86/lib/libmecab.la /build/dist-x86/lib/libgroonga.la -lmsgpackc /build/dist-x86/lib/libonigmo.la -llz4 -lz -lws2_32 -ldbghelp'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/scripts/ruby/command_line/grndb.rb

@@ -1,3 +1,5 @@
+require "groonga-log"
+
 module Groonga
   module CommandLine
     class Grndb
@@ -32,6 +34,9 @@ module Groonga
           options = command.options
           options.banner += " DB_PATH"
           options.string("--target", "Check only the target object.")
+          options.array("--groonga-log-path",
+                        "Path to Groonga log file to be checked.",
+                        "You can specify multiple times to specify multiple log files.")
 
           command.add_action do |options|
             open_database(command, options) do |database, rest_arguments|
@@ -45,6 +50,10 @@ module Groonga
 
           options = command.options
           options.banner += " DB_PATH"
+          options.boolean("--force-truncate",
+                          "Force to truncate corrupted objects.")
+          options.boolean("--force-lock-clear",
+                          "Force to clear lock of locked objects.")
 
           command.add_action do |options|
             open_database(command, options) do |database, rest_arguments|
@@ -92,8 +101,12 @@ module Groonga
       end
 
       def recover(database, options, arguments)
+        recoverer = Recoverer.new
+        recoverer.database = database
+        recoverer.force_truncate = options[:force_truncate]
+        recoverer.force_lock_clear = options[:force_lock_clear]
         begin
-          database.recover
+          recoverer.recover
         rescue Error => error
           failed("Failed to recover database: <#{@database_path}>",
                  error.message)
@@ -105,10 +118,13 @@ module Groonga
         checker.program_path = @program_path
         checker.database_path = @database_path
         checker.database = database
+        checker.log_paths = options[:groonga_log_path]
         checker.on_failure = lambda do |message|
           failed(message)
         end
 
+        checker.check_log_paths
+
         checker.check_database
 
         target_name = options[:target]
@@ -123,6 +139,7 @@ module Groonga
         attr_writer :program_path
         attr_writer :database_path
         attr_writer :database
+        attr_writer :log_paths
         attr_writer :on_failure
 
         def initialize
@@ -130,8 +147,29 @@ module Groonga
           @checked = {}
         end
 
+        def check_log_paths
+          @log_paths.each do |log_path|
+            begin
+              log_file = File.new(log_path)
+            rescue => error
+              message = "[#{log_path}] Can't open Groonga log path: "
+              message << "#{error.class}: #{error.message}"
+              failed(message)
+              next
+            end
+
+            begin
+              check_log_file(log_file)
+            ensure
+              log_file.close
+            end
+          end
+        end
+
         def check_database
+          check_database_orphan_inspect
           check_database_locked
+          check_database_corrupt
           check_database_dirty
         end
 
@@ -171,6 +209,27 @@ module Groonga
         end
 
         private
+        def check_log_file(log_file)
+          parser = GroongaLog::Parser.new
+          parser.parse(log_file) do |statistic|
+            p statistic.to_h
+          end
+        end
+
+        def check_database_orphan_inspect
+          open_database_cursor do |cursor|
+            cursor.each do |id|
+              if cursor.key == "inspect" and @context[id].nil?
+                message =
+                  "Database has orphan 'inspect' object. " +
+                  "Remove it by '#{@program_path} recover #{@database_path}'."
+                failed(message)
+                break
+              end
+            end
+          end
+        end
+
         def check_database_locked
           return unless @database.locked?
 
@@ -181,6 +240,15 @@ module Groonga
           failed(message)
         end
 
+        def check_database_corrupt
+          return unless @database.corrupt?
+
+          message =
+            "Database is corrupt. " +
+            "Re-create the database."
+          failed(message)
+        end
+
         def check_database_dirty
           return unless @database.dirty?
 
@@ -209,6 +277,11 @@ module Groonga
           return if @checked.key?(object.id)
           @checked[object.id] = true
 
+          check_object_locked(object)
+          check_object_corrupt(object)
+        end
+
+        def check_object_locked(object)
           case object
           when IndexColumn
             return unless object.locked?
@@ -240,6 +313,35 @@ module Groonga
           end
         end
 
+        def check_object_corrupt(object)
+          case object
+          when IndexColumn
+            return unless object.corrupt?
+            message =
+              "[#{object.name}] Index column is corrupt. " +
+              "Re-create index by '#{@program_path} recover #{@database_path}'."
+            failed(message)
+          when Column
+            return unless object.corrupt?
+            name = object.name
+            message =
+              "[#{name}] Data column is corrupt. " +
+              "(1) Truncate the column (truncate #{name} or " +
+              "'#{@program_path} recover --force-truncate #{@database_path}') " +
+              "and (2) load data again."
+            failed(message)
+          when Table
+            return unless object.corrupt?
+            name = object.name
+            message =
+              "[#{name}] Table is corrupt. " +
+              "(1) Truncate the table (truncate #{name} or " +
+              "'#{@program_path} recover --force-truncate #{@database_path}') " +
+              "and (2) load data again."
+            failed(message)
+          end
+        end
+
         def check_object_recursive(target)
           return if @checked.key?(target.id)
 
@@ -343,6 +445,66 @@ module Groonga
           failed(message)
         end
       end
+
+      class Recoverer
+        attr_writer :database
+        attr_writer :force_truncate
+        attr_writer :force_lock_clear
+
+        def initialize
+          @context = Context.instance
+        end
+
+        def recover
+          if @force_truncate
+            truncate_corrupt_objects
+          end
+          if @force_lock_clear
+            clear_locks
+          end
+          @database.recover
+        end
+
+        def truncate_corrupt_objects
+          @database.each do |object|
+            next unless object.corrupt?
+            logger = @context.logger
+            object_path = object.path
+            object_dirname = File.dirname(object_path)
+            object_basename = File.basename(object_path)
+            object.truncate
+            Dir.foreach(object_dirname) do |path|
+              if path.start_with?("#{object_basename}.")
+                begin
+                  File.unlink("#{object_dirname}/#{path}")
+                  message = "Corrupted <#{object_path}> related file is removed: <#{path}>"
+                  $stdout.puts(message)
+                  logger.log(Logger::Level::INFO.to_i, __FILE__, __LINE__, "truncate_corrupt_objects", message)
+                rescue Error => error
+                  message = "Failed to remove file which is related to corrupted <#{object_path}>: <#{path}>"
+                  $stderr.puts(message)
+                  logger.log_error(message)
+                end
+              end
+            end
+          end
+        end
+
+        def clear_locks
+          if @database.locked?
+            @database.clear_lock
+          end
+          @database.each do |object|
+            case object
+            when IndexColumn
+              # Ignore. It'll be recovered later.
+            when Column, Table
+              next unless object.locked?
+              object.clear_lock
+            end
+          end
+        end
+      end
     end
   end
 end

data/vendor/local/lib/groonga/scripts/ruby/command_line_parser.rb

@@ -30,6 +30,7 @@ module Groonga
 
     def add_command(name)
       command = Command.new(@program_name, name)
+      setup_common_options(command.options)
       yield(command)
       @commands << command
     end
@@ -81,12 +82,23 @@ module Groonga
     private
     def setup_options
       @options.banner = "Usage: #{@program_name} [OPTIONS]"
+      setup_common_options(@options)
       @options.on("-h", "--help", "Display this help message.",
                   :tail => true) do
         $stdout.puts(help_message)
       end
     end
 
+    def setup_common_options(options)
+      options.string("--log-path",
+                     "Change log path (#{Logger.default_path})",
+                     default: Logger.default_path)
+      default_log_level = Logger.default_level
+      options.string("--log-level",
+                     "Change log level (#{default_log_level.name})",
+                     default: default_log_level)
+    end
+
     def find_command(name)
       @commands.find do |command|
         command.name == name

data/vendor/local/lib/groonga/scripts/ruby/expression_tree/function_call.rb

@@ -21,9 +21,14 @@ module Groonga
 
         column, min, min_border, max, max_border = @arguments
 
-        index_info = column.column.find_index(Operator::CALL)
-        return table.size if index_info.nil?
-        index_column = index_info.index
+        if column.is_a?(Groonga::ExpressionTree::IndexColumn)
+          index_column = column.object
+        else
+          index_info = column.column.find_index(Operator::CALL)
+          return table.size if index_info.nil?
+          index_column = index_info.index
+        end
+
         while index_column.is_a?(Groonga::Accessor)
           if index_column.have_next?
             index_column = index_column.next

data/vendor/local/lib/groonga/scripts/ruby/expression_tree_builder.rb

@@ -21,6 +21,7 @@ module Groonga
       Operator::TERM_EXTRACT,
       Operator::REGEXP,
       Operator::FUZZY,
+      Operator::QUORUM,
     ]
 
     ARITHMETIC_OPERATORS = [

data/{lib/groonga/table.rb → vendor/local/lib/groonga/scripts/ruby/groonga-log.rb}

@@ -1,10 +1,9 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017 Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
 #
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -15,11 +14,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-module Groonga
-  class Table
-    def disk_usage
-      measurer = StatisticMeasurer.new
-      measurer.measure_disk_usage(path)
-    end
-  end
-end
+require "groonga-log/parser"
+require "groonga-log/statistic"
+require "groonga-log/version"

data/vendor/local/lib/groonga/scripts/ruby/groonga-log/parser.rb

@@ -0,0 +1,81 @@
+# Copyright (C) 2017 Yasuhiro Horimoto <horimoto@clear-code.com>
+# Copyright (C) 2017 Kentaro Hayashi <hayashi@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga-log/statistic"
+
+module GroongaLog
+  class Parser
+    PATTERN =
+      /\A(?<year>\d{4})-(?<month>\d\d)-(?<day>\d\d)
+          \ (?<hour>\d\d):(?<minute>\d\d):(?<second>\d\d)\.(?<micro_second>\d+)
+          \|(?<log_level>.)
+          \|(?:(?<pid>\d+):)?
+          \ (?<message>.*)/x
+
+    def parse(input)
+      return to_enum(:parse, input) unless block_given?
+
+      input.each_line do |line|
+        if line.respond_to?(:valid_encoding?)
+          next unless line.valid_encoding?
+        end
+
+        m = PATTERN.match(line)
+
+        statistic = Statistic.new
+
+        year = m["year"].to_i
+        month = m["month"].to_i
+        day = m["day"].to_i
+        hour = m["hour"].to_i
+        minute = m["minute"].to_i
+        second = m["second"].to_i
+        micro_second = m["micro_second"].to_i
+        statistic.timestamp = Time.local(year, month, day,
+                                         hour, minute, second, micro_second)
+        statistic.log_level = log_level_to_symbol(m["log_level"])
+        statistic.pid = m["pid"].to_i if m["pid"]
+        statistic.message = m["message"]
+        yield statistic
+      end
+    end
+
+    private
+    def log_level_to_symbol(level_text)
+      case level_text
+      when "E"
+        :emergency
+      when "A"
+        :alert
+      when "C"
+        :critical
+      when "e"
+        :error
+      when "w"
+        :warning
+      when "n"
+        :notice
+      when "i"
+        :information
+      when "d"
+        :debug
+      when "-"
+        :dump
+      end
+    end
+  end
+end