groonga-client 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31002e6c2a6cc64ea8249019c9118dcc785e5de9
-  data.tar.gz: '03379d77558768d720f2abf650fc5a2cca03e4d5'
+  metadata.gz: b31699984e9b4cfb9751dc8cf4481031d10a2fa1
+  data.tar.gz: ea990b190ba221e32990e5b2f08118acb3b2ca63
 SHA512:
-  metadata.gz: ec7bef6d724d2601fa4b757522ba50439784d50b503ba670831b01cbd0068d0dd310cd30e3513902925d6558a340d6319f89e5576a72cba5722513de1c3fe5dc
-  data.tar.gz: c466483d16efc8e6b1f66256bfba88faac81c11877552c9060d501b2c6d1f9befd5d71c1982833e9ea02e22152169ba80fb6fc0efcd973616d9b06586eae616f
+  metadata.gz: 3485123969cc7cdf43b7c1a8afba2ec7b58d5eff387c3506bcdc344bd0a6d4da3ddf5c65dca1018fe58fa94531510c5f47ba4b5e455ac988fbc44fd0b831631e
+  data.tar.gz: fc4ce6ee9206924e5b44d59243fb8a1544e15cf5cf7adcd87105c977443c4aa233e4165a99d3df01d1914841f3178a3c274779172d49449d835eb4fd1fc02dc0

data/bin/groonga-client

@@ -17,12 +17,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "net/http"
-
-if Net::HTTP.const_defined?(:IDEMPOTENT_METHODS_)
-  Net::HTTP::IDEMPOTENT_METHODS_.clear
-end
-
 require "groonga/client/command-line/groonga-client"
 
 command_line = Groonga::Client::CommandLine::GroongaClient.new

data/bin/groonga-client-index-check

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+#
+# Copyright (C) 2015-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017 Kentaro Hayashi <hayashi@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/client/command-line/groonga-client-index-check"
+
+command_line = Groonga::Client::CommandLine::GroongaClientIndexCheck.new
+exit(command_line.run(ARGV))

data/doc/text/news.md

@@ -1,5 +1,16 @@
 # NEWS
 
+## 0.5.5 - 2017-10-31
+
+### Improvements
+
+  * `groonga-client-index-check`: Added a new command that finds
+    indexes which doesn't have source or indexes whose content is
+    broken.
+
+  * Disabled auto retry feature by `net/http` explicitly because `GET`
+    isn't idempotent request in Groonga such as `delete` command.
+  
 ## 0.5.4 - 2017-10-27
 
 ### Improvements

data/groonga-client.gemspec

@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("gqtp", ">= 1.0.4")
   spec.add_runtime_dependency("groonga-command", ">= 1.2.8")
-  spec.add_runtime_dependency("groonga-command-parser", ">= 1.0.7")
+  spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
   spec.add_runtime_dependency("hashie")
 
   spec.add_development_dependency("bundler")

data/lib/groonga/client/command-line/groonga-client-index-check.rb

@@ -0,0 +1,190 @@
+# Copyright (C) 2015-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017 Kentaro Hayashi <hayashi@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/client"
+require "groonga/client/command-line/parser"
+require "groonga/client/command-line/runner"
+
+module Groonga
+  class Client
+    module CommandLine
+      class GroongaClientIndexCheck
+        def initialize
+          @available_methods = [:source, :content]
+          @methods = []
+        end
+
+        def run(arguments)
+          parser = Parser.new
+          target_names = parser.parse(arguments) do |option_parser|
+            parse_command_line(option_parser)
+          end
+
+          if @methods.empty?
+            @methods = @available_methods
+          end
+
+          parser.open_client do |client|
+            checker = Checker.new(client, @methods, target_names)
+            checker.run
+          end
+        end
+
+        private
+        def parse_command_line(parser)
+          parser.banner += " [LEXICON1.INDEX1 LEXICON2.INDEX2 LEXICON3 ...]"
+
+          parser.separator("")
+          parser.separator("If no indexes are specified, " +
+                           "all indexes are checked.")
+
+          parser.separator("")
+          parser.separator("Method:")
+
+          parser.on("--method=METHOD", @available_methods,
+                    "Specify a method how to check indexes.",
+                    "You can specify this option multiple times",
+                    "to use multiple methods in one execution.",
+                    "All methods are used by default.",
+                    "Available methods:",
+                    "  source: Find indexes that don't have source.",
+                    "  content: Find indexes whose content is broken.",
+                    "(#{@available_methods.join(", ")})") do |method|
+            @methods << method
+          end
+        end
+
+        class Checker < Runner
+          def initialize(client, methods, target_names)
+            super(client)
+            @methods = methods
+            @target_names = target_names
+          end
+
+          private
+          def run_internal
+            succeeded = true
+            each_target_index_column do |index_column|
+              @methods.each do |method|
+                unless __send__("check_#{method}", index_column)
+                  succeeded = false
+                end
+              end
+            end
+            succeeded
+          end
+
+          def each_target_index_column
+            table_list.each do |table|
+              next unless target_table?(table)
+              column_list(table.name).each do |column|
+                next unless column.index?
+                next unless target_column?(column)
+                yield(column)
+              end
+            end
+          end
+
+          def target_table?(table)
+            return true if @target_names.empty?
+            @target_names.any? do |name|
+              if name.include?(".")
+                index_table_name = name.split(".").first
+                index_table_name == table.name
+              else
+                name == table.name
+              end
+            end
+          end
+
+          def target_column?(column)
+            return true if @target_names.empty?
+            @target_names.any? do |name|
+              if name.include?(".")
+                name == column.full_name
+              else
+                name == column.domain
+              end
+            end
+          end
+
+          def check_source(column)
+            return true unless column.source.empty?
+            $stderr.puts("Source is missing: <#{column.full_name}>")
+            false
+          end
+
+          def valid_token?(source_table_name,
+                           full_index_column_name1,
+                           full_index_column_name2,
+                           token)
+            case token
+            when String
+              value = Groonga::Client::ScriptSyntax.format_string(token)
+            else
+              value = token
+            end
+            response1 = select(source_table_name,
+                               :filter => "#{full_index_column_name1} @ #{value}",
+                               :output_columns => "_id",
+                               :limit => "-1",
+                               :sort_keys => "_id")
+            response2 = select(source_table_name,
+                               :filter => "#{full_index_column_name2} @ #{value}",
+                               :output_columns => "_id",
+                               :limit => "-1",
+                               :sort_keys => "_id")
+            response1.records == response2.records
+          end
+
+          def check_content(index_column)
+            return if index_column.source.empty?
+
+            lexicon_name = index_column.domain
+            index_column_name = index_column.name
+            suffix = Time.now.strftime("%Y%m%d%H%M%S_%N")
+            new_index_column_name = "#{index_column_name}_#{suffix}"
+            full_index_column_name = index_column.full_name
+            full_new_index_column_name = "#{full_index_column_name}_#{suffix}"
+            source_table = index_column.range
+            column_create_similar(lexicon_name,
+                                  new_index_column_name,
+                                  index_column_name)
+            begin
+              response = select(lexicon_name,
+                                :limit => "-1",
+                                :output_columns => "_key")
+              response.records.each do |record|
+                token = record["_key"]
+                unless valid_token?(source_table,
+                                    full_index_column_name,
+                                    full_new_index_column_name,
+                                    token)
+                  $stderr.puts("Broken: #{index_column.full_name}: <#{token}>")
+                  return false
+                end
+              end
+              true
+            ensure
+              column_remove(lexicon_name, new_index_column_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/client/command-line/groonga-client-index-recreate.rb

@@ -14,39 +14,31 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "optparse"
 require "json"
 
 require "groonga/client"
+require "groonga/client/command-line/parser"
+require "groonga/client/command-line/runner"
 
 module Groonga
   class Client
     module CommandLine
       class GroongaClientIndexRecreate
         def initialize
-          @url      = nil
-          @protocol = :http
-          @host     = "localhost"
-          @port     = nil
-
-          @read_timeout = -1
-
           @interval = :day
 
           @n_workers = 0
         end
 
-        def run(argv)
-          target_indexes = parse_command_line(argv)
+        def run(arguments)
+          parser = Parser.new(:read_timeout => -1)
+          indexes = parser.parse(arguments) do |option_parser|
+            parse_command_line(option_parser)
+          end
 
-          Client.open(:url      => @url,
-                      :protocol => @protocol,
-                      :host     => @host,
-                      :port     => @port,
-                      :read_timeout => @read_timeout,
-                      :backend  => :synchronous) do |client|
-            runner = Runner.new(client, @interval, target_indexes)
-            runner.run do
+          parser.open_client do |client|
+            recreator = Recreator.new(client, @interval, indexes)
+            recreator.run do
               @n_workers.times do
                 client.database_unmap
               end
@@ -55,48 +47,9 @@ module Groonga
         end
 
         private
-        def parse_command_line(argv)
-          parser = OptionParser.new
-          parser.version = VERSION
+        def parse_command_line(parser)
           parser.banner += " LEXICON1.INDEX1 LEXICON2.INDEX2 ..."
 
-          parser.separator("")
-          parser.separator("Connection:")
-
-          parser.on("--url=URL",
-                    "URL to connect to Groonga server.",
-                    "If this option is specified,",
-                    "--protocol, --host and --port are ignored.") do |url|
-            @url = url
-          end
-
-          available_protocols = [:http, :gqtp]
-          parser.on("--protocol=PROTOCOL", [:http, :gqtp],
-                    "Protocol to connect to Groonga server.",
-                    "[#{available_protocols.join(", ")}]",
-                    "(#{@protocol})") do |protocol|
-            @protocol = protocol
-          end
-
-          parser.on("--host=HOST",
-                    "Groonga server to be connected.",
-                    "(#{@host})") do |host|
-            @host = host
-          end
-
-          parser.on("--port=PORT", Integer,
-                    "Port number of Groonga server to be connected.",
-                    "(auto)") do |port|
-            @port = port
-          end
-
-          parser.on("--read-timeout=TIMEOUT", Integer,
-                    "Timeout on reading response from Groonga server.",
-                    "You can disable timeout by specifying -1.",
-                    "(#{@read_timeout})") do |timeout|
-            @read_timeout = timeout
-          end
-
           parser.separator("")
           parser.separator("Configuration:")
 
@@ -117,91 +70,25 @@ module Groonga
                     "(#{@n_workers})") do |n|
             @n_workers = n
           end
-
-          target_indexes = parser.parse(argv)
-
-          @port ||= default_port(@protocol)
-
-          target_indexes
-        end
-
-        def default_port(protocol)
-          case protocol
-          when :http
-            10041
-          when :gqtp
-            10043
-          end
         end
 
-        class Runner
+        class Recreator < Runner
           def initialize(client, interval, target_indexes)
-            @client = client
+            super(client)
             @interval = interval
             @target_indexes = target_indexes
             @now = Time.now
           end
 
-          def run
-            catch do |tag|
-              @abort_tag = tag
-              alias_column = ensure_alias_column
-              @target_indexes.each do |index|
-                current_index = recreate_index(index, alias_column)
-                remove_old_indexes(index, current_index)
-              end
-              yield if block_given?
-              true
-            end
-          end
-
           private
-          def abort_run(message)
-            $stderr.puts(message)
-            throw(@abort_tag, false)
-          end
-
-          def execute_command(name, arguments={})
-            response = @client.execute(name, arguments)
-            unless response.success?
-              abort_run("Failed to run #{name}: #{response.inspect}")
+          def run_internal
+            alias_column = ensure_alias_column
+            @target_indexes.each do |index|
+              current_index = recreate_index(index, alias_column)
+              remove_old_indexes(index, current_index)
             end
-            response
-          end
-
-          def config_get(key)
-            execute_command(:config_get, :key => key).body
-          end
-
-          def config_set(key, value)
-            execute_command(:config_set, :key => key, :value => value).body
-          end
-
-          def object_exist?(name)
-            execute_command(:object_exist, :name => name).body
-          end
-
-          def column_rename(table, name, new_name)
-            execute_command(:column_rename,
-                            :table => table,
-                            :name => name,
-                            :new_name => new_name).body
-          end
-
-          def column_list(table)
-            execute_command(:column_list, :table => table)
-          end
-
-          def column_remove(table, column)
-            execute_command(:column_remove,
-                            :table => table,
-                            :name => column)
-          end
-
-          def column_create_similar(table, column_name, base_column_name)
-            info = execute_command(:schema)["#{table}.#{base_column_name}"]
-            arguments = info.command.arguments.merge("name" => column_name)
-            execute_command(:column_create, arguments).body
+            yield if block_given?
+            true
           end
 
           def set_alias(alias_column, alias_name, real_name)
@@ -229,10 +116,9 @@ module Groonga
           def resolve_alias(alias_column, key)
             table, column = alias_column.split(".", 2)
             filter = "_key == #{ScriptSyntax.format_string(key)}"
-            response = execute_command(:select,
-                                       :table => table,
-                                       :filter => filter,
-                                       :output_columns => column)
+            response = select(table,
+                              :filter => filter,
+                              :output_columns => column)
             return nil if response.n_hits.zero?
             response.records.first[column]
           end