groonga-client 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98c07600cda5089240101c97f1055cee06406515
-  data.tar.gz: c26124ec4e3364a316949bb21bb9fbc00cac6661
+  metadata.gz: e5a1481041e35f70cc0bdfc67c42726260d4db93
+  data.tar.gz: '029977cb46c61d06fdb07d8d8ce71498c0375f87'
 SHA512:
-  metadata.gz: d3aeb65b3107c978fdfef99efe64aadc4a5fd7c5ed4a4835c6377d2d1910acb795994e923ac2cd7bed0737d89740999e8a2c50f511ac762b2797593ad0be4e04
-  data.tar.gz: dedc41cf9ce0f590bb557574d310d55a02cff6c033eec799ff30253093e18182c1141a9801eed0d8aee46df5c583b936511560d9905aabe673aaf2abaf3f78ef
+  metadata.gz: 5433daf65d59587e66d23850d27c3bec597e9fc81c501b65ff7ede45e21976d038f0241f757d1817d1b182f2c8d2aa5d4b74e9ff6ed91806c2c0ee7fe6ed9715
+  data.tar.gz: 4e9d0c4d4573587bdc4f0e06979aa5d9ae33bc81b6814a2a909066adc9990377d7ffba132908950645e0d0340a1310dca11ae4da8709516d482154324b91e3c1

data/bin/groonga-client

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # -*- ruby -*-
 #
-# Copyright (C) 2015-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/client/cli"
+require "groonga/client/command-line/groonga-client"
 
-cli = Groonga::Client::CLI.new
-exit(cli.run(ARGV))
+command_line = Groonga::Client::CommandLine::GroongaClient.new
+exit(command_line.run(ARGV))

data/bin/groonga-client-index-recreate

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+#
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/client/command-line/groonga-client-index-recreate"
+
+command_line = Groonga::Client::CommandLine::GroongaClientIndexRecreate.new
+exit(command_line.run(ARGV))

data/doc/text/news.md

@@ -1,5 +1,31 @@
 # NEWS
 
+## 0.5.3 - 2017-10-26
+
+### Improvements
+
+  * `Groonga::Client::Response::ColumnList::Column#flags`: Changed
+    return type to `Array<Strinng>` from `String`.
+
+  * `Groonga::Client::Response::ColumnList::Column#scalar?`: Added.
+
+  * `Groonga::Client::Response::ColumnList::Column#vector?`: Added.
+
+  * `Groonga::Client::Response::ColumnList::Column#index?`: Added.
+
+  * `Groonga::Client::Response::Schema::Command`: Added.
+
+  * `Groonga::Client::Response::Schema::Table#command`: Changed return
+    type to `Hash` to `Groonga::Client::Response::Schema::Command`.
+
+  * `Groonga::Client::Response::Schema::Column#command`: Changed return
+    type to `Hash` to `Groonga::Client::Response::Schema::Command`.
+
+  * `Groonga::Client::Response::Schema#[]`: Added.
+
+  * `groonga-client-index-recreate`: Added a new command that
+    recreates indexes dynamically and safely.
+
 ## 0.5.2 - 2017-09-27
 
 ### Improvements

data/lib/groonga/client/command-line/groonga-client-index-recreate.rb

@@ -0,0 +1,325 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+require "json"
+
+require "groonga/client"
+
+module Groonga
+  class Client
+    module CommandLine
+      class GroongaClientIndexRecreate
+        def initialize
+          @url      = nil
+          @protocol = :http
+          @host     = "localhost"
+          @port     = nil
+
+          @read_timeout = -1
+
+          @interval = :day
+
+          @n_workers = 0
+        end
+
+        def run(argv)
+          target_indexes = parse_command_line(argv)
+
+          Client.open(:url      => @url,
+                      :protocol => @protocol,
+                      :host     => @host,
+                      :port     => @port,
+                      :read_timeout => @read_timeout,
+                      :backend  => :synchronous) do |client|
+            runner = Runner.new(client, @interval, target_indexes)
+            runner.run do
+              @n_workers.times do
+                client.database_unmap
+              end
+            end
+          end
+        end
+
+        private
+        def parse_command_line(argv)
+          parser = OptionParser.new
+          parser.version = VERSION
+          parser.banner += " LEXICON1.INDEX1 LEXICON2.INDEX2 ..."
+
+          parser.separator("")
+          parser.separator("Connection:")
+
+          parser.on("--url=URL",
+                    "URL to connect to Groonga server.",
+                    "If this option is specified,",
+                    "--protocol, --host and --port are ignored.") do |url|
+            @url = url
+          end
+
+          available_protocols = [:http, :gqtp]
+          parser.on("--protocol=PROTOCOL", [:http, :gqtp],
+                    "Protocol to connect to Groonga server.",
+                    "[#{available_protocols.join(", ")}]",
+                    "(#{@protocol})") do |protocol|
+            @protocol = protocol
+          end
+
+          parser.on("--host=HOST",
+                    "Groonga server to be connected.",
+                    "(#{@host})") do |host|
+            @host = host
+          end
+
+          parser.on("--port=PORT", Integer,
+                    "Port number of Groonga server to be connected.",
+                    "(auto)") do |port|
+            @port = port
+          end
+
+          parser.on("--read-timeout=TIMEOUT", Integer,
+                    "Timeout on reading response from Groonga server.",
+                    "You can disable timeout by specifying -1.",
+                    "(#{@read_timeout})") do |timeout|
+            @read_timeout = timeout
+          end
+
+          parser.separator("")
+          parser.separator("Configuration:")
+
+          available_intervals = [:day]
+          parser.on("--interval=INTERVAL", available_intervals,
+                    "Index create interval.",
+                    "[#{available_intervals.join(", ")}]",
+                    "(#{@interval})") do |interval|
+            @interval = interval
+          end
+
+          parser.separator("")
+          parser.separator("groonga-httpd:")
+
+          parser.on("--n-workers=N", Integer,
+                    "The number of groonga-httpd workers.",
+                    "This options is meaningless for groonga -s.",
+                    "(#{@n_workers})") do |n|
+            @n_workers = n
+          end
+
+          target_indexes = parser.parse(argv)
+
+          @port ||= default_port(@protocol)
+
+          target_indexes
+        end
+
+        def default_port(protocol)
+          case protocol
+          when :http
+            10041
+          when :gqtp
+            10043
+          end
+        end
+
+        class Runner
+          def initialize(client, interval, target_indexes)
+            @client = client
+            @interval = interval
+            @target_indexes = target_indexes
+            @now = Time.now
+          end
+
+          def run
+            catch do |tag|
+              @abort_tag = tag
+              alias_column = ensure_alias_column
+              @target_indexes.each do |index|
+                current_index = recreate_index(index, alias_column)
+                remove_old_indexes(index, current_index)
+              end
+              yield if block_given?
+              true
+            end
+          end
+
+          private
+          def abort_run(message)
+            $stderr.puts(message)
+            throw(@abort_tag, false)
+          end
+
+          def execute_command(name, arguments={})
+            response = @client.execute(name, arguments)
+            unless response.success?
+              abort_run("Failed to run #{name}: #{response.inspect}")
+            end
+            response
+          end
+
+          def config_get(key)
+            execute_command(:config_get, :key => key).body
+          end
+
+          def config_set(key, value)
+            execute_command(:config_set, :key => key, :value => value).body
+          end
+
+          def object_exist?(name)
+            execute_command(:object_exist, :name => name).body
+          end
+
+          def column_rename(table, name, new_name)
+            execute_command(:column_rename,
+                            :table => table,
+                            :name => name,
+                            :new_name => new_name).body
+          end
+
+          def column_list(table)
+            execute_command(:column_list, :table => table)
+          end
+
+          def column_remove(table, column)
+            execute_command(:column_remove,
+                            :table => table,
+                            :name => column)
+          end
+
+          def column_create_similar(table, column_name, base_column_name)
+            info = execute_command(:schema)["#{table}.#{base_column_name}"]
+            arguments = info.command.arguments.merge("name" => column_name)
+            execute_command(:column_create, arguments).body
+          end
+
+          def set_alias(alias_column, alias_name, real_name)
+            table, column = alias_column.split(".", 2)
+            values = [
+              {
+                "_key" => alias_name,
+                column => real_name,
+              },
+            ]
+            response = execute_command(:load,
+                                       :table => table,
+                                       :values => JSON.generate(values),
+                                       :command_version => "3",
+                                       :output_errors => "yes")
+            response.errors.each do |error|
+              unless error.return_code.zero?
+                abort_run("Failed to set alias: " +
+                          "<#{alias_name}> -> <#{real_name}>: " +
+                          "#{error.message}(#{error.return_code})")
+              end
+            end
+          end
+
+          def resolve_alias(alias_column, key)
+            table, column = alias_column.split(".", 2)
+            filter = "_key == #{ScriptSyntax.format_string(key)}"
+            response = execute_command(:select,
+                                       :table => table,
+                                       :filter => filter,
+                                       :output_columns => column)
+            return nil if response.n_hits.zero?
+            response.records.first[column]
+          end
+
+          def ensure_alias_column
+            alias_column = config_get("alias.column")
+            if alias_column.empty?
+              table = "Aliases"
+              column = "real_name"
+              alias_column = "#{table}.#{column}"
+              unless object_exist?(table)
+                execute_command(:table_create,
+                                :name => table,
+                                :flags => "TABLE_HASH_KEY",
+                                :key_type => "ShortText")
+              end
+              unless object_exist?(alias_column)
+                execute_command(:column_create,
+                                :table => table,
+                                :name => column,
+                                :flags => "COLUMN_SCALAR",
+                                :type => "ShortText")
+              end
+              config_set("alias.column", alias_column)
+            end
+            alias_column
+          end
+
+          def recreate_index(full_index_name, alias_column)
+            revision = generate_revision
+            table_name, index_name = full_index_name.split(".", 2)
+            real_index_name = "#{index_name}_#{revision}"
+            real_full_index_name = "#{table_name}.#{real_index_name}"
+            if object_exist?(full_index_name)
+              set_alias(alias_column, full_index_name, real_full_index_name)
+              column_rename(table_name, index_name, real_index_name)
+              nil
+            elsif object_exist?(real_full_index_name)
+              nil
+            else
+              full_current_index_name =
+                resolve_alias(alias_column, full_index_name)
+              current_table_name, current_index_name =
+                full_current_index_name.split(".", 2)
+              if current_table_name != table_name
+                abort_run("Different lexicon isn't supported: " +
+                          "<#{full_index_name}> -> <#{full_current_index_name}>")
+              end
+              if current_index_name == real_index_name
+                abort_run("Alias doesn't specify real index column: " +
+                          "<#{full_current_index_name}>")
+              end
+              column_create_similar(table_name,
+                                    real_index_name,
+                                    current_index_name)
+              set_alias(alias_column, full_index_name, real_full_index_name)
+              full_current_index_name
+            end
+          end
+
+          def remove_old_indexes(full_base_index_name, full_current_index_name)
+            return if full_current_index_name.nil?
+
+            table_name, base_index_name = full_base_index_name.split(".", 2)
+            _, current_index_name = full_current_index_name.split(".", 2)
+
+            target_index_columns = column_list(table_name).find_all do |column|
+              column.name.start_with?("#{base_index_name}_") and
+                column.index?
+            end
+            target_index_columns.collect(&:name).sort.each do |index_name|
+              next unless /_(\d{4})(\d{2})(\d{2})\z/ =~ index_name
+              next if index_name >= current_index_name
+              column_remove(table_name, index_name)
+            end
+          end
+
+          def generate_revision
+            case @interval
+            when :day
+              @now.strftime("%Y%m%d")
+            else
+              abort_run("Unsupported revision: #{@interval}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/client/command-line/groonga-client.rb

@@ -0,0 +1,244 @@
+# Copyright (C) 2015-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+require "json"
+require "securerandom"
+
+require "groonga/client"
+
+require "groonga/command/parser"
+
+module Groonga
+  class Client
+    module CommandLine
+      class GroongaClient
+        def initialize
+          @url      = nil
+          @protocol = :http
+          @host     = "localhost"
+          @port     = nil
+
+          @read_timeout = Client::Default::READ_TIMEOUT
+
+          @chunk = false
+
+          @runner_options = {
+            :split_load_chunk_size => 10000,
+            :generate_request_id   => false,
+          }
+        end
+
+        def run(argv)
+          command_file_paths = parse_command_line(argv)
+
+          Client.open(:url      => @url,
+                      :protocol => @protocol,
+                      :host     => @host,
+                      :port     => @port,
+                      :read_timeout => @read_timeout,
+                      :chunk    => @chunk,
+                      :backend  => :synchronous) do |client|
+            runner = Runner.new(client, @runner_options)
+
+            if command_file_paths.empty?
+              $stdin.each_line do |line|
+                runner << line
+              end
+            else
+              command_file_paths.each do |command_file_path|
+                File.open(command_file_path) do |command_file|
+                  last_line = nil
+                  command_file.each_line do |line|
+                    last_line = line
+                    runner << line
+                end
+                  if last_line and !last_line.end_with?("\n")
+                    runner << "\n"
+                  end
+                end
+              end
+            end
+            runner.finish
+          end
+
+          true
+        end
+
+        private
+        def parse_command_line(argv)
+          parser = OptionParser.new
+          parser.version = VERSION
+          parser.banner += " GROONGA_COMMAND_FILE1 GROONGA_COMMAND_FILE2 ..."
+
+          parser.separator("")
+
+          parser.separator("Connection:")
+
+          parser.on("--url=URL",
+                    "URL to connect to Groonga server.",
+                    "If this option is specified,",
+                    "--protocol, --host and --port are ignored.") do |url|
+            @url = url
+          end
+
+          available_protocols = [:http, :gqtp]
+          parser.on("--protocol=PROTOCOL", [:http, :gqtp],
+                    "Protocol to connect to Groonga server.",
+                    "[#{available_protocols.join(", ")}]",
+                    "(#{@protocol})") do |protocol|
+            @protocol = protocol
+          end
+
+          parser.on("--host=HOST",
+                    "Groonga server to be connected.",
+                    "(#{@host})") do |host|
+            @host = host
+          end
+
+          parser.on("--port=PORT", Integer,
+                    "Port number of Groonga server to be connected.",
+                    "(auto)") do |port|
+            @port = port
+          end
+
+          parser.on("--read-timeout=TIMEOUT", Integer,
+                    "Timeout on reading response from Groonga server.",
+                    "You can disable timeout by specifying -1.",
+                    "(#{@read_timeout})") do |timeout|
+            @read_timeout = timeout
+          end
+
+          parser.on("--split-load-chunk-size=SIZE", Integer,
+                    "Split a large load to small loads.",
+                    "Each small load has at most SIZE records.",
+                    "Set 0 to SIZE to disable this feature.",
+                    "(#{@runner_options[:split_load_chunk_size]})") do |size|
+            @runner_options[:split_load_chunk_size] = size
+          end
+
+          parser.on("--[no-]generate-request-id",
+                    "Add auto generated request ID to all commands.",
+                    "(#{@runner_options[:generate_request_id]})") do |boolean|
+            @runner_options[:generate_request_id] = boolean
+          end
+
+          parser.on("--[no-]chunk",
+                    "Use \"Transfer-Encoding: chunked\" for load command.",
+                    "HTTP only.",
+                    "(#{@chunk})") do |boolean|
+            @chunk = boolean
+          end
+
+          command_file_paths = parser.parse(argv)
+
+          @port ||= default_port(@protocol)
+
+          command_file_paths
+        end
+
+        def default_port(protocol)
+          case protocol
+          when :http
+            10041
+          when :gqtp
+            10043
+          end
+        end
+
+        class Runner
+          def initialize(client, options={})
+            @client = client
+            @split_load_chunk_size = options[:split_load_chunk_size] || 10000
+            @generate_request_id   = options[:generate_request_id]
+            @load_values = []
+            @parser = create_command_parser
+          end
+
+          def <<(line)
+            @parser << line
+          end
+
+          def finish
+            @parser.finish
+          end
+
+          private
+          def create_command_parser
+            parser = Groonga::Command::Parser.new
+
+            parser.on_command do |command|
+              run_command(command)
+            end
+
+            parser.on_load_columns do |command, columns|
+              command[:columns] ||= columns.join(",")
+            end
+
+            parser.on_load_value do |command, value|
+              unless command[:values]
+                @load_values << value
+                if @load_values.size == @split_load_chunk_size
+                  consume_load_values(command)
+                end
+              end
+              command.original_source.clear
+            end
+
+            parser.on_load_complete do |command|
+              if command[:values]
+                run_command(command)
+              else
+                consume_load_values(command)
+              end
+            end
+
+            parser
+          end
+
+          def consume_load_values(load_command)
+            return if @load_values.empty?
+
+            values_json = "["
+            @load_values.each_with_index do |value, i|
+              values_json << "," unless i.zero?
+              values_json << "\n"
+              values_json << JSON.generate(value)
+            end
+            values_json << "\n]\n"
+            load_command[:values] = values_json
+            run_command(load_command)
+            @load_values.clear
+            load_command[:values] = nil
+          end
+
+          def run_command(command)
+            command[:request_id] ||= SecureRandom.uuid if @generate_request_id
+            response = @client.execute(command)
+            case command.output_type
+            when :json
+              puts(JSON.pretty_generate([response.header, response.body]))
+            when :xml
+              puts(response.raw)
+            else
+              puts(response.body)
+            end
+          end
+        end
+      end
+    end
+  end
+end