groonga-client 0.5.9 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16493f7abb6c08faa1f10060a64bedca44071acc4772ed06ac400a91a4b96282
-  data.tar.gz: 0c94271fb51d35d2758e423a7d52cacc46488ae1c6afbc9ab947b3093dedcfee
+  metadata.gz: 78a494db54af1981f348843b06df72cfb4d2c09d1dda5095aa75d8b5b05a8cc9
+  data.tar.gz: 1551dc509c115d96a94d70816234b42baef30c7c19bcd74a01e62b595a0633ea
 SHA512:
-  metadata.gz: 8df38010cd75c8e29bd0461243ed0ce07a6cb7efe84450e2eae1e5a8612cd3237be4f00c37bda7a10bbd7016bf5be88507961016337664c988a27ee5c66487b5
-  data.tar.gz: 3a23a2605d9711c4e46582b5a06021a434763d7021a4bd1263be077e605ec1bdcfdfb7fd8d858e2166bd0213ec2feb24961317699a486d33c817182214c3106b
+  metadata.gz: 227087c01d247af0e6e4255fcafda882babc27aef9da37425ef1759b18dc4ec7839bf0d643ee2ac31844ab17f8d67a3643a516b296f04e8f78ca4fa347e00ae1
+  data.tar.gz: 3bd74b528c567aeb7679d59898232a35a622dc5a999df43cdbc6041721a29a4f625de74df70adf5fa0228bb459cd738c4d04a0b946d9bd2eff974b3807875cb4

data/doc/text/news.md

@@ -1,5 +1,59 @@
 # NEWS
 
+## 0.6.4 - 2021-02-24
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+## 0.6.3 - 2020-06-08
+
+### Improvements
+
+  * `groonga-client`:
+
+    * Added support for `-load-input-type=apache-arrow`.
+
+    * Added `--load-lock-table`.
+
+  * `http`: Added support for debugging by
+    `GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
+
+  * Added support for response in Apache Arrow.
+
+## 0.6.2 - 2019-09-02
+
+### Improvements
+
+  * `Groonga::Client::Response::LogicalSelect`: Added.
+
+  * `Groonga::Client::Response::Select#raw_columns`: Added.
+
+  * `Groonga::Client::Response::Select#raw_records`: Added.
+
+  * `Groonga::Client::Response::LogicalRangeFilter`: Added.
+
+## 0.6.1 - 2019-07-27
+
+### Improvements
+
+  * `Groonga::Client::Request::Select`: Added support for drilldowns
+    in slice.
+
+## 0.6.0 - 2018-08-30
+
+### Improvements
+
+  * `groonga-client`:
+
+    * Added `--target-command` option.
+
+    * Added `--target-table` option.
+
+    * Added `--target-column` option.
+
+  * Added support for JSONP.
+
 ## 0.5.9 - 2018-07-13
 
 ### Improvements

data/groonga-client.gemspec

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 #
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2014-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2021  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -48,9 +48,10 @@ Gem::Specification.new do |spec|
   end
 
   spec.add_runtime_dependency("gqtp", ">= 1.0.4")
-  spec.add_runtime_dependency("groonga-command", ">= 1.2.8")
+  spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
   spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
   spec.add_runtime_dependency("hashie")
+  spec.add_runtime_dependency("rexml")
 
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")

data/lib/groonga/client/command-line/groonga-client.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,6 +19,11 @@ require "json"
 require "pathname"
 require "securerandom"
 
+begin
+  require "arrow"
+rescue LoadError
+end
+
 require "groonga/command/parser"
 
 require "groonga/client"
@@ -30,10 +35,19 @@ module Groonga
       class GroongaClient
         def initialize
           @chunk = false
+          @load_input_type = "json"
+          @available_load_input_types = ["json"]
+          if Object.const_defined?(:Arrow)
+            @available_load_input_types << "apache-arrow"
+          end
+          @load_lock_table = false
 
           @runner_options = {
             :split_load_chunk_size => 10000,
             :generate_request_id   => false,
+            :target_commands       => [],
+            :target_tables         => [],
+            :target_columns        => [],
           }
         end
 
@@ -43,7 +57,9 @@ module Groonga
             parse_command_line(option_parser)
           end
 
-          parser.open_client(:chunk => @chunk) do |client|
+          parser.open_client(:chunk => @chunk,
+                             :load_input_type => @load_input_type,
+                             :load_lock_table => @load_lock_table) do |client|
             runner = Runner.new(client, @runner_options)
 
             if command_file_paths.empty?
@@ -96,6 +112,20 @@ module Groonga
             @runner_options[:split_load_chunk_size] = size
           end
 
+          parser.on("--load-input-type=TYPE",
+                    @available_load_input_types,
+                    "Use TYPE as input type for load.",
+                    "[#{@available_load_input_types.join(", ")}]",
+                    "(#{@load_input_types})") do |type|
+            @load_input_type = type
+          end
+
+          parser.on("--[no-]load-lock-table",
+                    "Use lock_table=yes for load.",
+                    "(#{@load_lock_table})") do |boolean|
+            @load_lock_table = boolean
+          end
+
           parser.on("--[no-]generate-request-id",
                     "Add auto generated request ID to all commands.",
                     "(#{@runner_options[:generate_request_id]})") do |boolean|
@@ -108,6 +138,39 @@ module Groonga
                     "(#{@chunk})") do |boolean|
             @chunk = boolean
           end
+
+          parser.on("--target-command=COMMAND",
+                    "Add COMMAND as target commands",
+                    "You can specify multiple times",
+                    "If COMMAND is /.../,",
+                    "it's treated as a regular expression") do |command|
+            add_target(@runner_options[:target_commands], command)
+          end
+
+          parser.on("--target-table=TABLE",
+                    "Add TABLE as target tables",
+                    "You can specify multiple times",
+                    "If TABLE is /.../,",
+                    "it's treated as a regular expression") do |table|
+            add_target(@runner_options[:target_tables], table)
+          end
+
+          parser.on("--target-column=COLUMN",
+                    "Add COLUMN as target columns",
+                    "You can specify multiple times",
+                    "If COLUMN is /.../,",
+                    "it's treated as a regular expression") do |column|
+            add_target(@runner_options[:target_columns], column)
+          end
+        end
+
+        def add_target(targets, target)
+          if /\A\\(.+?)\\(i)?\z/ =~ target
+            pattern = Regexp.new($1, $2 == "i")
+            targets << pattern
+          else
+            targets << target
+          end
         end
 
         class Runner
@@ -115,6 +178,9 @@ module Groonga
             @client = client
             @split_load_chunk_size = options[:split_load_chunk_size] || 10000
             @generate_request_id   = options[:generate_request_id]
+            @target_commands       = options[:target_commands]
+            @target_tables         = options[:target_tables]
+            @target_columns        = options[:target_columns]
             @load_values = []
             @parser = create_command_parser
           end
@@ -188,6 +254,12 @@ module Groonga
           end
 
           def run_command(command)
+            return unless target_command?(command)
+            return unless target_table?(command)
+            return unless target_column?(command)
+
+            command = Marshal.load(Marshal.dump(command))
+            apply_target_columns(command)
             command[:request_id] ||= SecureRandom.uuid if @generate_request_id
             response = @client.execute(command)
             case command.output_type
@@ -199,6 +271,92 @@ module Groonga
               puts(response.body)
             end
           end
+
+          def target_command?(command)
+            return true if @target_commands.empty?
+
+            @target_commands.any? do |name|
+              name === command.command_name
+            end
+          end
+
+          def target_table?(command)
+            return true if @target_tables.empty?
+
+            target = nil
+            case command.command_name
+            when "load", "column_create", "select"
+              target = command.table
+            when "table_create", "table_remove"
+              target = command.name
+            end
+            return true if target.nil?
+
+            @target_tables.any? do |name|
+              name === target
+            end
+          end
+
+          def target_column?(command)
+            return true if @target_columns.empty?
+
+            target = nil
+            case command.command_name
+            when "column_create"
+              target = command.name
+            end
+            return true if target.nil?
+
+            @target_columns.any? do |name|
+              name === target
+            end
+          end
+
+          def apply_target_columns(command)
+            return if @target_columns.empty?
+
+            values = command[:values]
+            return if values.nil?
+
+            command = command.dup
+
+            values = JSON.parse(values)
+            columns = command[:columns]
+            if columns
+              columns = columns.split(/\s*,\s*/)
+              target_indexes = []
+              new_columns = []
+              columns.each_with_index do |column, i|
+                if load_target_column?(column)
+                  target_indexes << i
+                  new_columns << column
+                end
+              end
+              command[:columns] = new_columns.join(",")
+              new_values = values.collect do |value|
+                target_indexes.collect do |i|
+                  value[i]
+                end
+              end
+              command[:values] = JSON.generate(new_values)
+            else
+              new_values = values.collect do |value|
+                new_value = {}
+                value.each do |key, value|
+                  if load_target_column?(key)
+                    new_value[key] = value
+                  end
+                end
+                new_value
+              end
+              command[:values] = JSON.generate(new_values)
+            end
+          end
+
+          def load_target_column?(column)
+            column == "_key" or
+              @target_columns.any? {|name| name === column}
+          end
         end
 
         class BareREPL

data/lib/groonga/client/protocol/http/synchronous.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2013-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -55,9 +55,15 @@ module Groonga
             @options = options
           end
 
+          DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
           def send(command)
             begin
-              HTTPClient.start(@url.host, @url.port, start_options) do |http|
+              http = HTTPClient.new(@url.host, @url.port)
+              http.set_debug_output($stderr) if DEBUG
+              start_options.each do |key, value|
+                http.__send__("#{key}=", value)
+              end
+              http.start do
                 http.read_timeout = read_timeout
                 response = send_request(http, command)
                 case response
@@ -141,18 +147,7 @@ module Groonga
 
           def send_request(http, command)
             if command.is_a?(Groonga::Command::Load)
-              raw_values = command[:values]
-              command[:values] = nil
-              path = resolve_path(@url, command.to_uri_format)
-              command[:values] = raw_values
-              request = Net::HTTP::Post.new(path, headers)
-              request.content_type = "application/json"
-              if @options[:chunk]
-                request["Transfer-Encoding"] = "chunked"
-              else
-                request.content_length = raw_values.bytesize
-              end
-              request.body_stream = StringIO.new(raw_values)
+              request = prepare_load_request(command)
             else
               path = resolve_path(@url, command.to_uri_format)
               request = Net::HTTP::Get.new(path, headers)
@@ -167,6 +162,42 @@ module Groonga
             }
           end
 
+          def prepare_load_request(command)
+            path_prefix = command.path_prefix
+            command = command.class.new(command.command_name,
+                                        command.arguments,
+                                        [])
+            command.path_prefix = path_prefix
+            case @options[:load_input_type]
+            when "apache-arrow"
+              command[:input_type] = "apache-arrow"
+              content_type = "application/x-apache-arrow-streaming"
+              arrow_table = command.build_arrow_table
+              if arrow_table
+                buffer = Arrow::ResizableBuffer.new(1024)
+                arrow_table.save(buffer, format: :stream)
+                body = buffer.data.to_s
+              else
+                body = ""
+              end
+              command.arguments.delete(:values)
+            else
+              content_type = "application/json"
+              body = command.arguments.delete(:values)
+            end
+            command[:lock_table] = "yes" if @options[:load_lock_table]
+            path = resolve_path(@url, command.to_uri_format)
+            request = Net::HTTP::Post.new(path, headers)
+            if @options[:chunk]
+              request["Transfer-Encoding"] = "chunked"
+            else
+              request.content_length = body.bytesize
+            end
+            request.content_type = content_type
+            request.body_stream = StringIO.new(body)
+            request
+          end
+
           def setup_authentication(request)
             userinfo = @url.userinfo
             return if userinfo.nil?

data/lib/groonga/client/response.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -30,6 +30,8 @@ require "groonga/client/response/lock-clear"
 require "groonga/client/response/log-level"
 require "groonga/client/response/log-put"
 require "groonga/client/response/log-reopen"
+require "groonga/client/response/logical-range-filter"
+require "groonga/client/response/logical-select"
 require "groonga/client/response/quit"
 require "groonga/client/response/register"
 require "groonga/client/response/schema"

data/lib/groonga/client/response/base.rb

@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -21,6 +19,10 @@ require "csv"
 require "rexml/document"
 require "json"
 
+begin
+  require "arrow"
+rescue LoadError
+end
 require "hashie"
 
 module Groonga
@@ -67,7 +69,13 @@ module Groonga
             return_code = nil
             case command.output_type
             when :json
-              response = JSON.parse(raw_response)
+              callback = command["callback"]
+              if callback and
+                  /\A#{Regexp.escape(callback)}\((.+)\);\z/ =~ raw_response
+                response = JSON.parse($1)
+              else
+                response = JSON.parse(raw_response)
+              end
               if response.is_a?(::Array)
                 header, body = response
                 return_code = header[0] if header
@@ -82,6 +90,9 @@ module Groonga
             when :tsv
               header, body = parse_tsv(raw_response)
               return_code = header["return_code"] if header
+            when :arrow, :"apache-arrow"
+              header, body = parse_apache_arrow(raw_response)
+              return_code = header["return_code"] if header
             else
               header = nil
               body = raw_response
@@ -173,6 +184,40 @@ module Groonga
             end
             body
           end
+
+          def parse_apache_arrow(response)
+            header = nil
+            body = nil
+            buffer = Arrow::Buffer.new(response)
+            Arrow::BufferInputStream.open(buffer) do |input|
+              while input.tell < response.bytesize
+                reader = Arrow::RecordBatchStreamReader.new(input)
+                schema = reader.schema
+                record_batches = reader.to_a
+                if apache_arrow_metadata?(schema)
+                  table = Arrow::Table.new(schema, record_batches)
+                  header = table.each_record.first.to_h
+                else
+                  body = {}
+                  body["columns"] = schema.fields.collect do |field|
+                    [field.name, field.data_type.to_s]
+                  end
+                  if record_batches.empty?
+                    records = []
+                  else
+                    table = Arrow::Table.new(schema, record_batches)
+                    records = table.raw_records
+                  end
+                  body["records"] = records
+                end
+              end
+            end
+            return header, body
+          end
+
+          def apache_arrow_metadata?(schema)
+            (schema.metadata || {})["GROONGA:data_type"] == "metadata"
+          end
         end
 
         # @return [Groonga::Command] The command for the request.