groonga-client 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e58e455931babab20e77e8944bce91b4fcd7dc6aee073afb016fbb127d341a2d
-  data.tar.gz: 3a5ba751046673d6d32ed7d27b15f2b036740c43dd424741b4e56a176efeab9c
+  metadata.gz: 29a5323d61e22fd67047fc6adbf076dfca390ef9fcde76e653d1d52ccff00c55
+  data.tar.gz: bcff102d164f18996b61e45e979d63ed4cf812ed1747fe7d454c349eafddbde8
 SHA512:
-  metadata.gz: 1e41055f62099dc806883965e9636e7d0be252f40600665e0cc4d53476409f34aaf4bc25fd06e41cfbb79b37bf3a3b562db275b7e4394ca45c3db022ecd9e900
-  data.tar.gz: '0284a3e869fb7fc1efd8d7048694f043f4d78794c7a8b279a22a5303f7597fa417be7cef9d2bc7391a475aec3b57de8bed7437d20f9663bb29c4d4ea3793cd60'
+  metadata.gz: f3bff560db004312b66a7f38bcf07e3a4a28b5078d6fabc413e0ee58b816dee31bd022539baab88d2adfbd57fc85ad27e43c339aa282b602e98efd67059c9595
+  data.tar.gz: e943e75c8985175fb96aeeccbe8189c70c6709d4cb4d6635f547b859e27afd2b80abd6ebe6a5c7b108ceeca1be13e1a2c4f1c12126472cd09d489551a0d41cfe

data/doc/text/news.md

@@ -1,5 +1,20 @@
 # NEWS
 
+## 0.6.3 - 2020-06-08
+
+### Improvements
+
+  * `groonga-client`:
+
+    * Added support for `-load-input-type=apache-arrow`.
+
+    * Added `--load-lock-table`.
+
+  * `http`: Added support for debugging by
+    `GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
+
+  * Added support for response in Apache Arrow.
+
 ## 0.6.2 - 2019-09-02
 
 ### Improvements

data/groonga-client.gemspec

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 #
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2014-2019  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2014-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -48,7 +48,7 @@ Gem::Specification.new do |spec|
   end
 
   spec.add_runtime_dependency("gqtp", ">= 1.0.4")
-  spec.add_runtime_dependency("groonga-command", ">= 1.4.5")
+  spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
   spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
   spec.add_runtime_dependency("hashie")
 

data/lib/groonga/client/command-line/groonga-client.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,6 +19,11 @@ require "json"
 require "pathname"
 require "securerandom"
 
+begin
+  require "arrow"
+rescue LoadError
+end
+
 require "groonga/command/parser"
 
 require "groonga/client"
@@ -30,6 +35,12 @@ module Groonga
       class GroongaClient
         def initialize
           @chunk = false
+          @load_input_type = "json"
+          @available_load_input_types = ["json"]
+          if Object.const_defined?(:Arrow)
+            @available_load_input_types << "apache-arrow"
+          end
+          @load_lock_table = false
 
           @runner_options = {
             :split_load_chunk_size => 10000,
@@ -46,7 +57,9 @@ module Groonga
             parse_command_line(option_parser)
           end
 
-          parser.open_client(:chunk => @chunk) do |client|
+          parser.open_client(:chunk => @chunk,
+                             :load_input_type => @load_input_type,
+                             :load_lock_table => @load_lock_table) do |client|
             runner = Runner.new(client, @runner_options)
 
             if command_file_paths.empty?
@@ -99,6 +112,20 @@ module Groonga
             @runner_options[:split_load_chunk_size] = size
           end
 
+          parser.on("--load-input-type=TYPE",
+                    @available_load_input_types,
+                    "Use TYPE as input type for load.",
+                    "[#{@available_load_input_types.join(", ")}]",
+                    "(#{@load_input_types})") do |type|
+            @load_input_type = type
+          end
+
+          parser.on("--[no-]load-lock-table",
+                    "Use lock_table=yes for load.",
+                    "(#{@load_lock_table})") do |boolean|
+            @load_lock_table = boolean
+          end
+
           parser.on("--[no-]generate-request-id",
                     "Add auto generated request ID to all commands.",
                     "(#{@runner_options[:generate_request_id]})") do |boolean|

data/lib/groonga/client/protocol/http/synchronous.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2013-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -55,9 +55,15 @@ module Groonga
             @options = options
           end
 
+          DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
           def send(command)
             begin
-              HTTPClient.start(@url.host, @url.port, start_options) do |http|
+              http = HTTPClient.new(@url.host, @url.port)
+              http.set_debug_output($stderr) if DEBUG
+              start_options.each do |key, value|
+                http.__send__("#{key}=", value)
+              end
+              http.start do
                 http.read_timeout = read_timeout
                 response = send_request(http, command)
                 case response
@@ -141,18 +147,7 @@ module Groonga
 
           def send_request(http, command)
             if command.is_a?(Groonga::Command::Load)
-              raw_values = command[:values]
-              command[:values] = nil
-              path = resolve_path(@url, command.to_uri_format)
-              command[:values] = raw_values
-              request = Net::HTTP::Post.new(path, headers)
-              request.content_type = "application/json"
-              if @options[:chunk]
-                request["Transfer-Encoding"] = "chunked"
-              else
-                request.content_length = raw_values.bytesize
-              end
-              request.body_stream = StringIO.new(raw_values)
+              request = prepare_load_request(command)
             else
               path = resolve_path(@url, command.to_uri_format)
               request = Net::HTTP::Get.new(path, headers)
@@ -167,6 +162,42 @@ module Groonga
             }
           end
 
+          def prepare_load_request(command)
+            path_prefix = command.path_prefix
+            command = command.class.new(command.command_name,
+                                        command.arguments,
+                                        [])
+            command.path_prefix = path_prefix
+            case @options[:load_input_type]
+            when "apache-arrow"
+              command[:input_type] = "apache-arrow"
+              content_type = "application/x-apache-arrow-streaming"
+              arrow_table = command.build_arrow_table
+              if arrow_table
+                buffer = Arrow::ResizableBuffer.new(1024)
+                arrow_table.save(buffer, format: :stream)
+                body = buffer.data.to_s
+              else
+                body = ""
+              end
+              command.arguments.delete(:values)
+            else
+              content_type = "application/json"
+              body = command.arguments.delete(:values)
+            end
+            command[:lock_table] = "yes" if @options[:load_lock_table]
+            path = resolve_path(@url, command.to_uri_format)
+            request = Net::HTTP::Post.new(path, headers)
+            if @options[:chunk]
+              request["Transfer-Encoding"] = "chunked"
+            else
+              request.content_length = body.bytesize
+            end
+            request.content_type = content_type
+            request.body_stream = StringIO.new(body)
+            request
+          end
+
           def setup_authentication(request)
             userinfo = @url.userinfo
             return if userinfo.nil?

data/lib/groonga/client/response/base.rb

@@ -19,6 +19,10 @@ require "csv"
 require "rexml/document"
 require "json"
 
+begin
+  require "arrow"
+rescue LoadError
+end
 require "hashie"
 
 module Groonga
@@ -86,6 +90,9 @@ module Groonga
             when :tsv
               header, body = parse_tsv(raw_response)
               return_code = header["return_code"] if header
+            when :arrow, :"apache-arrow"
+              header, body = parse_apache_arrow(raw_response)
+              return_code = header["return_code"] if header
             else
               header = nil
               body = raw_response
@@ -177,6 +184,40 @@ module Groonga
             end
             body
           end
+
+          def parse_apache_arrow(response)
+            header = nil
+            body = nil
+            buffer = Arrow::Buffer.new(response)
+            Arrow::BufferInputStream.open(buffer) do |input|
+              while input.tell < response.bytesize
+                reader = Arrow::RecordBatchStreamReader.new(input)
+                schema = reader.schema
+                record_batches = reader.to_a
+                if apache_arrow_metadata?(schema)
+                  table = Arrow::Table.new(schema, record_batches)
+                  header = table.each_record.first.to_h
+                else
+                  body = {}
+                  body["columns"] = schema.fields.collect do |field|
+                    [field.name, field.data_type.to_s]
+                  end
+                  if record_batches.empty?
+                    records = []
+                  else
+                    table = Arrow::Table.new(schema, record_batches)
+                    records = table.raw_records
+                  end
+                  body["records"] = records
+                end
+              end
+            end
+            return header, body
+          end
+
+          def apache_arrow_metadata?(schema)
+            (schema.metadata || {})["GROONGA:data_type"] == "metadata"
+          end
         end
 
         # @return [Groonga::Command] The command for the request.

data/lib/groonga/client/version.rb

@@ -16,6 +16,6 @@
 
 module Groonga
   class Client
-    VERSION = "0.6.2"
+    VERSION = "0.6.3"
   end
 end

data/test/response/test-select-command-version1.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2019  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2013  Kosuke Asami
 # Copyright (C) 2016  Masafumi Yokoyama <yokoyama@clear-code.com>
 #
@@ -418,8 +418,8 @@ class TestResponseSelectCommandVersion1 < Test::Unit::TestCase
         values = {}
         slices(body).each do |label, slice|
           drilldowns = {}
-          slice.drilldowns.each do |label, drilldown|
-            drilldowns[label] = drilldown.records
+          slice.drilldowns.each do |drilldown_label, drilldown|
+            drilldowns[drilldown_label] = drilldown.records
           end
           values[label] = {
             records: slice.records,

data/test/response/test-select-command-version3.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2019  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2013  Kosuke Asami
 # Copyright (C) 2016  Masafumi Yokoyama <yokoyama@clear-code.com>
 #
@@ -383,8 +383,8 @@ class TestResponseSelectCommandVersion3 < Test::Unit::TestCase
         values = {}
         slices(body).each do |label, slice|
           drilldowns = {}
-          slice.drilldowns.each do |label, drilldown|
-            drilldowns[label] = drilldown.records
+          slice.drilldowns.each do |drilldown_label, drilldown|
+            drilldowns[drilldown_label] = drilldown.records
           end
           values[label] = {
             records: slice.records,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: groonga-client
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Haruka Yoshihara
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-02 00:00:00.000000000 Z
+date: 2020-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gqtp
@@ -32,14 +32,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.5
+        version: 1.4.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.5
+        version: 1.4.7
 - !ruby/object:Gem::Dependency
   name: groonga-command-parser
   requirement: !ruby/object:Gem::Requirement
@@ -161,9 +161,9 @@ email:
 - kou@clear-code.com
 - tfortress58@gmail.com
 executables:
-- groonga-client-index-recreate
-- groonga-client-index-check
 - groonga-client
+- groonga-client-index-check
+- groonga-client-index-recreate
 extensions: []
 extra_rdoc_files: []
 files:
@@ -286,43 +286,42 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.2.0.pre1
 signing_key: 
 specification_version: 4
 summary: Groonga-client is a client for Groonga (http://groonga.org/) implemented
   with pure Ruby. You can use it without Groonga.
 test_files:
-- test/results/test-table-list.rb
-- test/test-script-syntax.rb
-- test/response/test-table-create.rb
-- test/response/helper.rb
-- test/response/test-select-command-version3.rb
-- test/response/test-column-list.rb
-- test/response/test-select-xml.rb
-- test/response/test-load.rb
-- test/response/test-select-command-version1.rb
-- test/response/test-select-tsv.rb
-- test/response/test-table-list.rb
-- test/response/test-status.rb
-- test/response/test-schema.rb
-- test/response/test-base.rb
-- test/response/test-error.rb
-- test/response/test-table-remove.rb
 - test/command-line/helper.rb
-- test/command-line/test-index-recreate.rb
 - test/command-line/test-index-check.rb
-- test/run-test.rb
-- test/test-command.rb
+- test/command-line/test-index-recreate.rb
+- test/protocol/test-gqtp.rb
+- test/protocol/test-http.rb
 - test/request/select/test-backward-compatible-sort-keys-parameter.rb
-- test/request/select/test-values-parameter.rb
-- test/request/select/test-scorer.rb
 - test/request/select/test-filter.rb
 - test/request/select/test-output-columns-parameter.rb
+- test/request/select/test-scorer.rb
 - test/request/select/test-sort-keys-parameter.rb
+- test/request/select/test-values-parameter.rb
 - test/request/test-generic.rb
 - test/request/test-merger.rb
 - test/request/test-select.rb
+- test/response/helper.rb
+- test/response/test-base.rb
+- test/response/test-column-list.rb
+- test/response/test-error.rb
+- test/response/test-load.rb
+- test/response/test-schema.rb
+- test/response/test-select-command-version1.rb
+- test/response/test-select-command-version3.rb
+- test/response/test-select-tsv.rb
+- test/response/test-select-xml.rb
+- test/response/test-status.rb
+- test/response/test-table-create.rb
+- test/response/test-table-list.rb
+- test/response/test-table-remove.rb
+- test/results/test-table-list.rb
+- test/run-test.rb
 - test/test-client.rb
-- test/protocol/test-gqtp.rb
-- test/protocol/test-http.rb
+- test/test-command.rb
+- test/test-script-syntax.rb