groonga-client 0.6.0 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eaec3aab0752df6b5832fe16cb3c79d107103636b06a603ecb2e453bd6e29bec
-  data.tar.gz: cc03ea25d9b87599a6fc10cc553716c3bd7e0b885b2887f7a414b0377490e37d
+  metadata.gz: 0c6eb335d9ea4011c1e5b3681dfd8e9252796b95d24c802df1a769757821b8bf
+  data.tar.gz: 19824fe10455c126d229c1af171a8483ee67b15df08df78a9b8ec43dc0e661cd
 SHA512:
-  metadata.gz: 16a4c67ddfa8bb7f401ff0cd5f49d852f3f279966c3839379ae0ed0f29629b8402ae7b14a87087f263440b8ee0365eb646688a8a3ad7b1cc9979d9ddd3bc61d1
-  data.tar.gz: 86c7edc062f853a4d56e18b0b43477f6317630471de314d4952ab8dd7b9a85146180ce8d5e93ce7f687e0345ec6a26f938846409b3ad3bdcd45677ec4e0029f2
+  metadata.gz: b48b94c52345192fbb20ffbde0d450cd819ca97097b4b0a549ed285645f7a85278392ce51abefb29b055c8505aca5574789e0312fc81b6a7615eeed261e57933
+  data.tar.gz: 2d58742abe16488dbcfcde075d9102b3988f27b58e8c398a60ba75406d0b17e11ac2ebf8d708c7ebe1b9e7b1a13adad0a59fdabaf646cd376c3463925f7f9f5c

data/doc/text/news.md

@@ -1,5 +1,52 @@
 # NEWS
 
+## 0.6.5 - 2021-03-15
+
+### Fixes
+
+  * `Groonga::Client::Request::Select::LabeledDrilldown#keys`: Fixed a
+    bug that this uses wrong parameter name.
+
+## 0.6.4 - 2021-02-24
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+## 0.6.3 - 2020-06-08
+
+### Improvements
+
+  * `groonga-client`:
+
+    * Added support for `-load-input-type=apache-arrow`.
+
+    * Added `--load-lock-table`.
+
+  * `http`: Added support for debugging by
+    `GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
+
+  * Added support for response in Apache Arrow.
+
+## 0.6.2 - 2019-09-02
+
+### Improvements
+
+  * `Groonga::Client::Response::LogicalSelect`: Added.
+
+  * `Groonga::Client::Response::Select#raw_columns`: Added.
+
+  * `Groonga::Client::Response::Select#raw_records`: Added.
+
+  * `Groonga::Client::Response::LogicalRangeFilter`: Added.
+
+## 0.6.1 - 2019-07-27
+
+### Improvements
+
+  * `Groonga::Client::Request::Select`: Added support for drilldowns
+    in slice.
+
 ## 0.6.0 - 2018-08-30
 
 ### Improvements

data/groonga-client.gemspec

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 #
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2014-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2021  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -48,9 +48,10 @@ Gem::Specification.new do |spec|
   end
 
   spec.add_runtime_dependency("gqtp", ">= 1.0.4")
-  spec.add_runtime_dependency("groonga-command", ">= 1.2.8")
+  spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
   spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
   spec.add_runtime_dependency("hashie")
+  spec.add_runtime_dependency("rexml")
 
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")

data/lib/groonga/client/command-line/groonga-client.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,6 +19,11 @@ require "json"
 require "pathname"
 require "securerandom"
 
+begin
+  require "arrow"
+rescue LoadError
+end
+
 require "groonga/command/parser"
 
 require "groonga/client"
@@ -30,6 +35,12 @@ module Groonga
       class GroongaClient
         def initialize
           @chunk = false
+          @load_input_type = "json"
+          @available_load_input_types = ["json"]
+          if Object.const_defined?(:Arrow)
+            @available_load_input_types << "apache-arrow"
+          end
+          @load_lock_table = false
 
           @runner_options = {
             :split_load_chunk_size => 10000,
@@ -46,7 +57,9 @@ module Groonga
             parse_command_line(option_parser)
           end
 
-          parser.open_client(:chunk => @chunk) do |client|
+          parser.open_client(:chunk => @chunk,
+                             :load_input_type => @load_input_type,
+                             :load_lock_table => @load_lock_table) do |client|
             runner = Runner.new(client, @runner_options)
 
             if command_file_paths.empty?
@@ -99,6 +112,20 @@ module Groonga
             @runner_options[:split_load_chunk_size] = size
           end
 
+          parser.on("--load-input-type=TYPE",
+                    @available_load_input_types,
+                    "Use TYPE as input type for load.",
+                    "[#{@available_load_input_types.join(", ")}]",
+                    "(#{@load_input_types})") do |type|
+            @load_input_type = type
+          end
+
+          parser.on("--[no-]load-lock-table",
+                    "Use lock_table=yes for load.",
+                    "(#{@load_lock_table})") do |boolean|
+            @load_lock_table = boolean
+          end
+
           parser.on("--[no-]generate-request-id",
                     "Add auto generated request ID to all commands.",
                     "(#{@runner_options[:generate_request_id]})") do |boolean|

data/lib/groonga/client/protocol/http/synchronous.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2013-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -55,9 +55,15 @@ module Groonga
             @options = options
           end
 
+          DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
           def send(command)
             begin
-              HTTPClient.start(@url.host, @url.port, start_options) do |http|
+              http = HTTPClient.new(@url.host, @url.port)
+              http.set_debug_output($stderr) if DEBUG
+              start_options.each do |key, value|
+                http.__send__("#{key}=", value)
+              end
+              http.start do
                 http.read_timeout = read_timeout
                 response = send_request(http, command)
                 case response
@@ -141,18 +147,7 @@ module Groonga
 
           def send_request(http, command)
             if command.is_a?(Groonga::Command::Load)
-              raw_values = command[:values]
-              command[:values] = nil
-              path = resolve_path(@url, command.to_uri_format)
-              command[:values] = raw_values
-              request = Net::HTTP::Post.new(path, headers)
-              request.content_type = "application/json"
-              if @options[:chunk]
-                request["Transfer-Encoding"] = "chunked"
-              else
-                request.content_length = raw_values.bytesize
-              end
-              request.body_stream = StringIO.new(raw_values)
+              request = prepare_load_request(command)
             else
               path = resolve_path(@url, command.to_uri_format)
               request = Net::HTTP::Get.new(path, headers)
@@ -167,6 +162,42 @@ module Groonga
             }
           end
 
+          def prepare_load_request(command)
+            path_prefix = command.path_prefix
+            command = command.class.new(command.command_name,
+                                        command.arguments,
+                                        [])
+            command.path_prefix = path_prefix
+            case @options[:load_input_type]
+            when "apache-arrow"
+              command[:input_type] = "apache-arrow"
+              content_type = "application/x-apache-arrow-streaming"
+              arrow_table = command.build_arrow_table
+              if arrow_table
+                buffer = Arrow::ResizableBuffer.new(1024)
+                arrow_table.save(buffer, format: :stream)
+                body = buffer.data.to_s
+              else
+                body = ""
+              end
+              command.arguments.delete(:values)
+            else
+              content_type = "application/json"
+              body = command.arguments.delete(:values)
+            end
+            command[:lock_table] = "yes" if @options[:load_lock_table]
+            path = resolve_path(@url, command.to_uri_format)
+            request = Net::HTTP::Post.new(path, headers)
+            if @options[:chunk]
+              request["Transfer-Encoding"] = "chunked"
+            else
+              request.content_length = body.bytesize
+            end
+            request.content_type = content_type
+            request.body_stream = StringIO.new(body)
+            request
+          end
+
           def setup_authentication(request)
             userinfo = @url.userinfo
             return if userinfo.nil?

data/lib/groonga/client/request/select.rb

@@ -595,7 +595,7 @@ module Groonga
           end
 
           def keys(values)
-            @request.values_parameter([:"#{prefix}keys"], values)
+            @request.values_parameter(:"#{prefix}keys", values)
           end
 
           def sort_keys(value)

data/lib/groonga/client/response.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2013  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -30,6 +30,8 @@ require "groonga/client/response/lock-clear"
 require "groonga/client/response/log-level"
 require "groonga/client/response/log-put"
 require "groonga/client/response/log-reopen"
+require "groonga/client/response/logical-range-filter"
+require "groonga/client/response/logical-select"
 require "groonga/client/response/quit"
 require "groonga/client/response/register"
 require "groonga/client/response/schema"

data/lib/groonga/client/response/base.rb

@@ -19,6 +19,10 @@ require "csv"
 require "rexml/document"
 require "json"
 
+begin
+  require "arrow"
+rescue LoadError
+end
 require "hashie"
 
 module Groonga
@@ -86,6 +90,9 @@ module Groonga
             when :tsv
               header, body = parse_tsv(raw_response)
               return_code = header["return_code"] if header
+            when :arrow, :"apache-arrow"
+              header, body = parse_apache_arrow(raw_response)
+              return_code = header["return_code"] if header
             else
               header = nil
               body = raw_response
@@ -177,6 +184,40 @@ module Groonga
             end
             body
           end
+
+          def parse_apache_arrow(response)
+            header = nil
+            body = nil
+            buffer = Arrow::Buffer.new(response)
+            Arrow::BufferInputStream.open(buffer) do |input|
+              while input.tell < response.bytesize
+                reader = Arrow::RecordBatchStreamReader.new(input)
+                schema = reader.schema
+                record_batches = reader.to_a
+                if apache_arrow_metadata?(schema)
+                  table = Arrow::Table.new(schema, record_batches)
+                  header = table.each_record.first.to_h
+                else
+                  body = {}
+                  body["columns"] = schema.fields.collect do |field|
+                    [field.name, field.data_type.to_s]
+                  end
+                  if record_batches.empty?
+                    records = []
+                  else
+                    table = Arrow::Table.new(schema, record_batches)
+                    records = table.raw_records
+                  end
+                  body["records"] = records
+                end
+              end
+            end
+            return header, body
+          end
+
+          def apache_arrow_metadata?(schema)
+            (schema.metadata || {})["GROONGA:data_type"] == "metadata"
+          end
         end
 
         # @return [Groonga::Command] The command for the request.

data/lib/groonga/client/response/drilldownable.rb

@@ -0,0 +1,85 @@
+# Copyright (C) 2019  Sutou Kouhei <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Groonga
+  class Client
+    module Response
+      module Drilldownable
+        # @return [::Array<Groonga::Client::Response::Select::Drilldown>,
+        #          ::Hash<String, Groonga::Client::Response::Select::Drilldown>]
+        #   If labeled drilldowns are used or command version 3 or
+        #   later is used, `{"label1" => drilldown1, "label2" => drilldown2}`
+        #   is returned since 0.3.1.
+        #
+        #   Otherwise, `[drilldown1, drilldown2]` is returned.
+        attr_accessor :drilldowns
+
+        private
+        def parse_drilldown(label, keys, raw_drilldown)
+          if raw_drilldown.is_a?(::Array)
+            n_hits = raw_drilldown[0][0]
+            raw_columns = raw_drilldown[1]
+            raw_records = raw_drilldown[2..-1]
+          else
+            n_hits = raw_drilldown["n_hits"]
+            raw_columns = raw_drilldown["columns"]
+            raw_records = raw_drilldown["records"]
+          end
+          records = parse_records(raw_columns, raw_records)
+          Drilldown.new(label,
+                        keys,
+                        n_hits,
+                        records,
+                        raw_columns,
+                        raw_records)
+        end
+
+        def parse_drilldowns(keys, raw_drilldowns)
+          (raw_drilldowns || []).collect.with_index do |raw_drilldown, i|
+            key = keys[i]
+            parse_drilldown(key, [key], raw_drilldown)
+          end
+        end
+
+        def parse_labeled_drilldowns(labeled_drilldown_requests,
+                                     raw_drilldowns)
+          drilldowns = {}
+          (raw_drilldowns || {}).each do |label, raw_drilldown|
+            labeled_drilldown_request = labeled_drilldown_requests[label]
+            drilldowns[label] = parse_drilldown(label,
+                                                labeled_drilldown_request.keys,
+                                                raw_drilldown)
+          end
+          drilldowns
+        end
+
+        class Drilldown < Struct.new(:label,
+                                     :keys,
+                                     :n_hits,
+                                     :records,
+                                     :raw_columns,
+                                     :raw_records)
+          # @deprecated since 0.2.6. Use {#records} instead.
+          alias_method :items, :records
+
+          def key
+            keys.join(", ")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/client/response/logical-range-filter.rb

@@ -0,0 +1,52 @@
+# Copyright (C) 2019  Sutou Kouhei <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/client/response/base"
+require "groonga/client/response/searchable"
+
+module Groonga
+  class Client
+    module Response
+      class LogicalRangeFilter < Base
+        Response.register("logical_range_filter", self)
+
+        include Searchable
+
+        attr_accessor :records
+
+        def body=(body)
+          super(body)
+          parse_body(body)
+        end
+
+        private
+        def parse_body(body)
+          if body.is_a?(::Array)
+            @raw_columns, *@raw_records = body.first
+            @raw_records ||= []
+            @records = parse_records(raw_columns, raw_records)
+          else
+            @raw_columns = body["columns"]
+            @raw_records = body["records"] || []
+          end
+          @records = parse_records(@raw_columns, @raw_records)
+          body
+        end
+      end
+    end
+  end
+end
+