groonga-client 0.6.2 → 0.6.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e58e455931babab20e77e8944bce91b4fcd7dc6aee073afb016fbb127d341a2d
4
- data.tar.gz: 3a5ba751046673d6d32ed7d27b15f2b036740c43dd424741b4e56a176efeab9c
3
+ metadata.gz: 29a5323d61e22fd67047fc6adbf076dfca390ef9fcde76e653d1d52ccff00c55
4
+ data.tar.gz: bcff102d164f18996b61e45e979d63ed4cf812ed1747fe7d454c349eafddbde8
5
5
  SHA512:
6
- metadata.gz: 1e41055f62099dc806883965e9636e7d0be252f40600665e0cc4d53476409f34aaf4bc25fd06e41cfbb79b37bf3a3b562db275b7e4394ca45c3db022ecd9e900
7
- data.tar.gz: '0284a3e869fb7fc1efd8d7048694f043f4d78794c7a8b279a22a5303f7597fa417be7cef9d2bc7391a475aec3b57de8bed7437d20f9663bb29c4d4ea3793cd60'
6
+ metadata.gz: f3bff560db004312b66a7f38bcf07e3a4a28b5078d6fabc413e0ee58b816dee31bd022539baab88d2adfbd57fc85ad27e43c339aa282b602e98efd67059c9595
7
+ data.tar.gz: e943e75c8985175fb96aeeccbe8189c70c6709d4cb4d6635f547b859e27afd2b80abd6ebe6a5c7b108ceeca1be13e1a2c4f1c12126472cd09d489551a0d41cfe
@@ -1,5 +1,20 @@
1
1
  # NEWS
2
2
 
3
+ ## 0.6.3 - 2020-06-08
4
+
5
+ ### Improvements
6
+
7
+ * `groonga-client`:
8
+
9
+ * Added support for `-load-input-type=apache-arrow`.
10
+
11
+ * Added `--load-lock-table`.
12
+
13
+ * `http`: Added support for debugging by
14
+ `GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
15
+
16
+ * Added support for response in Apache Arrow.
17
+
3
18
  ## 0.6.2 - 2019-09-02
4
19
 
5
20
  ### Improvements
@@ -1,7 +1,7 @@
1
1
  # -*- mode: ruby -*-
2
2
  #
3
3
  # Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
4
- # Copyright (C) 2014-2019 Sutou Kouhei <kou@clear-code.com>
4
+ # Copyright (C) 2014-2020 Sutou Kouhei <kou@clear-code.com>
5
5
  #
6
6
  # This library is free software; you can redistribute it and/or
7
7
  # modify it under the terms of the GNU Lesser General Public
@@ -48,7 +48,7 @@ Gem::Specification.new do |spec|
48
48
  end
49
49
 
50
50
  spec.add_runtime_dependency("gqtp", ">= 1.0.4")
51
- spec.add_runtime_dependency("groonga-command", ">= 1.4.5")
51
+ spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
52
52
  spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
53
53
  spec.add_runtime_dependency("hashie")
54
54
 
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2015-2018 Kouhei Sutou <kou@clear-code.com>
1
+ # Copyright (C) 2015-2020 Sutou Kouhei <kou@clear-code.com>
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -19,6 +19,11 @@ require "json"
19
19
  require "pathname"
20
20
  require "securerandom"
21
21
 
22
+ begin
23
+ require "arrow"
24
+ rescue LoadError
25
+ end
26
+
22
27
  require "groonga/command/parser"
23
28
 
24
29
  require "groonga/client"
@@ -30,6 +35,12 @@ module Groonga
30
35
  class GroongaClient
31
36
  def initialize
32
37
  @chunk = false
38
+ @load_input_type = "json"
39
+ @available_load_input_types = ["json"]
40
+ if Object.const_defined?(:Arrow)
41
+ @available_load_input_types << "apache-arrow"
42
+ end
43
+ @load_lock_table = false
33
44
 
34
45
  @runner_options = {
35
46
  :split_load_chunk_size => 10000,
@@ -46,7 +57,9 @@ module Groonga
46
57
  parse_command_line(option_parser)
47
58
  end
48
59
 
49
- parser.open_client(:chunk => @chunk) do |client|
60
+ parser.open_client(:chunk => @chunk,
61
+ :load_input_type => @load_input_type,
62
+ :load_lock_table => @load_lock_table) do |client|
50
63
  runner = Runner.new(client, @runner_options)
51
64
 
52
65
  if command_file_paths.empty?
@@ -99,6 +112,20 @@ module Groonga
99
112
  @runner_options[:split_load_chunk_size] = size
100
113
  end
101
114
 
115
+ parser.on("--load-input-type=TYPE",
116
+ @available_load_input_types,
117
+ "Use TYPE as input type for load.",
118
+ "[#{@available_load_input_types.join(", ")}]",
119
+ "(#{@load_input_types})") do |type|
120
+ @load_input_type = type
121
+ end
122
+
123
+ parser.on("--[no-]load-lock-table",
124
+ "Use lock_table=yes for load.",
125
+ "(#{@load_lock_table})") do |boolean|
126
+ @load_lock_table = boolean
127
+ end
128
+
102
129
  parser.on("--[no-]generate-request-id",
103
130
  "Add auto generated request ID to all commands.",
104
131
  "(#{@runner_options[:generate_request_id]})") do |boolean|
@@ -1,5 +1,5 @@
1
1
  # Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
2
- # Copyright (C) 2013-2016 Kouhei Sutou <kou@clear-code.com>
2
+ # Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
3
3
  #
4
4
  # This library is free software; you can redistribute it and/or
5
5
  # modify it under the terms of the GNU Lesser General Public
@@ -55,9 +55,15 @@ module Groonga
55
55
  @options = options
56
56
  end
57
57
 
58
+ DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
58
59
  def send(command)
59
60
  begin
60
- HTTPClient.start(@url.host, @url.port, start_options) do |http|
61
+ http = HTTPClient.new(@url.host, @url.port)
62
+ http.set_debug_output($stderr) if DEBUG
63
+ start_options.each do |key, value|
64
+ http.__send__("#{key}=", value)
65
+ end
66
+ http.start do
61
67
  http.read_timeout = read_timeout
62
68
  response = send_request(http, command)
63
69
  case response
@@ -141,18 +147,7 @@ module Groonga
141
147
 
142
148
  def send_request(http, command)
143
149
  if command.is_a?(Groonga::Command::Load)
144
- raw_values = command[:values]
145
- command[:values] = nil
146
- path = resolve_path(@url, command.to_uri_format)
147
- command[:values] = raw_values
148
- request = Net::HTTP::Post.new(path, headers)
149
- request.content_type = "application/json"
150
- if @options[:chunk]
151
- request["Transfer-Encoding"] = "chunked"
152
- else
153
- request.content_length = raw_values.bytesize
154
- end
155
- request.body_stream = StringIO.new(raw_values)
150
+ request = prepare_load_request(command)
156
151
  else
157
152
  path = resolve_path(@url, command.to_uri_format)
158
153
  request = Net::HTTP::Get.new(path, headers)
@@ -167,6 +162,42 @@ module Groonga
167
162
  }
168
163
  end
169
164
 
165
+ def prepare_load_request(command)
166
+ path_prefix = command.path_prefix
167
+ command = command.class.new(command.command_name,
168
+ command.arguments,
169
+ [])
170
+ command.path_prefix = path_prefix
171
+ case @options[:load_input_type]
172
+ when "apache-arrow"
173
+ command[:input_type] = "apache-arrow"
174
+ content_type = "application/x-apache-arrow-streaming"
175
+ arrow_table = command.build_arrow_table
176
+ if arrow_table
177
+ buffer = Arrow::ResizableBuffer.new(1024)
178
+ arrow_table.save(buffer, format: :stream)
179
+ body = buffer.data.to_s
180
+ else
181
+ body = ""
182
+ end
183
+ command.arguments.delete(:values)
184
+ else
185
+ content_type = "application/json"
186
+ body = command.arguments.delete(:values)
187
+ end
188
+ command[:lock_table] = "yes" if @options[:load_lock_table]
189
+ path = resolve_path(@url, command.to_uri_format)
190
+ request = Net::HTTP::Post.new(path, headers)
191
+ if @options[:chunk]
192
+ request["Transfer-Encoding"] = "chunked"
193
+ else
194
+ request.content_length = body.bytesize
195
+ end
196
+ request.content_type = content_type
197
+ request.body_stream = StringIO.new(body)
198
+ request
199
+ end
200
+
170
201
  def setup_authentication(request)
171
202
  userinfo = @url.userinfo
172
203
  return if userinfo.nil?
@@ -19,6 +19,10 @@ require "csv"
19
19
  require "rexml/document"
20
20
  require "json"
21
21
 
22
+ begin
23
+ require "arrow"
24
+ rescue LoadError
25
+ end
22
26
  require "hashie"
23
27
 
24
28
  module Groonga
@@ -86,6 +90,9 @@ module Groonga
86
90
  when :tsv
87
91
  header, body = parse_tsv(raw_response)
88
92
  return_code = header["return_code"] if header
93
+ when :arrow, :"apache-arrow"
94
+ header, body = parse_apache_arrow(raw_response)
95
+ return_code = header["return_code"] if header
89
96
  else
90
97
  header = nil
91
98
  body = raw_response
@@ -177,6 +184,40 @@ module Groonga
177
184
  end
178
185
  body
179
186
  end
187
+
188
+ def parse_apache_arrow(response)
189
+ header = nil
190
+ body = nil
191
+ buffer = Arrow::Buffer.new(response)
192
+ Arrow::BufferInputStream.open(buffer) do |input|
193
+ while input.tell < response.bytesize
194
+ reader = Arrow::RecordBatchStreamReader.new(input)
195
+ schema = reader.schema
196
+ record_batches = reader.to_a
197
+ if apache_arrow_metadata?(schema)
198
+ table = Arrow::Table.new(schema, record_batches)
199
+ header = table.each_record.first.to_h
200
+ else
201
+ body = {}
202
+ body["columns"] = schema.fields.collect do |field|
203
+ [field.name, field.data_type.to_s]
204
+ end
205
+ if record_batches.empty?
206
+ records = []
207
+ else
208
+ table = Arrow::Table.new(schema, record_batches)
209
+ records = table.raw_records
210
+ end
211
+ body["records"] = records
212
+ end
213
+ end
214
+ end
215
+ return header, body
216
+ end
217
+
218
+ def apache_arrow_metadata?(schema)
219
+ (schema.metadata || {})["GROONGA:data_type"] == "metadata"
220
+ end
180
221
  end
181
222
 
182
223
  # @return [Groonga::Command] The command for the request.
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Groonga
18
18
  class Client
19
- VERSION = "0.6.2"
19
+ VERSION = "0.6.3"
20
20
  end
21
21
  end
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2013-2019 Sutou Kouhei <kou@clear-code.com>
1
+ # Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
2
2
  # Copyright (C) 2013 Kosuke Asami
3
3
  # Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
4
4
  #
@@ -418,8 +418,8 @@ class TestResponseSelectCommandVersion1 < Test::Unit::TestCase
418
418
  values = {}
419
419
  slices(body).each do |label, slice|
420
420
  drilldowns = {}
421
- slice.drilldowns.each do |label, drilldown|
422
- drilldowns[label] = drilldown.records
421
+ slice.drilldowns.each do |drilldown_label, drilldown|
422
+ drilldowns[drilldown_label] = drilldown.records
423
423
  end
424
424
  values[label] = {
425
425
  records: slice.records,
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2013-2019 Sutou Kouhei <kou@clear-code.com>
1
+ # Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
2
2
  # Copyright (C) 2013 Kosuke Asami
3
3
  # Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
4
4
  #
@@ -383,8 +383,8 @@ class TestResponseSelectCommandVersion3 < Test::Unit::TestCase
383
383
  values = {}
384
384
  slices(body).each do |label, slice|
385
385
  drilldowns = {}
386
- slice.drilldowns.each do |label, drilldown|
387
- drilldowns[label] = drilldown.records
386
+ slice.drilldowns.each do |drilldown_label, drilldown|
387
+ drilldowns[drilldown_label] = drilldown.records
388
388
  end
389
389
  values[label] = {
390
390
  records: slice.records,
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: groonga-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.2
4
+ version: 0.6.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Haruka Yoshihara
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-09-02 00:00:00.000000000 Z
13
+ date: 2020-06-07 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: gqtp
@@ -32,14 +32,14 @@ dependencies:
32
32
  requirements:
33
33
  - - ">="
34
34
  - !ruby/object:Gem::Version
35
- version: 1.4.5
35
+ version: 1.4.7
36
36
  type: :runtime
37
37
  prerelease: false
38
38
  version_requirements: !ruby/object:Gem::Requirement
39
39
  requirements:
40
40
  - - ">="
41
41
  - !ruby/object:Gem::Version
42
- version: 1.4.5
42
+ version: 1.4.7
43
43
  - !ruby/object:Gem::Dependency
44
44
  name: groonga-command-parser
45
45
  requirement: !ruby/object:Gem::Requirement
@@ -161,9 +161,9 @@ email:
161
161
  - kou@clear-code.com
162
162
  - tfortress58@gmail.com
163
163
  executables:
164
- - groonga-client-index-recreate
165
- - groonga-client-index-check
166
164
  - groonga-client
165
+ - groonga-client-index-check
166
+ - groonga-client-index-recreate
167
167
  extensions: []
168
168
  extra_rdoc_files: []
169
169
  files:
@@ -286,43 +286,42 @@ required_rubygems_version: !ruby/object:Gem::Requirement
286
286
  - !ruby/object:Gem::Version
287
287
  version: '0'
288
288
  requirements: []
289
- rubyforge_project:
290
- rubygems_version: 2.7.6.2
289
+ rubygems_version: 3.2.0.pre1
291
290
  signing_key:
292
291
  specification_version: 4
293
292
  summary: Groonga-client is a client for Groonga (http://groonga.org/) implemented
294
293
  with pure Ruby. You can use it without Groonga.
295
294
  test_files:
296
- - test/results/test-table-list.rb
297
- - test/test-script-syntax.rb
298
- - test/response/test-table-create.rb
299
- - test/response/helper.rb
300
- - test/response/test-select-command-version3.rb
301
- - test/response/test-column-list.rb
302
- - test/response/test-select-xml.rb
303
- - test/response/test-load.rb
304
- - test/response/test-select-command-version1.rb
305
- - test/response/test-select-tsv.rb
306
- - test/response/test-table-list.rb
307
- - test/response/test-status.rb
308
- - test/response/test-schema.rb
309
- - test/response/test-base.rb
310
- - test/response/test-error.rb
311
- - test/response/test-table-remove.rb
312
295
  - test/command-line/helper.rb
313
- - test/command-line/test-index-recreate.rb
314
296
  - test/command-line/test-index-check.rb
315
- - test/run-test.rb
316
- - test/test-command.rb
297
+ - test/command-line/test-index-recreate.rb
298
+ - test/protocol/test-gqtp.rb
299
+ - test/protocol/test-http.rb
317
300
  - test/request/select/test-backward-compatible-sort-keys-parameter.rb
318
- - test/request/select/test-values-parameter.rb
319
- - test/request/select/test-scorer.rb
320
301
  - test/request/select/test-filter.rb
321
302
  - test/request/select/test-output-columns-parameter.rb
303
+ - test/request/select/test-scorer.rb
322
304
  - test/request/select/test-sort-keys-parameter.rb
305
+ - test/request/select/test-values-parameter.rb
323
306
  - test/request/test-generic.rb
324
307
  - test/request/test-merger.rb
325
308
  - test/request/test-select.rb
309
+ - test/response/helper.rb
310
+ - test/response/test-base.rb
311
+ - test/response/test-column-list.rb
312
+ - test/response/test-error.rb
313
+ - test/response/test-load.rb
314
+ - test/response/test-schema.rb
315
+ - test/response/test-select-command-version1.rb
316
+ - test/response/test-select-command-version3.rb
317
+ - test/response/test-select-tsv.rb
318
+ - test/response/test-select-xml.rb
319
+ - test/response/test-status.rb
320
+ - test/response/test-table-create.rb
321
+ - test/response/test-table-list.rb
322
+ - test/response/test-table-remove.rb
323
+ - test/results/test-table-list.rb
324
+ - test/run-test.rb
326
325
  - test/test-client.rb
327
- - test/protocol/test-gqtp.rb
328
- - test/protocol/test-http.rb
326
+ - test/test-command.rb
327
+ - test/test-script-syntax.rb