groonga-client 0.6.2 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/news.md +35 -0
- data/groonga-client.gemspec +3 -2
- data/lib/groonga/client/command-line/groonga-client.rb +33 -189
- data/lib/groonga/client/command-processor.rb +211 -0
- data/lib/groonga/client/protocol/http/synchronous.rb +45 -14
- data/lib/groonga/client/request/select.rb +1 -1
- data/lib/groonga/client/response/base.rb +41 -0
- data/lib/groonga/client/version.rb +2 -2
- data/lib/groonga/client.rb +4 -3
- data/test/request/test-select.rb +2 -2
- data/test/response/test-select-command-version1.rb +3 -3
- data/test/response/test-select-command-version3.rb +3 -3
- data/test/test-command.rb +7 -6
- metadata +45 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad8ac6fe851f63509ad13e5c8abb1541ff2f47631764b86bc23ffc85719ae56a
|
4
|
+
data.tar.gz: 558bc6f1e08568791ca791a3be107b9f075864096bf003ca9cb37dc273a7479b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9bde0f05e3d9a17f545bacad7cd6c91099ea65e1371d015860b5f18d714b31d875ba615b21ac6c8bba15333ce7f760b673c6ecf31e8daa8ebc726b79f53618a8
|
7
|
+
data.tar.gz: e992b6e14dac0073485a909886c128acca19af991a9d45dc89a57ed1557754a9ab5fe38edd687bcbd86a571dc0155264752b671cf1861e2353c9a12f40309507
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,40 @@
|
|
1
1
|
# NEWS
|
2
2
|
|
3
|
+
## 0.6.6 - 2021-12-10
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* `Groonga::Client::CommandProcessor`: Extracted Groonga command processor
|
8
|
+
from `groonga-client` command implementation.
|
9
|
+
|
10
|
+
## 0.6.5 - 2021-03-15
|
11
|
+
|
12
|
+
### Fixes
|
13
|
+
|
14
|
+
* `Groonga::Client::Request::Select::LabeledDrilldown#keys`: Fixed a
|
15
|
+
bug that this uses wrong parameter name.
|
16
|
+
|
17
|
+
## 0.6.4 - 2021-02-24
|
18
|
+
|
19
|
+
### Improvements
|
20
|
+
|
21
|
+
* Added support for Ruby 3.0.
|
22
|
+
|
23
|
+
## 0.6.3 - 2020-06-08
|
24
|
+
|
25
|
+
### Improvements
|
26
|
+
|
27
|
+
* `groonga-client`:
|
28
|
+
|
29
|
+
* Added support for `-load-input-type=apache-arrow`.
|
30
|
+
|
31
|
+
* Added `--load-lock-table`.
|
32
|
+
|
33
|
+
* `http`: Added support for debugging by
|
34
|
+
`GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
|
35
|
+
|
36
|
+
* Added support for response in Apache Arrow.
|
37
|
+
|
3
38
|
## 0.6.2 - 2019-09-02
|
4
39
|
|
5
40
|
### Improvements
|
data/groonga-client.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- mode: ruby -*-
|
2
2
|
#
|
3
3
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
4
|
-
# Copyright (C) 2014-
|
4
|
+
# Copyright (C) 2014-2021 Sutou Kouhei <kou@clear-code.com>
|
5
5
|
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
7
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -48,9 +48,10 @@ Gem::Specification.new do |spec|
|
|
48
48
|
end
|
49
49
|
|
50
50
|
spec.add_runtime_dependency("gqtp", ">= 1.0.4")
|
51
|
-
spec.add_runtime_dependency("groonga-command", ">= 1.4.
|
51
|
+
spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
|
52
52
|
spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
|
53
53
|
spec.add_runtime_dependency("hashie")
|
54
|
+
spec.add_runtime_dependency("rexml")
|
54
55
|
|
55
56
|
spec.add_development_dependency("bundler")
|
56
57
|
spec.add_development_dependency("rake")
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2015-
|
1
|
+
# Copyright (C) 2015-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -19,6 +19,11 @@ require "json"
|
|
19
19
|
require "pathname"
|
20
20
|
require "securerandom"
|
21
21
|
|
22
|
+
begin
|
23
|
+
require "arrow"
|
24
|
+
rescue LoadError
|
25
|
+
end
|
26
|
+
|
22
27
|
require "groonga/command/parser"
|
23
28
|
|
24
29
|
require "groonga/client"
|
@@ -30,6 +35,12 @@ module Groonga
|
|
30
35
|
class GroongaClient
|
31
36
|
def initialize
|
32
37
|
@chunk = false
|
38
|
+
@load_input_type = "json"
|
39
|
+
@available_load_input_types = ["json"]
|
40
|
+
if Object.const_defined?(:Arrow)
|
41
|
+
@available_load_input_types << "apache-arrow"
|
42
|
+
end
|
43
|
+
@load_lock_table = false
|
33
44
|
|
34
45
|
@runner_options = {
|
35
46
|
:split_load_chunk_size => 10000,
|
@@ -46,7 +57,9 @@ module Groonga
|
|
46
57
|
parse_command_line(option_parser)
|
47
58
|
end
|
48
59
|
|
49
|
-
parser.open_client(:chunk => @chunk
|
60
|
+
parser.open_client(:chunk => @chunk,
|
61
|
+
:load_input_type => @load_input_type,
|
62
|
+
:load_lock_table => @load_lock_table) do |client|
|
50
63
|
runner = Runner.new(client, @runner_options)
|
51
64
|
|
52
65
|
if command_file_paths.empty?
|
@@ -60,22 +73,11 @@ module Groonga
|
|
60
73
|
end
|
61
74
|
repl.run
|
62
75
|
else
|
63
|
-
$stdin
|
64
|
-
runner << line
|
65
|
-
end
|
76
|
+
runner.consume($stdin)
|
66
77
|
end
|
67
78
|
else
|
68
79
|
command_file_paths.each do |command_file_path|
|
69
|
-
|
70
|
-
last_line = nil
|
71
|
-
command_file.each_line do |line|
|
72
|
-
last_line = line
|
73
|
-
runner << line
|
74
|
-
end
|
75
|
-
if last_line and !last_line.end_with?("\n")
|
76
|
-
runner << "\n"
|
77
|
-
end
|
78
|
-
end
|
80
|
+
runner.load(command_file_path)
|
79
81
|
end
|
80
82
|
end
|
81
83
|
runner.finish
|
@@ -99,6 +101,20 @@ module Groonga
|
|
99
101
|
@runner_options[:split_load_chunk_size] = size
|
100
102
|
end
|
101
103
|
|
104
|
+
parser.on("--load-input-type=TYPE",
|
105
|
+
@available_load_input_types,
|
106
|
+
"Use TYPE as input type for load.",
|
107
|
+
"[#{@available_load_input_types.join(", ")}]",
|
108
|
+
"(#{@load_input_types})") do |type|
|
109
|
+
@load_input_type = type
|
110
|
+
end
|
111
|
+
|
112
|
+
parser.on("--[no-]load-lock-table",
|
113
|
+
"Use lock_table=yes for load.",
|
114
|
+
"(#{@load_lock_table})") do |boolean|
|
115
|
+
@load_lock_table = boolean
|
116
|
+
end
|
117
|
+
|
102
118
|
parser.on("--[no-]generate-request-id",
|
103
119
|
"Add auto generated request ID to all commands.",
|
104
120
|
"(#{@runner_options[:generate_request_id]})") do |boolean|
|
@@ -146,95 +162,9 @@ module Groonga
|
|
146
162
|
end
|
147
163
|
end
|
148
164
|
|
149
|
-
class Runner
|
150
|
-
def initialize(client, options={})
|
151
|
-
@client = client
|
152
|
-
@split_load_chunk_size = options[:split_load_chunk_size] || 10000
|
153
|
-
@generate_request_id = options[:generate_request_id]
|
154
|
-
@target_commands = options[:target_commands]
|
155
|
-
@target_tables = options[:target_tables]
|
156
|
-
@target_columns = options[:target_columns]
|
157
|
-
@load_values = []
|
158
|
-
@parser = create_command_parser
|
159
|
-
end
|
160
|
-
|
161
|
-
def <<(line)
|
162
|
-
@parser << line
|
163
|
-
end
|
164
|
-
|
165
|
-
def finish
|
166
|
-
@parser.finish
|
167
|
-
end
|
168
|
-
|
169
|
-
def repl
|
170
|
-
begin
|
171
|
-
require "readline"
|
172
|
-
rescue LoadError
|
173
|
-
repl = BareREPL.new(self)
|
174
|
-
else
|
175
|
-
repl = ReadlineREPL.new(self)
|
176
|
-
repl_readline
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
165
|
+
class Runner < CommandProcessor
|
180
166
|
private
|
181
|
-
def
|
182
|
-
parser = Groonga::Command::Parser.new
|
183
|
-
|
184
|
-
parser.on_command do |command|
|
185
|
-
run_command(command)
|
186
|
-
end
|
187
|
-
|
188
|
-
parser.on_load_columns do |command, columns|
|
189
|
-
command[:columns] ||= columns.join(",")
|
190
|
-
end
|
191
|
-
|
192
|
-
parser.on_load_value do |command, value|
|
193
|
-
unless command[:values]
|
194
|
-
@load_values << value
|
195
|
-
if @load_values.size == @split_load_chunk_size
|
196
|
-
consume_load_values(command)
|
197
|
-
end
|
198
|
-
end
|
199
|
-
command.original_source.clear
|
200
|
-
end
|
201
|
-
|
202
|
-
parser.on_load_complete do |command|
|
203
|
-
if command[:values]
|
204
|
-
run_command(command)
|
205
|
-
else
|
206
|
-
consume_load_values(command)
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
|
-
parser
|
211
|
-
end
|
212
|
-
|
213
|
-
def consume_load_values(load_command)
|
214
|
-
return if @load_values.empty?
|
215
|
-
|
216
|
-
values_json = "["
|
217
|
-
@load_values.each_with_index do |value, i|
|
218
|
-
values_json << "," unless i.zero?
|
219
|
-
values_json << "\n"
|
220
|
-
values_json << JSON.generate(value)
|
221
|
-
end
|
222
|
-
values_json << "\n]\n"
|
223
|
-
load_command[:values] = values_json
|
224
|
-
run_command(load_command)
|
225
|
-
@load_values.clear
|
226
|
-
load_command[:values] = nil
|
227
|
-
end
|
228
|
-
|
229
|
-
def run_command(command)
|
230
|
-
return unless target_command?(command)
|
231
|
-
return unless target_table?(command)
|
232
|
-
return unless target_column?(command)
|
233
|
-
|
234
|
-
command = Marshal.load(Marshal.dump(command))
|
235
|
-
apply_target_columns(command)
|
236
|
-
command[:request_id] ||= SecureRandom.uuid if @generate_request_id
|
237
|
-
response = @client.execute(command)
|
167
|
+
def process_response(response, command)
|
238
168
|
case command.output_type
|
239
169
|
when :json
|
240
170
|
puts(JSON.pretty_generate([response.header, response.body]))
|
@@ -244,92 +174,6 @@ module Groonga
|
|
244
174
|
puts(response.body)
|
245
175
|
end
|
246
176
|
end
|
247
|
-
|
248
|
-
def target_command?(command)
|
249
|
-
return true if @target_commands.empty?
|
250
|
-
|
251
|
-
@target_commands.any? do |name|
|
252
|
-
name === command.command_name
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
def target_table?(command)
|
257
|
-
return true if @target_tables.empty?
|
258
|
-
|
259
|
-
target = nil
|
260
|
-
case command.command_name
|
261
|
-
when "load", "column_create", "select"
|
262
|
-
target = command.table
|
263
|
-
when "table_create", "table_remove"
|
264
|
-
target = command.name
|
265
|
-
end
|
266
|
-
return true if target.nil?
|
267
|
-
|
268
|
-
@target_tables.any? do |name|
|
269
|
-
name === target
|
270
|
-
end
|
271
|
-
end
|
272
|
-
|
273
|
-
def target_column?(command)
|
274
|
-
return true if @target_columns.empty?
|
275
|
-
|
276
|
-
target = nil
|
277
|
-
case command.command_name
|
278
|
-
when "column_create"
|
279
|
-
target = command.name
|
280
|
-
end
|
281
|
-
return true if target.nil?
|
282
|
-
|
283
|
-
@target_columns.any? do |name|
|
284
|
-
name === target
|
285
|
-
end
|
286
|
-
end
|
287
|
-
|
288
|
-
def apply_target_columns(command)
|
289
|
-
return if @target_columns.empty?
|
290
|
-
|
291
|
-
values = command[:values]
|
292
|
-
return if values.nil?
|
293
|
-
|
294
|
-
command = command.dup
|
295
|
-
|
296
|
-
values = JSON.parse(values)
|
297
|
-
columns = command[:columns]
|
298
|
-
if columns
|
299
|
-
columns = columns.split(/\s*,\s*/)
|
300
|
-
target_indexes = []
|
301
|
-
new_columns = []
|
302
|
-
columns.each_with_index do |column, i|
|
303
|
-
if load_target_column?(column)
|
304
|
-
target_indexes << i
|
305
|
-
new_columns << column
|
306
|
-
end
|
307
|
-
end
|
308
|
-
command[:columns] = new_columns.join(",")
|
309
|
-
new_values = values.collect do |value|
|
310
|
-
target_indexes.collect do |i|
|
311
|
-
value[i]
|
312
|
-
end
|
313
|
-
end
|
314
|
-
command[:values] = JSON.generate(new_values)
|
315
|
-
else
|
316
|
-
new_values = values.collect do |value|
|
317
|
-
new_value = {}
|
318
|
-
value.each do |key, value|
|
319
|
-
if load_target_column?(key)
|
320
|
-
new_value[key] = value
|
321
|
-
end
|
322
|
-
end
|
323
|
-
new_value
|
324
|
-
end
|
325
|
-
command[:values] = JSON.generate(new_values)
|
326
|
-
end
|
327
|
-
end
|
328
|
-
|
329
|
-
def load_target_column?(column)
|
330
|
-
column == "_key" or
|
331
|
-
@target_columns.any? {|name| name === column}
|
332
|
-
end
|
333
177
|
end
|
334
178
|
|
335
179
|
class BareREPL
|
@@ -0,0 +1,211 @@
|
|
1
|
+
# Copyright (C) 2015-2021 Sutou Kouhei <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require "json"
|
18
|
+
|
19
|
+
require "groonga/command/parser"
|
20
|
+
|
21
|
+
module Groonga
|
22
|
+
class Client
|
23
|
+
class CommandProcessor
|
24
|
+
def initialize(client, options={})
|
25
|
+
@client = client
|
26
|
+
@split_load_chunk_size = options[:split_load_chunk_size] || 10000
|
27
|
+
@generate_request_id = options[:generate_request_id]
|
28
|
+
@target_commands = options[:target_commands]
|
29
|
+
@target_tables = options[:target_tables]
|
30
|
+
@target_columns = options[:target_columns]
|
31
|
+
@load_values = []
|
32
|
+
@parser = create_command_parser
|
33
|
+
end
|
34
|
+
|
35
|
+
def <<(line)
|
36
|
+
@parser << line
|
37
|
+
end
|
38
|
+
|
39
|
+
def finish
|
40
|
+
@parser.finish
|
41
|
+
end
|
42
|
+
|
43
|
+
def consume(input)
|
44
|
+
last_line = nil
|
45
|
+
input.each_line do |line|
|
46
|
+
last_line = line
|
47
|
+
@parser << line
|
48
|
+
end
|
49
|
+
if last_line and last_line.end_with?("\n")
|
50
|
+
@parser << "\n"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def load(path)
|
55
|
+
File.open(path) do |input|
|
56
|
+
consume(input)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
def create_command_parser
|
62
|
+
parser = Groonga::Command::Parser.new
|
63
|
+
|
64
|
+
parser.on_command do |command|
|
65
|
+
run_command(command)
|
66
|
+
end
|
67
|
+
|
68
|
+
parser.on_load_columns do |command, columns|
|
69
|
+
command[:columns] ||= columns.join(",")
|
70
|
+
end
|
71
|
+
|
72
|
+
parser.on_load_value do |command, value|
|
73
|
+
unless command[:values]
|
74
|
+
@load_values << value
|
75
|
+
if @load_values.size == @split_load_chunk_size
|
76
|
+
consume_load_values(command)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
command.original_source.clear
|
80
|
+
end
|
81
|
+
|
82
|
+
parser.on_load_complete do |command|
|
83
|
+
if command[:values]
|
84
|
+
run_command(command)
|
85
|
+
else
|
86
|
+
consume_load_values(command)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
parser
|
91
|
+
end
|
92
|
+
|
93
|
+
def consume_load_values(load_command)
|
94
|
+
return if @load_values.empty?
|
95
|
+
|
96
|
+
values_json = "["
|
97
|
+
@load_values.each_with_index do |value, i|
|
98
|
+
values_json << "," unless i.zero?
|
99
|
+
values_json << "\n"
|
100
|
+
values_json << JSON.generate(value)
|
101
|
+
end
|
102
|
+
values_json << "\n]\n"
|
103
|
+
load_command[:values] = values_json
|
104
|
+
run_command(load_command)
|
105
|
+
@load_values.clear
|
106
|
+
load_command[:values] = nil
|
107
|
+
end
|
108
|
+
|
109
|
+
def run_command(command)
|
110
|
+
return unless target_command?(command)
|
111
|
+
return unless target_table?(command)
|
112
|
+
return unless target_column?(command)
|
113
|
+
|
114
|
+
command = Marshal.load(Marshal.dump(command))
|
115
|
+
apply_target_columns(command)
|
116
|
+
command[:request_id] ||= SecureRandom.uuid if @generate_request_id
|
117
|
+
response = @client.execute(command)
|
118
|
+
process_response(response, command)
|
119
|
+
end
|
120
|
+
|
121
|
+
def process_response(response, command)
|
122
|
+
end
|
123
|
+
|
124
|
+
def target_command?(command)
|
125
|
+
return true if @target_commands.empty?
|
126
|
+
|
127
|
+
@target_commands.any? do |name|
|
128
|
+
name === command.command_name
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def target_table?(command)
|
133
|
+
return true if @target_tables.empty?
|
134
|
+
|
135
|
+
target = nil
|
136
|
+
case command.command_name
|
137
|
+
when "load", "column_create", "select"
|
138
|
+
target = command.table
|
139
|
+
when "table_create", "table_remove"
|
140
|
+
target = command.name
|
141
|
+
end
|
142
|
+
return true if target.nil?
|
143
|
+
|
144
|
+
@target_tables.any? do |name|
|
145
|
+
name === target
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def target_column?(command)
|
150
|
+
return true if @target_columns.empty?
|
151
|
+
|
152
|
+
target = nil
|
153
|
+
case command.command_name
|
154
|
+
when "column_create"
|
155
|
+
target = command.name
|
156
|
+
end
|
157
|
+
return true if target.nil?
|
158
|
+
|
159
|
+
@target_columns.any? do |name|
|
160
|
+
name === target
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def apply_target_columns(command)
|
165
|
+
return if @target_columns.empty?
|
166
|
+
|
167
|
+
values = command[:values]
|
168
|
+
return if values.nil?
|
169
|
+
|
170
|
+
command = command.dup
|
171
|
+
|
172
|
+
values = JSON.parse(values)
|
173
|
+
columns = command[:columns]
|
174
|
+
if columns
|
175
|
+
columns = columns.split(/\s*,\s*/)
|
176
|
+
target_indexes = []
|
177
|
+
new_columns = []
|
178
|
+
columns.each_with_index do |column, i|
|
179
|
+
if load_target_column?(column)
|
180
|
+
target_indexes << i
|
181
|
+
new_columns << column
|
182
|
+
end
|
183
|
+
end
|
184
|
+
command[:columns] = new_columns.join(",")
|
185
|
+
new_values = values.collect do |value|
|
186
|
+
target_indexes.collect do |i|
|
187
|
+
value[i]
|
188
|
+
end
|
189
|
+
end
|
190
|
+
command[:values] = JSON.generate(new_values)
|
191
|
+
else
|
192
|
+
new_values = values.collect do |value|
|
193
|
+
new_value = {}
|
194
|
+
value.each do |key, value|
|
195
|
+
if load_target_column?(key)
|
196
|
+
new_value[key] = value
|
197
|
+
end
|
198
|
+
end
|
199
|
+
new_value
|
200
|
+
end
|
201
|
+
command[:values] = JSON.generate(new_values)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def load_target_column?(column)
|
206
|
+
column == "_key" or
|
207
|
+
@target_columns.any? {|name| name === column}
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
2
|
-
# Copyright (C) 2013-
|
2
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
3
3
|
#
|
4
4
|
# This library is free software; you can redistribute it and/or
|
5
5
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -55,9 +55,15 @@ module Groonga
|
|
55
55
|
@options = options
|
56
56
|
end
|
57
57
|
|
58
|
+
DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
|
58
59
|
def send(command)
|
59
60
|
begin
|
60
|
-
HTTPClient.
|
61
|
+
http = HTTPClient.new(@url.host, @url.port)
|
62
|
+
http.set_debug_output($stderr) if DEBUG
|
63
|
+
start_options.each do |key, value|
|
64
|
+
http.__send__("#{key}=", value)
|
65
|
+
end
|
66
|
+
http.start do
|
61
67
|
http.read_timeout = read_timeout
|
62
68
|
response = send_request(http, command)
|
63
69
|
case response
|
@@ -141,18 +147,7 @@ module Groonga
|
|
141
147
|
|
142
148
|
def send_request(http, command)
|
143
149
|
if command.is_a?(Groonga::Command::Load)
|
144
|
-
|
145
|
-
command[:values] = nil
|
146
|
-
path = resolve_path(@url, command.to_uri_format)
|
147
|
-
command[:values] = raw_values
|
148
|
-
request = Net::HTTP::Post.new(path, headers)
|
149
|
-
request.content_type = "application/json"
|
150
|
-
if @options[:chunk]
|
151
|
-
request["Transfer-Encoding"] = "chunked"
|
152
|
-
else
|
153
|
-
request.content_length = raw_values.bytesize
|
154
|
-
end
|
155
|
-
request.body_stream = StringIO.new(raw_values)
|
150
|
+
request = prepare_load_request(command)
|
156
151
|
else
|
157
152
|
path = resolve_path(@url, command.to_uri_format)
|
158
153
|
request = Net::HTTP::Get.new(path, headers)
|
@@ -167,6 +162,42 @@ module Groonga
|
|
167
162
|
}
|
168
163
|
end
|
169
164
|
|
165
|
+
def prepare_load_request(command)
|
166
|
+
path_prefix = command.path_prefix
|
167
|
+
command = command.class.new(command.command_name,
|
168
|
+
command.arguments,
|
169
|
+
[])
|
170
|
+
command.path_prefix = path_prefix
|
171
|
+
case @options[:load_input_type]
|
172
|
+
when "apache-arrow"
|
173
|
+
command[:input_type] = "apache-arrow"
|
174
|
+
content_type = "application/x-apache-arrow-streaming"
|
175
|
+
arrow_table = command.build_arrow_table
|
176
|
+
if arrow_table
|
177
|
+
buffer = Arrow::ResizableBuffer.new(1024)
|
178
|
+
arrow_table.save(buffer, format: :stream)
|
179
|
+
body = buffer.data.to_s
|
180
|
+
else
|
181
|
+
body = ""
|
182
|
+
end
|
183
|
+
command.arguments.delete(:values)
|
184
|
+
else
|
185
|
+
content_type = "application/json"
|
186
|
+
body = command.arguments.delete(:values)
|
187
|
+
end
|
188
|
+
command[:lock_table] = "yes" if @options[:load_lock_table]
|
189
|
+
path = resolve_path(@url, command.to_uri_format)
|
190
|
+
request = Net::HTTP::Post.new(path, headers)
|
191
|
+
if @options[:chunk]
|
192
|
+
request["Transfer-Encoding"] = "chunked"
|
193
|
+
else
|
194
|
+
request.content_length = body.bytesize
|
195
|
+
end
|
196
|
+
request.content_type = content_type
|
197
|
+
request.body_stream = StringIO.new(body)
|
198
|
+
request
|
199
|
+
end
|
200
|
+
|
170
201
|
def setup_authentication(request)
|
171
202
|
userinfo = @url.userinfo
|
172
203
|
return if userinfo.nil?
|
@@ -19,6 +19,10 @@ require "csv"
|
|
19
19
|
require "rexml/document"
|
20
20
|
require "json"
|
21
21
|
|
22
|
+
begin
|
23
|
+
require "arrow"
|
24
|
+
rescue LoadError
|
25
|
+
end
|
22
26
|
require "hashie"
|
23
27
|
|
24
28
|
module Groonga
|
@@ -86,6 +90,9 @@ module Groonga
|
|
86
90
|
when :tsv
|
87
91
|
header, body = parse_tsv(raw_response)
|
88
92
|
return_code = header["return_code"] if header
|
93
|
+
when :arrow, :"apache-arrow"
|
94
|
+
header, body = parse_apache_arrow(raw_response)
|
95
|
+
return_code = header["return_code"] if header
|
89
96
|
else
|
90
97
|
header = nil
|
91
98
|
body = raw_response
|
@@ -177,6 +184,40 @@ module Groonga
|
|
177
184
|
end
|
178
185
|
body
|
179
186
|
end
|
187
|
+
|
188
|
+
def parse_apache_arrow(response)
|
189
|
+
header = nil
|
190
|
+
body = nil
|
191
|
+
buffer = Arrow::Buffer.new(response)
|
192
|
+
Arrow::BufferInputStream.open(buffer) do |input|
|
193
|
+
while input.tell < response.bytesize
|
194
|
+
reader = Arrow::RecordBatchStreamReader.new(input)
|
195
|
+
schema = reader.schema
|
196
|
+
record_batches = reader.to_a
|
197
|
+
if apache_arrow_metadata?(schema)
|
198
|
+
table = Arrow::Table.new(schema, record_batches)
|
199
|
+
header = table.each_record.first.to_h
|
200
|
+
else
|
201
|
+
body = {}
|
202
|
+
body["columns"] = schema.fields.collect do |field|
|
203
|
+
[field.name, field.data_type.to_s]
|
204
|
+
end
|
205
|
+
if record_batches.empty?
|
206
|
+
records = []
|
207
|
+
else
|
208
|
+
table = Arrow::Table.new(schema, record_batches)
|
209
|
+
records = table.raw_records
|
210
|
+
end
|
211
|
+
body["records"] = records
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
return header, body
|
216
|
+
end
|
217
|
+
|
218
|
+
def apache_arrow_metadata?(schema)
|
219
|
+
(schema.metadata || {})["GROONGA:data_type"] == "metadata"
|
220
|
+
end
|
180
221
|
end
|
181
222
|
|
182
223
|
# @return [Groonga::Command] The command for the request.
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2021 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -16,6 +16,6 @@
|
|
16
16
|
|
17
17
|
module Groonga
|
18
18
|
class Client
|
19
|
-
VERSION = "0.6.
|
19
|
+
VERSION = "0.6.6"
|
20
20
|
end
|
21
21
|
end
|
data/lib/groonga/client.rb
CHANGED
@@ -18,12 +18,13 @@
|
|
18
18
|
require "uri"
|
19
19
|
require "json"
|
20
20
|
|
21
|
-
require "groonga/client/default"
|
22
21
|
require "groonga/client/command"
|
22
|
+
require "groonga/client/command-processor"
|
23
|
+
require "groonga/client/default"
|
23
24
|
require "groonga/client/empty-request"
|
25
|
+
require "groonga/client/protocol/file"
|
24
26
|
require "groonga/client/protocol/gqtp"
|
25
27
|
require "groonga/client/protocol/http"
|
26
|
-
require "groonga/client/protocol/file"
|
27
28
|
require "groonga/client/request"
|
28
29
|
|
29
30
|
module Groonga
|
@@ -202,7 +203,7 @@ module Groonga
|
|
202
203
|
end
|
203
204
|
|
204
205
|
def method_missing(name, *args, &block)
|
205
|
-
if groonga_command_name?(name)
|
206
|
+
if groonga_command_name?(name) and args.size <= 1
|
206
207
|
execute(name, *args, &block)
|
207
208
|
else
|
208
209
|
super
|
data/test/request/test-select.rb
CHANGED
@@ -186,8 +186,8 @@ class TestRequestSelect < Test::Unit::TestCase
|
|
186
186
|
end
|
187
187
|
|
188
188
|
sub_test_case("#paginate") do
|
189
|
-
def paginate(*args)
|
190
|
-
@request.paginate(*args).to_parameters
|
189
|
+
def paginate(*args, **kwargs)
|
190
|
+
@request.paginate(*args, **kwargs).to_parameters
|
191
191
|
end
|
192
192
|
|
193
193
|
sub_test_case("page") do
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
# Copyright (C) 2013 Kosuke Asami
|
3
3
|
# Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
|
4
4
|
#
|
@@ -418,8 +418,8 @@ class TestResponseSelectCommandVersion1 < Test::Unit::TestCase
|
|
418
418
|
values = {}
|
419
419
|
slices(body).each do |label, slice|
|
420
420
|
drilldowns = {}
|
421
|
-
slice.drilldowns.each do |
|
422
|
-
drilldowns[
|
421
|
+
slice.drilldowns.each do |drilldown_label, drilldown|
|
422
|
+
drilldowns[drilldown_label] = drilldown.records
|
423
423
|
end
|
424
424
|
values[label] = {
|
425
425
|
records: slice.records,
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
# Copyright (C) 2013 Kosuke Asami
|
3
3
|
# Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
|
4
4
|
#
|
@@ -383,8 +383,8 @@ class TestResponseSelectCommandVersion3 < Test::Unit::TestCase
|
|
383
383
|
values = {}
|
384
384
|
slices(body).each do |label, slice|
|
385
385
|
drilldowns = {}
|
386
|
-
slice.drilldowns.each do |
|
387
|
-
drilldowns[
|
386
|
+
slice.drilldowns.each do |drilldown_label, drilldown|
|
387
|
+
drilldowns[drilldown_label] = drilldown.records
|
388
388
|
end
|
389
389
|
values[label] = {
|
390
390
|
records: slice.records,
|
data/test/test-command.rb
CHANGED
@@ -5,7 +5,8 @@ class TestCommand < Test::Unit::TestCase
|
|
5
5
|
|
6
6
|
def test_column_create
|
7
7
|
response = Object.new
|
8
|
-
mock(@client).execute(:column_create,
|
8
|
+
mock(@client).execute(:column_create,
|
9
|
+
{table: :Test, name: :Body, type: :ShortText}) do
|
9
10
|
response
|
10
11
|
end
|
11
12
|
@client.column_create(:table => :Test, :name => :Body, :type => :ShortText)
|
@@ -13,7 +14,7 @@ class TestCommand < Test::Unit::TestCase
|
|
13
14
|
|
14
15
|
def test_column_list
|
15
16
|
response = Object.new
|
16
|
-
mock(@client).execute(:column_list, :
|
17
|
+
mock(@client).execute(:column_list, {table: :Test}) do
|
17
18
|
response
|
18
19
|
end
|
19
20
|
@client.column_list(:table => :Test)
|
@@ -27,7 +28,7 @@ class TestCommand < Test::Unit::TestCase
|
|
27
28
|
}
|
28
29
|
]
|
29
30
|
response = Object.new
|
30
|
-
mock(@client).execute(:load, :
|
31
|
+
mock(@client).execute(:load, {table: :Test, values: values.to_json}) do
|
31
32
|
response
|
32
33
|
end
|
33
34
|
@client.load(:table => :Test, :values => values.to_json)
|
@@ -35,7 +36,7 @@ class TestCommand < Test::Unit::TestCase
|
|
35
36
|
|
36
37
|
def test_select
|
37
38
|
response = Object.new
|
38
|
-
mock(@client).execute(:select, :
|
39
|
+
mock(@client).execute(:select, {table: :Test}) do
|
39
40
|
response
|
40
41
|
end
|
41
42
|
@client.select(:table => :Test)
|
@@ -43,7 +44,7 @@ class TestCommand < Test::Unit::TestCase
|
|
43
44
|
|
44
45
|
def test_table_create
|
45
46
|
response = Object.new
|
46
|
-
mock(@client).execute(:table_create, :
|
47
|
+
mock(@client).execute(:table_create, {name: :Test}) do
|
47
48
|
response
|
48
49
|
end
|
49
50
|
@client.table_create(:name => :Test)
|
@@ -58,7 +59,7 @@ class TestCommand < Test::Unit::TestCase
|
|
58
59
|
end
|
59
60
|
|
60
61
|
def test_table_remove
|
61
|
-
mock(@client).execute(:table_remove, :
|
62
|
+
mock(@client).execute(:table_remove, {name: "Test"})
|
62
63
|
@client.table_remove(:name => "Test")
|
63
64
|
end
|
64
65
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: groonga-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Haruka Yoshihara
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-12-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: gqtp
|
@@ -32,14 +32,14 @@ dependencies:
|
|
32
32
|
requirements:
|
33
33
|
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
|
-
version: 1.4.
|
35
|
+
version: 1.4.7
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version: 1.4.
|
42
|
+
version: 1.4.7
|
43
43
|
- !ruby/object:Gem::Dependency
|
44
44
|
name: groonga-command-parser
|
45
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -68,6 +68,20 @@ dependencies:
|
|
68
68
|
- - ">="
|
69
69
|
- !ruby/object:Gem::Version
|
70
70
|
version: '0'
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: rexml
|
73
|
+
requirement: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
type: :runtime
|
79
|
+
prerelease: false
|
80
|
+
version_requirements: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
71
85
|
- !ruby/object:Gem::Dependency
|
72
86
|
name: bundler
|
73
87
|
requirement: !ruby/object:Gem::Requirement
|
@@ -161,9 +175,9 @@ email:
|
|
161
175
|
- kou@clear-code.com
|
162
176
|
- tfortress58@gmail.com
|
163
177
|
executables:
|
164
|
-
- groonga-client-index-recreate
|
165
|
-
- groonga-client-index-check
|
166
178
|
- groonga-client
|
179
|
+
- groonga-client-index-check
|
180
|
+
- groonga-client-index-recreate
|
167
181
|
extensions: []
|
168
182
|
extra_rdoc_files: []
|
169
183
|
files:
|
@@ -183,6 +197,7 @@ files:
|
|
183
197
|
- lib/groonga/client/command-line/groonga-client.rb
|
184
198
|
- lib/groonga/client/command-line/parser.rb
|
185
199
|
- lib/groonga/client/command-line/runner.rb
|
200
|
+
- lib/groonga/client/command-processor.rb
|
186
201
|
- lib/groonga/client/command.rb
|
187
202
|
- lib/groonga/client/default.rb
|
188
203
|
- lib/groonga/client/empty-request.rb
|
@@ -286,43 +301,42 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
286
301
|
- !ruby/object:Gem::Version
|
287
302
|
version: '0'
|
288
303
|
requirements: []
|
289
|
-
|
290
|
-
rubygems_version: 2.7.6.2
|
304
|
+
rubygems_version: 3.3.0.dev
|
291
305
|
signing_key:
|
292
306
|
specification_version: 4
|
293
307
|
summary: Groonga-client is a client for Groonga (http://groonga.org/) implemented
|
294
308
|
with pure Ruby. You can use it without Groonga.
|
295
309
|
test_files:
|
296
|
-
- test/results/test-table-list.rb
|
297
|
-
- test/test-script-syntax.rb
|
298
|
-
- test/response/test-table-create.rb
|
299
|
-
- test/response/helper.rb
|
300
|
-
- test/response/test-select-command-version3.rb
|
301
|
-
- test/response/test-column-list.rb
|
302
|
-
- test/response/test-select-xml.rb
|
303
|
-
- test/response/test-load.rb
|
304
|
-
- test/response/test-select-command-version1.rb
|
305
|
-
- test/response/test-select-tsv.rb
|
306
|
-
- test/response/test-table-list.rb
|
307
|
-
- test/response/test-status.rb
|
308
|
-
- test/response/test-schema.rb
|
309
|
-
- test/response/test-base.rb
|
310
|
-
- test/response/test-error.rb
|
311
|
-
- test/response/test-table-remove.rb
|
312
310
|
- test/command-line/helper.rb
|
313
|
-
- test/command-line/test-index-recreate.rb
|
314
311
|
- test/command-line/test-index-check.rb
|
315
|
-
- test/
|
316
|
-
- test/test-
|
312
|
+
- test/command-line/test-index-recreate.rb
|
313
|
+
- test/protocol/test-gqtp.rb
|
314
|
+
- test/protocol/test-http.rb
|
317
315
|
- test/request/select/test-backward-compatible-sort-keys-parameter.rb
|
318
|
-
- test/request/select/test-values-parameter.rb
|
319
|
-
- test/request/select/test-scorer.rb
|
320
316
|
- test/request/select/test-filter.rb
|
321
317
|
- test/request/select/test-output-columns-parameter.rb
|
318
|
+
- test/request/select/test-scorer.rb
|
322
319
|
- test/request/select/test-sort-keys-parameter.rb
|
320
|
+
- test/request/select/test-values-parameter.rb
|
323
321
|
- test/request/test-generic.rb
|
324
322
|
- test/request/test-merger.rb
|
325
323
|
- test/request/test-select.rb
|
324
|
+
- test/response/helper.rb
|
325
|
+
- test/response/test-base.rb
|
326
|
+
- test/response/test-column-list.rb
|
327
|
+
- test/response/test-error.rb
|
328
|
+
- test/response/test-load.rb
|
329
|
+
- test/response/test-schema.rb
|
330
|
+
- test/response/test-select-command-version1.rb
|
331
|
+
- test/response/test-select-command-version3.rb
|
332
|
+
- test/response/test-select-tsv.rb
|
333
|
+
- test/response/test-select-xml.rb
|
334
|
+
- test/response/test-status.rb
|
335
|
+
- test/response/test-table-create.rb
|
336
|
+
- test/response/test-table-list.rb
|
337
|
+
- test/response/test-table-remove.rb
|
338
|
+
- test/results/test-table-list.rb
|
339
|
+
- test/run-test.rb
|
326
340
|
- test/test-client.rb
|
327
|
-
- test/
|
328
|
-
- test/
|
341
|
+
- test/test-command.rb
|
342
|
+
- test/test-script-syntax.rb
|