groonga-client 0.6.2 → 0.6.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/doc/text/news.md +35 -0
- data/groonga-client.gemspec +3 -2
- data/lib/groonga/client/command-line/groonga-client.rb +33 -189
- data/lib/groonga/client/command-processor.rb +211 -0
- data/lib/groonga/client/protocol/http/synchronous.rb +45 -14
- data/lib/groonga/client/request/select.rb +1 -1
- data/lib/groonga/client/response/base.rb +41 -0
- data/lib/groonga/client/version.rb +2 -2
- data/lib/groonga/client.rb +4 -3
- data/test/request/test-select.rb +2 -2
- data/test/response/test-select-command-version1.rb +3 -3
- data/test/response/test-select-command-version3.rb +3 -3
- data/test/test-command.rb +7 -6
- metadata +45 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad8ac6fe851f63509ad13e5c8abb1541ff2f47631764b86bc23ffc85719ae56a
|
4
|
+
data.tar.gz: 558bc6f1e08568791ca791a3be107b9f075864096bf003ca9cb37dc273a7479b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9bde0f05e3d9a17f545bacad7cd6c91099ea65e1371d015860b5f18d714b31d875ba615b21ac6c8bba15333ce7f760b673c6ecf31e8daa8ebc726b79f53618a8
|
7
|
+
data.tar.gz: e992b6e14dac0073485a909886c128acca19af991a9d45dc89a57ed1557754a9ab5fe38edd687bcbd86a571dc0155264752b671cf1861e2353c9a12f40309507
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,40 @@
|
|
1
1
|
# NEWS
|
2
2
|
|
3
|
+
## 0.6.6 - 2021-12-10
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* `Groonga::Client::CommandProcessor`: Extracted Groonga command processor
|
8
|
+
from `groonga-client` command implementation.
|
9
|
+
|
10
|
+
## 0.6.5 - 2021-03-15
|
11
|
+
|
12
|
+
### Fixes
|
13
|
+
|
14
|
+
* `Groonga::Client::Request::Select::LabeledDrilldown#keys`: Fixed a
|
15
|
+
bug that this uses wrong parameter name.
|
16
|
+
|
17
|
+
## 0.6.4 - 2021-02-24
|
18
|
+
|
19
|
+
### Improvements
|
20
|
+
|
21
|
+
* Added support for Ruby 3.0.
|
22
|
+
|
23
|
+
## 0.6.3 - 2020-06-08
|
24
|
+
|
25
|
+
### Improvements
|
26
|
+
|
27
|
+
* `groonga-client`:
|
28
|
+
|
29
|
+
* Added support for `-load-input-type=apache-arrow`.
|
30
|
+
|
31
|
+
* Added `--load-lock-table`.
|
32
|
+
|
33
|
+
* `http`: Added support for debugging by
|
34
|
+
`GROONGA_CLIENT_HTTP_DEBUG=yes` environment variable.
|
35
|
+
|
36
|
+
* Added support for response in Apache Arrow.
|
37
|
+
|
3
38
|
## 0.6.2 - 2019-09-02
|
4
39
|
|
5
40
|
### Improvements
|
data/groonga-client.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- mode: ruby -*-
|
2
2
|
#
|
3
3
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
4
|
-
# Copyright (C) 2014-
|
4
|
+
# Copyright (C) 2014-2021 Sutou Kouhei <kou@clear-code.com>
|
5
5
|
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
7
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -48,9 +48,10 @@ Gem::Specification.new do |spec|
|
|
48
48
|
end
|
49
49
|
|
50
50
|
spec.add_runtime_dependency("gqtp", ">= 1.0.4")
|
51
|
-
spec.add_runtime_dependency("groonga-command", ">= 1.4.
|
51
|
+
spec.add_runtime_dependency("groonga-command", ">= 1.4.7")
|
52
52
|
spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
|
53
53
|
spec.add_runtime_dependency("hashie")
|
54
|
+
spec.add_runtime_dependency("rexml")
|
54
55
|
|
55
56
|
spec.add_development_dependency("bundler")
|
56
57
|
spec.add_development_dependency("rake")
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2015-
|
1
|
+
# Copyright (C) 2015-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -19,6 +19,11 @@ require "json"
|
|
19
19
|
require "pathname"
|
20
20
|
require "securerandom"
|
21
21
|
|
22
|
+
begin
|
23
|
+
require "arrow"
|
24
|
+
rescue LoadError
|
25
|
+
end
|
26
|
+
|
22
27
|
require "groonga/command/parser"
|
23
28
|
|
24
29
|
require "groonga/client"
|
@@ -30,6 +35,12 @@ module Groonga
|
|
30
35
|
class GroongaClient
|
31
36
|
def initialize
|
32
37
|
@chunk = false
|
38
|
+
@load_input_type = "json"
|
39
|
+
@available_load_input_types = ["json"]
|
40
|
+
if Object.const_defined?(:Arrow)
|
41
|
+
@available_load_input_types << "apache-arrow"
|
42
|
+
end
|
43
|
+
@load_lock_table = false
|
33
44
|
|
34
45
|
@runner_options = {
|
35
46
|
:split_load_chunk_size => 10000,
|
@@ -46,7 +57,9 @@ module Groonga
|
|
46
57
|
parse_command_line(option_parser)
|
47
58
|
end
|
48
59
|
|
49
|
-
parser.open_client(:chunk => @chunk
|
60
|
+
parser.open_client(:chunk => @chunk,
|
61
|
+
:load_input_type => @load_input_type,
|
62
|
+
:load_lock_table => @load_lock_table) do |client|
|
50
63
|
runner = Runner.new(client, @runner_options)
|
51
64
|
|
52
65
|
if command_file_paths.empty?
|
@@ -60,22 +73,11 @@ module Groonga
|
|
60
73
|
end
|
61
74
|
repl.run
|
62
75
|
else
|
63
|
-
$stdin
|
64
|
-
runner << line
|
65
|
-
end
|
76
|
+
runner.consume($stdin)
|
66
77
|
end
|
67
78
|
else
|
68
79
|
command_file_paths.each do |command_file_path|
|
69
|
-
|
70
|
-
last_line = nil
|
71
|
-
command_file.each_line do |line|
|
72
|
-
last_line = line
|
73
|
-
runner << line
|
74
|
-
end
|
75
|
-
if last_line and !last_line.end_with?("\n")
|
76
|
-
runner << "\n"
|
77
|
-
end
|
78
|
-
end
|
80
|
+
runner.load(command_file_path)
|
79
81
|
end
|
80
82
|
end
|
81
83
|
runner.finish
|
@@ -99,6 +101,20 @@ module Groonga
|
|
99
101
|
@runner_options[:split_load_chunk_size] = size
|
100
102
|
end
|
101
103
|
|
104
|
+
parser.on("--load-input-type=TYPE",
|
105
|
+
@available_load_input_types,
|
106
|
+
"Use TYPE as input type for load.",
|
107
|
+
"[#{@available_load_input_types.join(", ")}]",
|
108
|
+
"(#{@load_input_types})") do |type|
|
109
|
+
@load_input_type = type
|
110
|
+
end
|
111
|
+
|
112
|
+
parser.on("--[no-]load-lock-table",
|
113
|
+
"Use lock_table=yes for load.",
|
114
|
+
"(#{@load_lock_table})") do |boolean|
|
115
|
+
@load_lock_table = boolean
|
116
|
+
end
|
117
|
+
|
102
118
|
parser.on("--[no-]generate-request-id",
|
103
119
|
"Add auto generated request ID to all commands.",
|
104
120
|
"(#{@runner_options[:generate_request_id]})") do |boolean|
|
@@ -146,95 +162,9 @@ module Groonga
|
|
146
162
|
end
|
147
163
|
end
|
148
164
|
|
149
|
-
class Runner
|
150
|
-
def initialize(client, options={})
|
151
|
-
@client = client
|
152
|
-
@split_load_chunk_size = options[:split_load_chunk_size] || 10000
|
153
|
-
@generate_request_id = options[:generate_request_id]
|
154
|
-
@target_commands = options[:target_commands]
|
155
|
-
@target_tables = options[:target_tables]
|
156
|
-
@target_columns = options[:target_columns]
|
157
|
-
@load_values = []
|
158
|
-
@parser = create_command_parser
|
159
|
-
end
|
160
|
-
|
161
|
-
def <<(line)
|
162
|
-
@parser << line
|
163
|
-
end
|
164
|
-
|
165
|
-
def finish
|
166
|
-
@parser.finish
|
167
|
-
end
|
168
|
-
|
169
|
-
def repl
|
170
|
-
begin
|
171
|
-
require "readline"
|
172
|
-
rescue LoadError
|
173
|
-
repl = BareREPL.new(self)
|
174
|
-
else
|
175
|
-
repl = ReadlineREPL.new(self)
|
176
|
-
repl_readline
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
165
|
+
class Runner < CommandProcessor
|
180
166
|
private
|
181
|
-
def
|
182
|
-
parser = Groonga::Command::Parser.new
|
183
|
-
|
184
|
-
parser.on_command do |command|
|
185
|
-
run_command(command)
|
186
|
-
end
|
187
|
-
|
188
|
-
parser.on_load_columns do |command, columns|
|
189
|
-
command[:columns] ||= columns.join(",")
|
190
|
-
end
|
191
|
-
|
192
|
-
parser.on_load_value do |command, value|
|
193
|
-
unless command[:values]
|
194
|
-
@load_values << value
|
195
|
-
if @load_values.size == @split_load_chunk_size
|
196
|
-
consume_load_values(command)
|
197
|
-
end
|
198
|
-
end
|
199
|
-
command.original_source.clear
|
200
|
-
end
|
201
|
-
|
202
|
-
parser.on_load_complete do |command|
|
203
|
-
if command[:values]
|
204
|
-
run_command(command)
|
205
|
-
else
|
206
|
-
consume_load_values(command)
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
|
-
parser
|
211
|
-
end
|
212
|
-
|
213
|
-
def consume_load_values(load_command)
|
214
|
-
return if @load_values.empty?
|
215
|
-
|
216
|
-
values_json = "["
|
217
|
-
@load_values.each_with_index do |value, i|
|
218
|
-
values_json << "," unless i.zero?
|
219
|
-
values_json << "\n"
|
220
|
-
values_json << JSON.generate(value)
|
221
|
-
end
|
222
|
-
values_json << "\n]\n"
|
223
|
-
load_command[:values] = values_json
|
224
|
-
run_command(load_command)
|
225
|
-
@load_values.clear
|
226
|
-
load_command[:values] = nil
|
227
|
-
end
|
228
|
-
|
229
|
-
def run_command(command)
|
230
|
-
return unless target_command?(command)
|
231
|
-
return unless target_table?(command)
|
232
|
-
return unless target_column?(command)
|
233
|
-
|
234
|
-
command = Marshal.load(Marshal.dump(command))
|
235
|
-
apply_target_columns(command)
|
236
|
-
command[:request_id] ||= SecureRandom.uuid if @generate_request_id
|
237
|
-
response = @client.execute(command)
|
167
|
+
def process_response(response, command)
|
238
168
|
case command.output_type
|
239
169
|
when :json
|
240
170
|
puts(JSON.pretty_generate([response.header, response.body]))
|
@@ -244,92 +174,6 @@ module Groonga
|
|
244
174
|
puts(response.body)
|
245
175
|
end
|
246
176
|
end
|
247
|
-
|
248
|
-
def target_command?(command)
|
249
|
-
return true if @target_commands.empty?
|
250
|
-
|
251
|
-
@target_commands.any? do |name|
|
252
|
-
name === command.command_name
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
def target_table?(command)
|
257
|
-
return true if @target_tables.empty?
|
258
|
-
|
259
|
-
target = nil
|
260
|
-
case command.command_name
|
261
|
-
when "load", "column_create", "select"
|
262
|
-
target = command.table
|
263
|
-
when "table_create", "table_remove"
|
264
|
-
target = command.name
|
265
|
-
end
|
266
|
-
return true if target.nil?
|
267
|
-
|
268
|
-
@target_tables.any? do |name|
|
269
|
-
name === target
|
270
|
-
end
|
271
|
-
end
|
272
|
-
|
273
|
-
def target_column?(command)
|
274
|
-
return true if @target_columns.empty?
|
275
|
-
|
276
|
-
target = nil
|
277
|
-
case command.command_name
|
278
|
-
when "column_create"
|
279
|
-
target = command.name
|
280
|
-
end
|
281
|
-
return true if target.nil?
|
282
|
-
|
283
|
-
@target_columns.any? do |name|
|
284
|
-
name === target
|
285
|
-
end
|
286
|
-
end
|
287
|
-
|
288
|
-
def apply_target_columns(command)
|
289
|
-
return if @target_columns.empty?
|
290
|
-
|
291
|
-
values = command[:values]
|
292
|
-
return if values.nil?
|
293
|
-
|
294
|
-
command = command.dup
|
295
|
-
|
296
|
-
values = JSON.parse(values)
|
297
|
-
columns = command[:columns]
|
298
|
-
if columns
|
299
|
-
columns = columns.split(/\s*,\s*/)
|
300
|
-
target_indexes = []
|
301
|
-
new_columns = []
|
302
|
-
columns.each_with_index do |column, i|
|
303
|
-
if load_target_column?(column)
|
304
|
-
target_indexes << i
|
305
|
-
new_columns << column
|
306
|
-
end
|
307
|
-
end
|
308
|
-
command[:columns] = new_columns.join(",")
|
309
|
-
new_values = values.collect do |value|
|
310
|
-
target_indexes.collect do |i|
|
311
|
-
value[i]
|
312
|
-
end
|
313
|
-
end
|
314
|
-
command[:values] = JSON.generate(new_values)
|
315
|
-
else
|
316
|
-
new_values = values.collect do |value|
|
317
|
-
new_value = {}
|
318
|
-
value.each do |key, value|
|
319
|
-
if load_target_column?(key)
|
320
|
-
new_value[key] = value
|
321
|
-
end
|
322
|
-
end
|
323
|
-
new_value
|
324
|
-
end
|
325
|
-
command[:values] = JSON.generate(new_values)
|
326
|
-
end
|
327
|
-
end
|
328
|
-
|
329
|
-
def load_target_column?(column)
|
330
|
-
column == "_key" or
|
331
|
-
@target_columns.any? {|name| name === column}
|
332
|
-
end
|
333
177
|
end
|
334
178
|
|
335
179
|
class BareREPL
|
@@ -0,0 +1,211 @@
|
|
1
|
+
# Copyright (C) 2015-2021 Sutou Kouhei <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require "json"
|
18
|
+
|
19
|
+
require "groonga/command/parser"
|
20
|
+
|
21
|
+
module Groonga
|
22
|
+
class Client
|
23
|
+
class CommandProcessor
|
24
|
+
def initialize(client, options={})
|
25
|
+
@client = client
|
26
|
+
@split_load_chunk_size = options[:split_load_chunk_size] || 10000
|
27
|
+
@generate_request_id = options[:generate_request_id]
|
28
|
+
@target_commands = options[:target_commands]
|
29
|
+
@target_tables = options[:target_tables]
|
30
|
+
@target_columns = options[:target_columns]
|
31
|
+
@load_values = []
|
32
|
+
@parser = create_command_parser
|
33
|
+
end
|
34
|
+
|
35
|
+
def <<(line)
|
36
|
+
@parser << line
|
37
|
+
end
|
38
|
+
|
39
|
+
def finish
|
40
|
+
@parser.finish
|
41
|
+
end
|
42
|
+
|
43
|
+
def consume(input)
|
44
|
+
last_line = nil
|
45
|
+
input.each_line do |line|
|
46
|
+
last_line = line
|
47
|
+
@parser << line
|
48
|
+
end
|
49
|
+
if last_line and last_line.end_with?("\n")
|
50
|
+
@parser << "\n"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def load(path)
|
55
|
+
File.open(path) do |input|
|
56
|
+
consume(input)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
def create_command_parser
|
62
|
+
parser = Groonga::Command::Parser.new
|
63
|
+
|
64
|
+
parser.on_command do |command|
|
65
|
+
run_command(command)
|
66
|
+
end
|
67
|
+
|
68
|
+
parser.on_load_columns do |command, columns|
|
69
|
+
command[:columns] ||= columns.join(",")
|
70
|
+
end
|
71
|
+
|
72
|
+
parser.on_load_value do |command, value|
|
73
|
+
unless command[:values]
|
74
|
+
@load_values << value
|
75
|
+
if @load_values.size == @split_load_chunk_size
|
76
|
+
consume_load_values(command)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
command.original_source.clear
|
80
|
+
end
|
81
|
+
|
82
|
+
parser.on_load_complete do |command|
|
83
|
+
if command[:values]
|
84
|
+
run_command(command)
|
85
|
+
else
|
86
|
+
consume_load_values(command)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
parser
|
91
|
+
end
|
92
|
+
|
93
|
+
def consume_load_values(load_command)
|
94
|
+
return if @load_values.empty?
|
95
|
+
|
96
|
+
values_json = "["
|
97
|
+
@load_values.each_with_index do |value, i|
|
98
|
+
values_json << "," unless i.zero?
|
99
|
+
values_json << "\n"
|
100
|
+
values_json << JSON.generate(value)
|
101
|
+
end
|
102
|
+
values_json << "\n]\n"
|
103
|
+
load_command[:values] = values_json
|
104
|
+
run_command(load_command)
|
105
|
+
@load_values.clear
|
106
|
+
load_command[:values] = nil
|
107
|
+
end
|
108
|
+
|
109
|
+
def run_command(command)
|
110
|
+
return unless target_command?(command)
|
111
|
+
return unless target_table?(command)
|
112
|
+
return unless target_column?(command)
|
113
|
+
|
114
|
+
command = Marshal.load(Marshal.dump(command))
|
115
|
+
apply_target_columns(command)
|
116
|
+
command[:request_id] ||= SecureRandom.uuid if @generate_request_id
|
117
|
+
response = @client.execute(command)
|
118
|
+
process_response(response, command)
|
119
|
+
end
|
120
|
+
|
121
|
+
def process_response(response, command)
|
122
|
+
end
|
123
|
+
|
124
|
+
def target_command?(command)
|
125
|
+
return true if @target_commands.empty?
|
126
|
+
|
127
|
+
@target_commands.any? do |name|
|
128
|
+
name === command.command_name
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def target_table?(command)
|
133
|
+
return true if @target_tables.empty?
|
134
|
+
|
135
|
+
target = nil
|
136
|
+
case command.command_name
|
137
|
+
when "load", "column_create", "select"
|
138
|
+
target = command.table
|
139
|
+
when "table_create", "table_remove"
|
140
|
+
target = command.name
|
141
|
+
end
|
142
|
+
return true if target.nil?
|
143
|
+
|
144
|
+
@target_tables.any? do |name|
|
145
|
+
name === target
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def target_column?(command)
|
150
|
+
return true if @target_columns.empty?
|
151
|
+
|
152
|
+
target = nil
|
153
|
+
case command.command_name
|
154
|
+
when "column_create"
|
155
|
+
target = command.name
|
156
|
+
end
|
157
|
+
return true if target.nil?
|
158
|
+
|
159
|
+
@target_columns.any? do |name|
|
160
|
+
name === target
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def apply_target_columns(command)
|
165
|
+
return if @target_columns.empty?
|
166
|
+
|
167
|
+
values = command[:values]
|
168
|
+
return if values.nil?
|
169
|
+
|
170
|
+
command = command.dup
|
171
|
+
|
172
|
+
values = JSON.parse(values)
|
173
|
+
columns = command[:columns]
|
174
|
+
if columns
|
175
|
+
columns = columns.split(/\s*,\s*/)
|
176
|
+
target_indexes = []
|
177
|
+
new_columns = []
|
178
|
+
columns.each_with_index do |column, i|
|
179
|
+
if load_target_column?(column)
|
180
|
+
target_indexes << i
|
181
|
+
new_columns << column
|
182
|
+
end
|
183
|
+
end
|
184
|
+
command[:columns] = new_columns.join(",")
|
185
|
+
new_values = values.collect do |value|
|
186
|
+
target_indexes.collect do |i|
|
187
|
+
value[i]
|
188
|
+
end
|
189
|
+
end
|
190
|
+
command[:values] = JSON.generate(new_values)
|
191
|
+
else
|
192
|
+
new_values = values.collect do |value|
|
193
|
+
new_value = {}
|
194
|
+
value.each do |key, value|
|
195
|
+
if load_target_column?(key)
|
196
|
+
new_value[key] = value
|
197
|
+
end
|
198
|
+
end
|
199
|
+
new_value
|
200
|
+
end
|
201
|
+
command[:values] = JSON.generate(new_values)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def load_target_column?(column)
|
206
|
+
column == "_key" or
|
207
|
+
@target_columns.any? {|name| name === column}
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
2
|
-
# Copyright (C) 2013-
|
2
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
3
3
|
#
|
4
4
|
# This library is free software; you can redistribute it and/or
|
5
5
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -55,9 +55,15 @@ module Groonga
|
|
55
55
|
@options = options
|
56
56
|
end
|
57
57
|
|
58
|
+
DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
|
58
59
|
def send(command)
|
59
60
|
begin
|
60
|
-
HTTPClient.
|
61
|
+
http = HTTPClient.new(@url.host, @url.port)
|
62
|
+
http.set_debug_output($stderr) if DEBUG
|
63
|
+
start_options.each do |key, value|
|
64
|
+
http.__send__("#{key}=", value)
|
65
|
+
end
|
66
|
+
http.start do
|
61
67
|
http.read_timeout = read_timeout
|
62
68
|
response = send_request(http, command)
|
63
69
|
case response
|
@@ -141,18 +147,7 @@ module Groonga
|
|
141
147
|
|
142
148
|
def send_request(http, command)
|
143
149
|
if command.is_a?(Groonga::Command::Load)
|
144
|
-
|
145
|
-
command[:values] = nil
|
146
|
-
path = resolve_path(@url, command.to_uri_format)
|
147
|
-
command[:values] = raw_values
|
148
|
-
request = Net::HTTP::Post.new(path, headers)
|
149
|
-
request.content_type = "application/json"
|
150
|
-
if @options[:chunk]
|
151
|
-
request["Transfer-Encoding"] = "chunked"
|
152
|
-
else
|
153
|
-
request.content_length = raw_values.bytesize
|
154
|
-
end
|
155
|
-
request.body_stream = StringIO.new(raw_values)
|
150
|
+
request = prepare_load_request(command)
|
156
151
|
else
|
157
152
|
path = resolve_path(@url, command.to_uri_format)
|
158
153
|
request = Net::HTTP::Get.new(path, headers)
|
@@ -167,6 +162,42 @@ module Groonga
|
|
167
162
|
}
|
168
163
|
end
|
169
164
|
|
165
|
+
def prepare_load_request(command)
|
166
|
+
path_prefix = command.path_prefix
|
167
|
+
command = command.class.new(command.command_name,
|
168
|
+
command.arguments,
|
169
|
+
[])
|
170
|
+
command.path_prefix = path_prefix
|
171
|
+
case @options[:load_input_type]
|
172
|
+
when "apache-arrow"
|
173
|
+
command[:input_type] = "apache-arrow"
|
174
|
+
content_type = "application/x-apache-arrow-streaming"
|
175
|
+
arrow_table = command.build_arrow_table
|
176
|
+
if arrow_table
|
177
|
+
buffer = Arrow::ResizableBuffer.new(1024)
|
178
|
+
arrow_table.save(buffer, format: :stream)
|
179
|
+
body = buffer.data.to_s
|
180
|
+
else
|
181
|
+
body = ""
|
182
|
+
end
|
183
|
+
command.arguments.delete(:values)
|
184
|
+
else
|
185
|
+
content_type = "application/json"
|
186
|
+
body = command.arguments.delete(:values)
|
187
|
+
end
|
188
|
+
command[:lock_table] = "yes" if @options[:load_lock_table]
|
189
|
+
path = resolve_path(@url, command.to_uri_format)
|
190
|
+
request = Net::HTTP::Post.new(path, headers)
|
191
|
+
if @options[:chunk]
|
192
|
+
request["Transfer-Encoding"] = "chunked"
|
193
|
+
else
|
194
|
+
request.content_length = body.bytesize
|
195
|
+
end
|
196
|
+
request.content_type = content_type
|
197
|
+
request.body_stream = StringIO.new(body)
|
198
|
+
request
|
199
|
+
end
|
200
|
+
|
170
201
|
def setup_authentication(request)
|
171
202
|
userinfo = @url.userinfo
|
172
203
|
return if userinfo.nil?
|
@@ -19,6 +19,10 @@ require "csv"
|
|
19
19
|
require "rexml/document"
|
20
20
|
require "json"
|
21
21
|
|
22
|
+
begin
|
23
|
+
require "arrow"
|
24
|
+
rescue LoadError
|
25
|
+
end
|
22
26
|
require "hashie"
|
23
27
|
|
24
28
|
module Groonga
|
@@ -86,6 +90,9 @@ module Groonga
|
|
86
90
|
when :tsv
|
87
91
|
header, body = parse_tsv(raw_response)
|
88
92
|
return_code = header["return_code"] if header
|
93
|
+
when :arrow, :"apache-arrow"
|
94
|
+
header, body = parse_apache_arrow(raw_response)
|
95
|
+
return_code = header["return_code"] if header
|
89
96
|
else
|
90
97
|
header = nil
|
91
98
|
body = raw_response
|
@@ -177,6 +184,40 @@ module Groonga
|
|
177
184
|
end
|
178
185
|
body
|
179
186
|
end
|
187
|
+
|
188
|
+
def parse_apache_arrow(response)
|
189
|
+
header = nil
|
190
|
+
body = nil
|
191
|
+
buffer = Arrow::Buffer.new(response)
|
192
|
+
Arrow::BufferInputStream.open(buffer) do |input|
|
193
|
+
while input.tell < response.bytesize
|
194
|
+
reader = Arrow::RecordBatchStreamReader.new(input)
|
195
|
+
schema = reader.schema
|
196
|
+
record_batches = reader.to_a
|
197
|
+
if apache_arrow_metadata?(schema)
|
198
|
+
table = Arrow::Table.new(schema, record_batches)
|
199
|
+
header = table.each_record.first.to_h
|
200
|
+
else
|
201
|
+
body = {}
|
202
|
+
body["columns"] = schema.fields.collect do |field|
|
203
|
+
[field.name, field.data_type.to_s]
|
204
|
+
end
|
205
|
+
if record_batches.empty?
|
206
|
+
records = []
|
207
|
+
else
|
208
|
+
table = Arrow::Table.new(schema, record_batches)
|
209
|
+
records = table.raw_records
|
210
|
+
end
|
211
|
+
body["records"] = records
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
return header, body
|
216
|
+
end
|
217
|
+
|
218
|
+
def apache_arrow_metadata?(schema)
|
219
|
+
(schema.metadata || {})["GROONGA:data_type"] == "metadata"
|
220
|
+
end
|
180
221
|
end
|
181
222
|
|
182
223
|
# @return [Groonga::Command] The command for the request.
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2021 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -16,6 +16,6 @@
|
|
16
16
|
|
17
17
|
module Groonga
|
18
18
|
class Client
|
19
|
-
VERSION = "0.6.
|
19
|
+
VERSION = "0.6.6"
|
20
20
|
end
|
21
21
|
end
|
data/lib/groonga/client.rb
CHANGED
@@ -18,12 +18,13 @@
|
|
18
18
|
require "uri"
|
19
19
|
require "json"
|
20
20
|
|
21
|
-
require "groonga/client/default"
|
22
21
|
require "groonga/client/command"
|
22
|
+
require "groonga/client/command-processor"
|
23
|
+
require "groonga/client/default"
|
23
24
|
require "groonga/client/empty-request"
|
25
|
+
require "groonga/client/protocol/file"
|
24
26
|
require "groonga/client/protocol/gqtp"
|
25
27
|
require "groonga/client/protocol/http"
|
26
|
-
require "groonga/client/protocol/file"
|
27
28
|
require "groonga/client/request"
|
28
29
|
|
29
30
|
module Groonga
|
@@ -202,7 +203,7 @@ module Groonga
|
|
202
203
|
end
|
203
204
|
|
204
205
|
def method_missing(name, *args, &block)
|
205
|
-
if groonga_command_name?(name)
|
206
|
+
if groonga_command_name?(name) and args.size <= 1
|
206
207
|
execute(name, *args, &block)
|
207
208
|
else
|
208
209
|
super
|
data/test/request/test-select.rb
CHANGED
@@ -186,8 +186,8 @@ class TestRequestSelect < Test::Unit::TestCase
|
|
186
186
|
end
|
187
187
|
|
188
188
|
sub_test_case("#paginate") do
|
189
|
-
def paginate(*args)
|
190
|
-
@request.paginate(*args).to_parameters
|
189
|
+
def paginate(*args, **kwargs)
|
190
|
+
@request.paginate(*args, **kwargs).to_parameters
|
191
191
|
end
|
192
192
|
|
193
193
|
sub_test_case("page") do
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
# Copyright (C) 2013 Kosuke Asami
|
3
3
|
# Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
|
4
4
|
#
|
@@ -418,8 +418,8 @@ class TestResponseSelectCommandVersion1 < Test::Unit::TestCase
|
|
418
418
|
values = {}
|
419
419
|
slices(body).each do |label, slice|
|
420
420
|
drilldowns = {}
|
421
|
-
slice.drilldowns.each do |
|
422
|
-
drilldowns[
|
421
|
+
slice.drilldowns.each do |drilldown_label, drilldown|
|
422
|
+
drilldowns[drilldown_label] = drilldown.records
|
423
423
|
end
|
424
424
|
values[label] = {
|
425
425
|
records: slice.records,
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013-
|
1
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
# Copyright (C) 2013 Kosuke Asami
|
3
3
|
# Copyright (C) 2016 Masafumi Yokoyama <yokoyama@clear-code.com>
|
4
4
|
#
|
@@ -383,8 +383,8 @@ class TestResponseSelectCommandVersion3 < Test::Unit::TestCase
|
|
383
383
|
values = {}
|
384
384
|
slices(body).each do |label, slice|
|
385
385
|
drilldowns = {}
|
386
|
-
slice.drilldowns.each do |
|
387
|
-
drilldowns[
|
386
|
+
slice.drilldowns.each do |drilldown_label, drilldown|
|
387
|
+
drilldowns[drilldown_label] = drilldown.records
|
388
388
|
end
|
389
389
|
values[label] = {
|
390
390
|
records: slice.records,
|
data/test/test-command.rb
CHANGED
@@ -5,7 +5,8 @@ class TestCommand < Test::Unit::TestCase
|
|
5
5
|
|
6
6
|
def test_column_create
|
7
7
|
response = Object.new
|
8
|
-
mock(@client).execute(:column_create,
|
8
|
+
mock(@client).execute(:column_create,
|
9
|
+
{table: :Test, name: :Body, type: :ShortText}) do
|
9
10
|
response
|
10
11
|
end
|
11
12
|
@client.column_create(:table => :Test, :name => :Body, :type => :ShortText)
|
@@ -13,7 +14,7 @@ class TestCommand < Test::Unit::TestCase
|
|
13
14
|
|
14
15
|
def test_column_list
|
15
16
|
response = Object.new
|
16
|
-
mock(@client).execute(:column_list, :
|
17
|
+
mock(@client).execute(:column_list, {table: :Test}) do
|
17
18
|
response
|
18
19
|
end
|
19
20
|
@client.column_list(:table => :Test)
|
@@ -27,7 +28,7 @@ class TestCommand < Test::Unit::TestCase
|
|
27
28
|
}
|
28
29
|
]
|
29
30
|
response = Object.new
|
30
|
-
mock(@client).execute(:load, :
|
31
|
+
mock(@client).execute(:load, {table: :Test, values: values.to_json}) do
|
31
32
|
response
|
32
33
|
end
|
33
34
|
@client.load(:table => :Test, :values => values.to_json)
|
@@ -35,7 +36,7 @@ class TestCommand < Test::Unit::TestCase
|
|
35
36
|
|
36
37
|
def test_select
|
37
38
|
response = Object.new
|
38
|
-
mock(@client).execute(:select, :
|
39
|
+
mock(@client).execute(:select, {table: :Test}) do
|
39
40
|
response
|
40
41
|
end
|
41
42
|
@client.select(:table => :Test)
|
@@ -43,7 +44,7 @@ class TestCommand < Test::Unit::TestCase
|
|
43
44
|
|
44
45
|
def test_table_create
|
45
46
|
response = Object.new
|
46
|
-
mock(@client).execute(:table_create, :
|
47
|
+
mock(@client).execute(:table_create, {name: :Test}) do
|
47
48
|
response
|
48
49
|
end
|
49
50
|
@client.table_create(:name => :Test)
|
@@ -58,7 +59,7 @@ class TestCommand < Test::Unit::TestCase
|
|
58
59
|
end
|
59
60
|
|
60
61
|
def test_table_remove
|
61
|
-
mock(@client).execute(:table_remove, :
|
62
|
+
mock(@client).execute(:table_remove, {name: "Test"})
|
62
63
|
@client.table_remove(:name => "Test")
|
63
64
|
end
|
64
65
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: groonga-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Haruka Yoshihara
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-12-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: gqtp
|
@@ -32,14 +32,14 @@ dependencies:
|
|
32
32
|
requirements:
|
33
33
|
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
|
-
version: 1.4.
|
35
|
+
version: 1.4.7
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
40
|
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version: 1.4.
|
42
|
+
version: 1.4.7
|
43
43
|
- !ruby/object:Gem::Dependency
|
44
44
|
name: groonga-command-parser
|
45
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -68,6 +68,20 @@ dependencies:
|
|
68
68
|
- - ">="
|
69
69
|
- !ruby/object:Gem::Version
|
70
70
|
version: '0'
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: rexml
|
73
|
+
requirement: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
type: :runtime
|
79
|
+
prerelease: false
|
80
|
+
version_requirements: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
71
85
|
- !ruby/object:Gem::Dependency
|
72
86
|
name: bundler
|
73
87
|
requirement: !ruby/object:Gem::Requirement
|
@@ -161,9 +175,9 @@ email:
|
|
161
175
|
- kou@clear-code.com
|
162
176
|
- tfortress58@gmail.com
|
163
177
|
executables:
|
164
|
-
- groonga-client-index-recreate
|
165
|
-
- groonga-client-index-check
|
166
178
|
- groonga-client
|
179
|
+
- groonga-client-index-check
|
180
|
+
- groonga-client-index-recreate
|
167
181
|
extensions: []
|
168
182
|
extra_rdoc_files: []
|
169
183
|
files:
|
@@ -183,6 +197,7 @@ files:
|
|
183
197
|
- lib/groonga/client/command-line/groonga-client.rb
|
184
198
|
- lib/groonga/client/command-line/parser.rb
|
185
199
|
- lib/groonga/client/command-line/runner.rb
|
200
|
+
- lib/groonga/client/command-processor.rb
|
186
201
|
- lib/groonga/client/command.rb
|
187
202
|
- lib/groonga/client/default.rb
|
188
203
|
- lib/groonga/client/empty-request.rb
|
@@ -286,43 +301,42 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
286
301
|
- !ruby/object:Gem::Version
|
287
302
|
version: '0'
|
288
303
|
requirements: []
|
289
|
-
|
290
|
-
rubygems_version: 2.7.6.2
|
304
|
+
rubygems_version: 3.3.0.dev
|
291
305
|
signing_key:
|
292
306
|
specification_version: 4
|
293
307
|
summary: Groonga-client is a client for Groonga (http://groonga.org/) implemented
|
294
308
|
with pure Ruby. You can use it without Groonga.
|
295
309
|
test_files:
|
296
|
-
- test/results/test-table-list.rb
|
297
|
-
- test/test-script-syntax.rb
|
298
|
-
- test/response/test-table-create.rb
|
299
|
-
- test/response/helper.rb
|
300
|
-
- test/response/test-select-command-version3.rb
|
301
|
-
- test/response/test-column-list.rb
|
302
|
-
- test/response/test-select-xml.rb
|
303
|
-
- test/response/test-load.rb
|
304
|
-
- test/response/test-select-command-version1.rb
|
305
|
-
- test/response/test-select-tsv.rb
|
306
|
-
- test/response/test-table-list.rb
|
307
|
-
- test/response/test-status.rb
|
308
|
-
- test/response/test-schema.rb
|
309
|
-
- test/response/test-base.rb
|
310
|
-
- test/response/test-error.rb
|
311
|
-
- test/response/test-table-remove.rb
|
312
310
|
- test/command-line/helper.rb
|
313
|
-
- test/command-line/test-index-recreate.rb
|
314
311
|
- test/command-line/test-index-check.rb
|
315
|
-
- test/
|
316
|
-
- test/test-
|
312
|
+
- test/command-line/test-index-recreate.rb
|
313
|
+
- test/protocol/test-gqtp.rb
|
314
|
+
- test/protocol/test-http.rb
|
317
315
|
- test/request/select/test-backward-compatible-sort-keys-parameter.rb
|
318
|
-
- test/request/select/test-values-parameter.rb
|
319
|
-
- test/request/select/test-scorer.rb
|
320
316
|
- test/request/select/test-filter.rb
|
321
317
|
- test/request/select/test-output-columns-parameter.rb
|
318
|
+
- test/request/select/test-scorer.rb
|
322
319
|
- test/request/select/test-sort-keys-parameter.rb
|
320
|
+
- test/request/select/test-values-parameter.rb
|
323
321
|
- test/request/test-generic.rb
|
324
322
|
- test/request/test-merger.rb
|
325
323
|
- test/request/test-select.rb
|
324
|
+
- test/response/helper.rb
|
325
|
+
- test/response/test-base.rb
|
326
|
+
- test/response/test-column-list.rb
|
327
|
+
- test/response/test-error.rb
|
328
|
+
- test/response/test-load.rb
|
329
|
+
- test/response/test-schema.rb
|
330
|
+
- test/response/test-select-command-version1.rb
|
331
|
+
- test/response/test-select-command-version3.rb
|
332
|
+
- test/response/test-select-tsv.rb
|
333
|
+
- test/response/test-select-xml.rb
|
334
|
+
- test/response/test-status.rb
|
335
|
+
- test/response/test-table-create.rb
|
336
|
+
- test/response/test-table-list.rb
|
337
|
+
- test/response/test-table-remove.rb
|
338
|
+
- test/results/test-table-list.rb
|
339
|
+
- test/run-test.rb
|
326
340
|
- test/test-client.rb
|
327
|
-
- test/
|
328
|
-
- test/
|
341
|
+
- test/test-command.rb
|
342
|
+
- test/test-script-syntax.rb
|