groonga-client 0.5.8 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/doc/text/news.md +54 -0
- data/groonga-client.gemspec +2 -2
- data/lib/groonga/client.rb +15 -3
- data/lib/groonga/client/command-line/groonga-client.rb +278 -4
- data/lib/groonga/client/protocol/file.rb +76 -0
- data/lib/groonga/client/protocol/http/path-resolvable.rb +6 -1
- data/lib/groonga/client/protocol/http/synchronous.rb +45 -14
- data/lib/groonga/client/response.rb +3 -1
- data/lib/groonga/client/response/base.rb +93 -4
- data/lib/groonga/client/response/drilldownable.rb +85 -0
- data/lib/groonga/client/response/error.rb +42 -2
- data/lib/groonga/client/response/logical-range-filter.rb +52 -0
- data/lib/groonga/client/response/logical-select.rb +28 -0
- data/lib/groonga/client/response/searchable.rb +97 -0
- data/lib/groonga/client/response/select.rb +136 -125
- data/lib/groonga/client/version.rb +2 -2
- data/test/response/helper.rb +5 -0
- data/test/response/test-base.rb +14 -1
- data/test/response/test-error.rb +1 -1
- data/test/response/test-select-command-version1.rb +58 -12
- data/test/response/test-select-command-version3.rb +54 -8
- data/test/response/test-select-tsv.rb +149 -0
- data/test/response/test-select-xml.rb +26 -4
- data/test/test-client.rb +2 -2
- metadata +35 -29
@@ -0,0 +1,76 @@
|
|
1
|
+
# Copyright (C) 2017 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require "groonga/client/version"
|
18
|
+
require "groonga/client/protocol/error"
|
19
|
+
|
20
|
+
module Groonga
|
21
|
+
class Client
|
22
|
+
module Protocol
|
23
|
+
class File
|
24
|
+
def initialize(url, options)
|
25
|
+
@url = url
|
26
|
+
@options = options
|
27
|
+
end
|
28
|
+
|
29
|
+
def send(command, &block)
|
30
|
+
open_pipes do |input, output, error|
|
31
|
+
options = {
|
32
|
+
:in => input[0],
|
33
|
+
:out => output[1],
|
34
|
+
:err => error[1],
|
35
|
+
}
|
36
|
+
pid = spawn("groonga", @url.path, options)
|
37
|
+
input[0].close
|
38
|
+
output[1].close
|
39
|
+
error[1].close
|
40
|
+
|
41
|
+
input[1].puts(command.to_command_format)
|
42
|
+
input[1].close
|
43
|
+
response = output[0].read
|
44
|
+
Process.waitpid(pid)
|
45
|
+
yield(response)
|
46
|
+
EmptyRequest.new
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def connected?
|
51
|
+
false
|
52
|
+
end
|
53
|
+
|
54
|
+
def close(&block)
|
55
|
+
if block_given?
|
56
|
+
yield
|
57
|
+
EmptyRequest.new
|
58
|
+
else
|
59
|
+
false
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
def open_pipes
|
65
|
+
IO.pipe do |input|
|
66
|
+
IO.pipe do |output|
|
67
|
+
IO.pipe do |error|
|
68
|
+
yield(input, output, error)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -21,7 +21,12 @@ module Groonga
|
|
21
21
|
module PathResolvable
|
22
22
|
private
|
23
23
|
def resolve_path(uri, path)
|
24
|
-
uri.path.chomp("/")
|
24
|
+
path_prefix = uri.path.chomp("/")
|
25
|
+
if path_prefix.empty?
|
26
|
+
path
|
27
|
+
else
|
28
|
+
path.sub(/\A\/d/) {path_prefix}
|
29
|
+
end
|
25
30
|
end
|
26
31
|
end
|
27
32
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
2
|
-
# Copyright (C) 2013-
|
2
|
+
# Copyright (C) 2013-2020 Sutou Kouhei <kou@clear-code.com>
|
3
3
|
#
|
4
4
|
# This library is free software; you can redistribute it and/or
|
5
5
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -55,9 +55,15 @@ module Groonga
|
|
55
55
|
@options = options
|
56
56
|
end
|
57
57
|
|
58
|
+
DEBUG = (ENV["GROONGA_CLIENT_HTTP_DEBUG"] == "yes")
|
58
59
|
def send(command)
|
59
60
|
begin
|
60
|
-
HTTPClient.
|
61
|
+
http = HTTPClient.new(@url.host, @url.port)
|
62
|
+
http.set_debug_output($stderr) if DEBUG
|
63
|
+
start_options.each do |key, value|
|
64
|
+
http.__send__("#{key}=", value)
|
65
|
+
end
|
66
|
+
http.start do
|
61
67
|
http.read_timeout = read_timeout
|
62
68
|
response = send_request(http, command)
|
63
69
|
case response
|
@@ -141,18 +147,7 @@ module Groonga
|
|
141
147
|
|
142
148
|
def send_request(http, command)
|
143
149
|
if command.is_a?(Groonga::Command::Load)
|
144
|
-
|
145
|
-
command[:values] = nil
|
146
|
-
path = resolve_path(@url, command.to_uri_format)
|
147
|
-
command[:values] = raw_values
|
148
|
-
request = Net::HTTP::Post.new(path, headers)
|
149
|
-
request.content_type = "application/json"
|
150
|
-
if @options[:chunk]
|
151
|
-
request["Transfer-Encoding"] = "chunked"
|
152
|
-
else
|
153
|
-
request.content_length = raw_values.bytesize
|
154
|
-
end
|
155
|
-
request.body_stream = StringIO.new(raw_values)
|
150
|
+
request = prepare_load_request(command)
|
156
151
|
else
|
157
152
|
path = resolve_path(@url, command.to_uri_format)
|
158
153
|
request = Net::HTTP::Get.new(path, headers)
|
@@ -167,6 +162,42 @@ module Groonga
|
|
167
162
|
}
|
168
163
|
end
|
169
164
|
|
165
|
+
def prepare_load_request(command)
|
166
|
+
path_prefix = command.path_prefix
|
167
|
+
command = command.class.new(command.command_name,
|
168
|
+
command.arguments,
|
169
|
+
[])
|
170
|
+
command.path_prefix = path_prefix
|
171
|
+
case @options[:load_input_type]
|
172
|
+
when "apache-arrow"
|
173
|
+
command[:input_type] = "apache-arrow"
|
174
|
+
content_type = "application/x-apache-arrow-streaming"
|
175
|
+
arrow_table = command.build_arrow_table
|
176
|
+
if arrow_table
|
177
|
+
buffer = Arrow::ResizableBuffer.new(1024)
|
178
|
+
arrow_table.save(buffer, format: :stream)
|
179
|
+
body = buffer.data.to_s
|
180
|
+
else
|
181
|
+
body = ""
|
182
|
+
end
|
183
|
+
command.arguments.delete(:values)
|
184
|
+
else
|
185
|
+
content_type = "application/json"
|
186
|
+
body = command.arguments.delete(:values)
|
187
|
+
end
|
188
|
+
command[:lock_table] = "yes" if @options[:load_lock_table]
|
189
|
+
path = resolve_path(@url, command.to_uri_format)
|
190
|
+
request = Net::HTTP::Post.new(path, headers)
|
191
|
+
if @options[:chunk]
|
192
|
+
request["Transfer-Encoding"] = "chunked"
|
193
|
+
else
|
194
|
+
request.content_length = body.bytesize
|
195
|
+
end
|
196
|
+
request.content_type = content_type
|
197
|
+
request.body_stream = StringIO.new(body)
|
198
|
+
request
|
199
|
+
end
|
200
|
+
|
170
201
|
def setup_authentication(request)
|
171
202
|
userinfo = @url.userinfo
|
172
203
|
return if userinfo.nil?
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2014-
|
1
|
+
# Copyright (C) 2014-2019 Sutou Kouhei <kou@clear-code.com>
|
2
2
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
3
3
|
#
|
4
4
|
# This library is free software; you can redistribute it and/or
|
@@ -30,6 +30,8 @@ require "groonga/client/response/lock-clear"
|
|
30
30
|
require "groonga/client/response/log-level"
|
31
31
|
require "groonga/client/response/log-put"
|
32
32
|
require "groonga/client/response/log-reopen"
|
33
|
+
require "groonga/client/response/logical-range-filter"
|
34
|
+
require "groonga/client/response/logical-select"
|
33
35
|
require "groonga/client/response/quit"
|
34
36
|
require "groonga/client/response/register"
|
35
37
|
require "groonga/client/response/schema"
|
@@ -1,7 +1,5 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
#
|
3
1
|
# Copyright (C) 2013 Haruka Yoshihara <yoshihara@clear-code.com>
|
4
|
-
# Copyright (C) 2013-
|
2
|
+
# Copyright (C) 2013-2018 Kouhei Sutou <kou@clear-code.com>
|
5
3
|
#
|
6
4
|
# This library is free software; you can redistribute it and/or
|
7
5
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -17,9 +15,14 @@
|
|
17
15
|
# License along with this library; if not, write to the Free Software
|
18
16
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
19
17
|
|
18
|
+
require "csv"
|
20
19
|
require "rexml/document"
|
21
20
|
require "json"
|
22
21
|
|
22
|
+
begin
|
23
|
+
require "arrow"
|
24
|
+
rescue LoadError
|
25
|
+
end
|
23
26
|
require "hashie"
|
24
27
|
|
25
28
|
module Groonga
|
@@ -66,7 +69,13 @@ module Groonga
|
|
66
69
|
return_code = nil
|
67
70
|
case command.output_type
|
68
71
|
when :json
|
69
|
-
|
72
|
+
callback = command["callback"]
|
73
|
+
if callback and
|
74
|
+
/\A#{Regexp.escape(callback)}\((.+)\);\z/ =~ raw_response
|
75
|
+
response = JSON.parse($1)
|
76
|
+
else
|
77
|
+
response = JSON.parse(raw_response)
|
78
|
+
end
|
70
79
|
if response.is_a?(::Array)
|
71
80
|
header, body = response
|
72
81
|
return_code = header[0] if header
|
@@ -78,6 +87,12 @@ module Groonga
|
|
78
87
|
when :xml
|
79
88
|
header, body = parse_xml(raw_response)
|
80
89
|
return_code = header[0] if header
|
90
|
+
when :tsv
|
91
|
+
header, body = parse_tsv(raw_response)
|
92
|
+
return_code = header["return_code"] if header
|
93
|
+
when :arrow, :"apache-arrow"
|
94
|
+
header, body = parse_apache_arrow(raw_response)
|
95
|
+
return_code = header["return_code"] if header
|
81
96
|
else
|
82
97
|
header = nil
|
83
98
|
body = raw_response
|
@@ -129,6 +144,80 @@ module Groonga
|
|
129
144
|
end
|
130
145
|
end
|
131
146
|
end
|
147
|
+
|
148
|
+
def parse_tsv(response)
|
149
|
+
tsv = CSV.new(response, col_sep: "\t")
|
150
|
+
raw_header = tsv.shift
|
151
|
+
return nil, nil if raw_header.nil?
|
152
|
+
|
153
|
+
header = parse_tsv_header(raw_header)
|
154
|
+
return header, nil unless header["return_code"].zero?
|
155
|
+
|
156
|
+
body = parse_tsv_body(tsv)
|
157
|
+
[header, body]
|
158
|
+
end
|
159
|
+
|
160
|
+
def parse_tsv_header(raw_header)
|
161
|
+
header = {
|
162
|
+
"return_code" => Integer(raw_header[0], 10),
|
163
|
+
"start_time" => Float(raw_header[1]),
|
164
|
+
"elapsed_time" => Float(raw_header[2]),
|
165
|
+
}
|
166
|
+
if raw_header.size >= 4
|
167
|
+
header["error"] = {
|
168
|
+
"message" => raw_header[3],
|
169
|
+
}
|
170
|
+
if raw_header.size >= 5
|
171
|
+
header["error"]["function"] = raw_header[4]
|
172
|
+
header["error"]["file"] = raw_header[5]
|
173
|
+
header["error"]["line"] = Integer(raw_header[6])
|
174
|
+
end
|
175
|
+
end
|
176
|
+
header
|
177
|
+
end
|
178
|
+
|
179
|
+
def parse_tsv_body(tsv)
|
180
|
+
body = []
|
181
|
+
tsv.each do |row|
|
182
|
+
break if row.size == 1 and row[0] == "END"
|
183
|
+
body << row
|
184
|
+
end
|
185
|
+
body
|
186
|
+
end
|
187
|
+
|
188
|
+
def parse_apache_arrow(response)
|
189
|
+
header = nil
|
190
|
+
body = nil
|
191
|
+
buffer = Arrow::Buffer.new(response)
|
192
|
+
Arrow::BufferInputStream.open(buffer) do |input|
|
193
|
+
while input.tell < response.bytesize
|
194
|
+
reader = Arrow::RecordBatchStreamReader.new(input)
|
195
|
+
schema = reader.schema
|
196
|
+
record_batches = reader.to_a
|
197
|
+
if apache_arrow_metadata?(schema)
|
198
|
+
table = Arrow::Table.new(schema, record_batches)
|
199
|
+
header = table.each_record.first.to_h
|
200
|
+
else
|
201
|
+
body = {}
|
202
|
+
body["columns"] = schema.fields.collect do |field|
|
203
|
+
[field.name, field.data_type.to_s]
|
204
|
+
end
|
205
|
+
if record_batches.empty?
|
206
|
+
records = []
|
207
|
+
else
|
208
|
+
table = Arrow::Table.new(schema, record_batches)
|
209
|
+
records = table.raw_records
|
210
|
+
end
|
211
|
+
body["records"] = records
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
return header, body
|
216
|
+
end
|
217
|
+
|
218
|
+
def apache_arrow_metadata?(schema)
|
219
|
+
(schema.metadata || {})["GROONGA:data_type"] == "metadata"
|
220
|
+
end
|
132
221
|
end
|
133
222
|
|
134
223
|
# @return [Groonga::Command] The command for the request.
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# Copyright (C) 2019 Sutou Kouhei <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
module Groonga
|
18
|
+
class Client
|
19
|
+
module Response
|
20
|
+
module Drilldownable
|
21
|
+
# @return [::Array<Groonga::Client::Response::Select::Drilldown>,
|
22
|
+
# ::Hash<String, Groonga::Client::Response::Select::Drilldown>]
|
23
|
+
# If labeled drilldowns are used or command version 3 or
|
24
|
+
# later is used, `{"label1" => drilldown1, "label2" => drilldown2}`
|
25
|
+
# is returned since 0.3.1.
|
26
|
+
#
|
27
|
+
# Otherwise, `[drilldown1, drilldown2]` is returned.
|
28
|
+
attr_accessor :drilldowns
|
29
|
+
|
30
|
+
private
|
31
|
+
def parse_drilldown(label, keys, raw_drilldown)
|
32
|
+
if raw_drilldown.is_a?(::Array)
|
33
|
+
n_hits = raw_drilldown[0][0]
|
34
|
+
raw_columns = raw_drilldown[1]
|
35
|
+
raw_records = raw_drilldown[2..-1]
|
36
|
+
else
|
37
|
+
n_hits = raw_drilldown["n_hits"]
|
38
|
+
raw_columns = raw_drilldown["columns"]
|
39
|
+
raw_records = raw_drilldown["records"]
|
40
|
+
end
|
41
|
+
records = parse_records(raw_columns, raw_records)
|
42
|
+
Drilldown.new(label,
|
43
|
+
keys,
|
44
|
+
n_hits,
|
45
|
+
records,
|
46
|
+
raw_columns,
|
47
|
+
raw_records)
|
48
|
+
end
|
49
|
+
|
50
|
+
def parse_drilldowns(keys, raw_drilldowns)
|
51
|
+
(raw_drilldowns || []).collect.with_index do |raw_drilldown, i|
|
52
|
+
key = keys[i]
|
53
|
+
parse_drilldown(key, [key], raw_drilldown)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def parse_labeled_drilldowns(labeled_drilldown_requests,
|
58
|
+
raw_drilldowns)
|
59
|
+
drilldowns = {}
|
60
|
+
(raw_drilldowns || {}).each do |label, raw_drilldown|
|
61
|
+
labeled_drilldown_request = labeled_drilldown_requests[label]
|
62
|
+
drilldowns[label] = parse_drilldown(label,
|
63
|
+
labeled_drilldown_request.keys,
|
64
|
+
raw_drilldown)
|
65
|
+
end
|
66
|
+
drilldowns
|
67
|
+
end
|
68
|
+
|
69
|
+
class Drilldown < Struct.new(:label,
|
70
|
+
:keys,
|
71
|
+
:n_hits,
|
72
|
+
:records,
|
73
|
+
:raw_columns,
|
74
|
+
:raw_records)
|
75
|
+
# @deprecated since 0.2.6. Use {#records} instead.
|
76
|
+
alias_method :items, :records
|
77
|
+
|
78
|
+
def key
|
79
|
+
keys.join(", ")
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -20,10 +20,50 @@ module Groonga
|
|
20
20
|
class Client
|
21
21
|
module Response
|
22
22
|
class Error < Base
|
23
|
-
# @return [String] The error message of the error response.
|
23
|
+
# @return [String, nil] The error message of the error response.
|
24
|
+
#
|
24
25
|
# @since 0.1.0
|
25
26
|
def message
|
26
|
-
|
27
|
+
error_message
|
28
|
+
end
|
29
|
+
|
30
|
+
# @return [String, nil] The function name where the error is occurred.
|
31
|
+
#
|
32
|
+
# @since 0.5.9
|
33
|
+
def function
|
34
|
+
if header.nil?
|
35
|
+
nil
|
36
|
+
elsif header_v1?
|
37
|
+
header[4]
|
38
|
+
else
|
39
|
+
(header["error"] || {})["function"]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# @return [String, nil] The file name where the error is occurred.
|
44
|
+
#
|
45
|
+
# @since 0.5.9
|
46
|
+
def file
|
47
|
+
if header.nil?
|
48
|
+
nil
|
49
|
+
elsif header_v1?
|
50
|
+
header[5]
|
51
|
+
else
|
52
|
+
(header["error"] || {})["file"]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# @return [String, nil] The line where the error is occurred.
|
57
|
+
#
|
58
|
+
# @since 0.5.9
|
59
|
+
def line
|
60
|
+
if header.nil?
|
61
|
+
nil
|
62
|
+
elsif header_v1?
|
63
|
+
header[5]
|
64
|
+
else
|
65
|
+
(header["error"] || {})["line"]
|
66
|
+
end
|
27
67
|
end
|
28
68
|
end
|
29
69
|
end
|