fluent-plugin-groonga 1.1.7 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/configuration.md +19 -0
- data/doc/text/constitution.md +23 -19
- data/doc/text/news.md +7 -0
- data/fluent-plugin-groonga.gemspec +4 -3
- data/lib/fluent/plugin/in_groonga.rb +16 -10
- data/lib/fluent/plugin/out_groonga.rb +10 -13
- data/sample/command.conf +3 -1
- data/sample/gqtp.conf +3 -1
- data/sample/http.conf +3 -1
- data/test/test_input.rb +55 -1
- data/test/test_output.rb +41 -3
- metadata +14 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f225a28b0bd052b37c5180b7707ac898f550ca57
|
4
|
+
data.tar.gz: 0c93017fa42a9a2edac9b50d97d0e3d3e5cf0a94
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: deb8587173008cf6ddc636aebe1fd7c2eb41ed01b806e91c020af81c1dc38eeccdbac196391ecc2289354458d3172707db85f9d9865a7690bc18f8ff00e917df
|
7
|
+
data.tar.gz: 737f5f9db324b14199f4e2b1734d7bc76e1008b80906eba78febe99f0aae91a428a1db69d809715661473bbc4e4252557be4f75512796f42df6f178195eb89ef
|
data/doc/text/configuration.md
CHANGED
@@ -32,6 +32,24 @@ Here are available parameters:
|
|
32
32
|
|
33
33
|
* default: `10041`
|
34
34
|
|
35
|
+
* `command_name_position`: It specifies where Groonga command's
|
36
|
+
name.
|
37
|
+
|
38
|
+
If `tag` is specified, the plugin puts Groonga command's name to
|
39
|
+
tag as `groonga.command.#{COMMAND_NAME}` format and record
|
40
|
+
is the arguments of the command.
|
41
|
+
|
42
|
+
If `record` is specified, the plugin puts both Groonga command's
|
43
|
+
name and arguments to record as `{"name": "#{COMMAND_NAME}",
|
44
|
+
"arguments": {...}}` format. Tag is always `groonga.command`.
|
45
|
+
|
46
|
+
`record` is suitable when you want to implement replication. If
|
47
|
+
you `tag` for replication, Groonga command's order may be changed.
|
48
|
+
|
49
|
+
* Available values: `tag`, `record`
|
50
|
+
|
51
|
+
* default: `tag`
|
52
|
+
|
35
53
|
* `emit_commands`: TODO
|
36
54
|
|
37
55
|
Here is an example:
|
@@ -43,6 +61,7 @@ Here is an example:
|
|
43
61
|
port 10041
|
44
62
|
real_host 192.168.0.1
|
45
63
|
real_port 10041
|
64
|
+
command_name_position record
|
46
65
|
</source>
|
47
66
|
|
48
67
|
## The `groonga` output plugin
|
data/doc/text/constitution.md
CHANGED
@@ -66,19 +66,21 @@ Here is an example configuration file:
|
|
66
66
|
# For master Groonga server
|
67
67
|
<source>
|
68
68
|
@type groonga
|
69
|
-
protocol gqtp
|
69
|
+
protocol gqtp # Or use the below line
|
70
70
|
# protocol http
|
71
|
-
bind 127.0.0.1
|
72
|
-
# bind 192.168.0.1
|
71
|
+
bind 127.0.0.1 # For client side Fluentd
|
72
|
+
# bind 192.168.0.1 # For master Groonga server side Fluentd
|
73
73
|
port 10041
|
74
|
-
real_host 192.168.29.1
|
75
|
-
real_port 10041
|
76
|
-
# real_port 20041
|
77
|
-
|
74
|
+
real_host 192.168.29.1 # IP address of master Groonga server
|
75
|
+
real_port 10041 # Port number of master Groonga server
|
76
|
+
# real_port 20041 # Use different port number
|
77
|
+
# for master Groonga server side Fluentd
|
78
|
+
|
79
|
+
command_name_position record # To keep command order
|
78
80
|
</source>
|
79
81
|
|
80
82
|
# For slave Groonga server
|
81
|
-
<match groonga.command
|
83
|
+
<match groonga.command.**>
|
82
84
|
@type groonga
|
83
85
|
protocol gqtp # Or use the below line
|
84
86
|
# protocol http # You can use different protocol for
|
@@ -126,7 +128,7 @@ You cannot update data until fluentd is up.
|
|
126
128
|
Here are recover steps when master Groonga server is down:
|
127
129
|
|
128
130
|
1. Stop fluentd.
|
129
|
-
2. Run `
|
131
|
+
2. Run `groonga /PATH/TO/SLAVE/GROONGA/SERVER/DB dump >
|
130
132
|
SLAVE_GROONGA_DUMP.grn` on slave Groonga server host.
|
131
133
|
3. Run `groonga -n /PATH/TO/MASTER/GROONGA/SERVER/DB <
|
132
134
|
SLAVE_GROONGA_DUMP.grn` on master Groonga server.
|
@@ -139,7 +141,7 @@ You cannot update data until you finish to recover.
|
|
139
141
|
|
140
142
|
Here are recover steps when slave Groonga server is down:
|
141
143
|
|
142
|
-
1. Run `
|
144
|
+
1. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
|
143
145
|
MASTER_GROONGA_DUMP.grn` on master Groonga server host.
|
144
146
|
2. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
|
145
147
|
MASTER_GROONGA_DUMP.grn` on slave Groonga server.
|
@@ -155,7 +157,7 @@ is full (see `buffer_queue_limit`) or fluentd gives up retrying (see
|
|
155
157
|
Groonga server before those situations:
|
156
158
|
|
157
159
|
1. Stop fluentd.
|
158
|
-
2. Run `
|
160
|
+
2. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
|
159
161
|
MASTER_GROONGA_DUMP.grn` on master Groonga server host.
|
160
162
|
3. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
|
161
163
|
MASTER_GROONGA_DUMP.grn` on slave Groonga server host.
|
@@ -194,19 +196,21 @@ Here is an example configuration file:
|
|
194
196
|
# For master Groonga server
|
195
197
|
<source>
|
196
198
|
@type groonga
|
197
|
-
protocol gqtp
|
199
|
+
protocol gqtp # Or use the below line
|
198
200
|
# protocol http
|
199
|
-
bind 127.0.0.1
|
200
|
-
# bind 192.168.0.1
|
201
|
+
bind 127.0.0.1 # For client side Fluentd
|
202
|
+
# bind 192.168.0.1 # For master Groonga server side Fluentd
|
201
203
|
port 10041
|
202
|
-
real_host 192.168.29.1
|
203
|
-
real_port 10041
|
204
|
-
# real_port 20041
|
205
|
-
|
204
|
+
real_host 192.168.29.1 # IP address of master Groonga server
|
205
|
+
real_port 10041 # Port number of master Groonga server
|
206
|
+
# real_port 20041 # Use different port number
|
207
|
+
# for master Groonga server side fluentd
|
208
|
+
|
209
|
+
command_name_position record # To keep command order
|
206
210
|
</source>
|
207
211
|
|
208
212
|
# For slave Groonga servers
|
209
|
-
<match groonga.command
|
213
|
+
<match groonga.command.**>
|
210
214
|
@type copy
|
211
215
|
|
212
216
|
# The first slave Groonga server
|
data/doc/text/news.md
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- mode: ruby; coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C)
|
3
|
+
# Copyright (C) 2018 Yasuhiro Horimoto <horimoto@clear-code.com>
|
4
|
+
# Copyright (C) 2012-2017 Kouhei Sutou <kou@clear-code.com>
|
4
5
|
#
|
5
6
|
# This library is free software; you can redistribute it and/or
|
6
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -17,7 +18,7 @@
|
|
17
18
|
|
18
19
|
Gem::Specification.new do |spec|
|
19
20
|
spec.name = "fluent-plugin-groonga"
|
20
|
-
spec.version = "1.1.
|
21
|
+
spec.version = "1.1.8"
|
21
22
|
spec.authors = ["Kouhei Sutou"]
|
22
23
|
spec.email = ["kou@clear-code.com"]
|
23
24
|
spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
|
@@ -34,7 +35,7 @@ Gem::Specification.new do |spec|
|
|
34
35
|
spec.test_files += Dir.glob("test/**/*")
|
35
36
|
spec.require_paths = ["lib"]
|
36
37
|
|
37
|
-
spec.add_runtime_dependency("fluentd")
|
38
|
+
spec.add_runtime_dependency("fluentd", ">= 0.12.20", "< 0.14.0")
|
38
39
|
spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
|
39
40
|
spec.add_runtime_dependency("groonga-command-parser")
|
40
41
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# Copyright (C) 2018 Yasuhiro Horimoto <horimoto@clear-code.com>
|
1
2
|
# Copyright (C) 2012-2017 Kouhei Sutou <kou@clear-code.com>
|
2
3
|
#
|
3
4
|
# This library is free software; you can redistribute it and/or
|
@@ -33,14 +34,8 @@ module Fluent
|
|
33
34
|
super
|
34
35
|
end
|
35
36
|
|
36
|
-
config_param :protocol, :
|
37
|
-
|
38
|
-
when "http", "gqtp"
|
39
|
-
value.to_sym
|
40
|
-
else
|
41
|
-
raise ConfigError, "must be http or gqtp: <#{value}>"
|
42
|
-
end
|
43
|
-
end
|
37
|
+
config_param :protocol, :enum, :list => [:http, :gqtp], :default => :http
|
38
|
+
config_param :command_name_position, :enum, :list => [:tag, :record], :default => :tag
|
44
39
|
|
45
40
|
def configure(conf)
|
46
41
|
super
|
@@ -171,9 +166,20 @@ module Fluent
|
|
171
166
|
def emit(command, params)
|
172
167
|
normalized_command = command.split(".")[0]
|
173
168
|
return unless emit_command?(normalized_command)
|
174
|
-
@input_plugin.
|
169
|
+
case @input_plugin.command_name_position
|
170
|
+
when :tag
|
171
|
+
tag = "groonga.command.#{normalized_command}"
|
172
|
+
record = params
|
173
|
+
else
|
174
|
+
tag = "groonga.command"
|
175
|
+
record = {
|
176
|
+
"name" => normalized_command,
|
177
|
+
"arguments" => params
|
178
|
+
}
|
179
|
+
end
|
180
|
+
@input_plugin.router.emit(tag,
|
175
181
|
Engine.now,
|
176
|
-
|
182
|
+
record)
|
177
183
|
end
|
178
184
|
|
179
185
|
private
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C)
|
3
|
+
# Copyright (C) 2018 Yasuhiro Horimoto <horimoto@clear-code.com>
|
4
|
+
# Copyright (C) 2012-2017 Kouhei Sutou <kou@clear-code.com>
|
4
5
|
#
|
5
6
|
# This library is free software; you can redistribute it and/or
|
6
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -29,15 +30,7 @@ module Fluent
|
|
29
30
|
super
|
30
31
|
end
|
31
32
|
|
32
|
-
config_param :protocol, :
|
33
|
-
case value
|
34
|
-
when "http", "gqtp", "command"
|
35
|
-
value.to_sym
|
36
|
-
else
|
37
|
-
raise ConfigError, "must be http, gqtp or command: <#{value}>"
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
33
|
+
config_param :protocol, :enum, :list => [:http, :gqtp, :command], :default => :http
|
41
34
|
# alias is just for backward compatibility
|
42
35
|
config_param :store_table, :string, :default => nil, :alias => :table
|
43
36
|
|
@@ -340,8 +333,7 @@ module Fluent
|
|
340
333
|
@columns = {}
|
341
334
|
column_list.each do |column|
|
342
335
|
name = column.name
|
343
|
-
|
344
|
-
@columns[name] = Column.new(name, column.range, vector_p)
|
336
|
+
@columns[name] = Column.new(name, column.range, column.vector?)
|
345
337
|
ensure_column_indexes(name)
|
346
338
|
end
|
347
339
|
end
|
@@ -575,13 +567,18 @@ module Fluent
|
|
575
567
|
records = []
|
576
568
|
chunk.msgpack_each do |message|
|
577
569
|
tag, _, record = message
|
578
|
-
|
570
|
+
case tag
|
571
|
+
when /\Agroonga\.command\./
|
579
572
|
name = $POSTMATCH
|
580
573
|
unless records.empty?
|
581
574
|
store_records(records)
|
582
575
|
records.clear
|
583
576
|
end
|
584
577
|
@client.execute(name, record)
|
578
|
+
when "groonga.command"
|
579
|
+
name = record["name"]
|
580
|
+
arguments = record["arguments"]
|
581
|
+
@client.execute(name, arguments)
|
585
582
|
else
|
586
583
|
records << record
|
587
584
|
end
|
data/sample/command.conf
CHANGED
data/sample/gqtp.conf
CHANGED
data/sample/http.conf
CHANGED
data/test/test_input.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C)
|
3
|
+
# Copyright (C) 2018 Yasuhiro Horimoto <horimoto@clear-code.com>
|
4
|
+
# Copyright (C) 2012-2017 Kouhei Sutou <kou@clear-code.com>
|
4
5
|
#
|
5
6
|
# This library is free software; you can redistribute it and/or
|
6
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -153,6 +154,59 @@ EOJ
|
|
153
154
|
end
|
154
155
|
end
|
155
156
|
|
157
|
+
sub_test_case("command_name_position") do
|
158
|
+
sub_test_case("record") do
|
159
|
+
def configuration
|
160
|
+
<<-CONFIGURATION
|
161
|
+
#{super}
|
162
|
+
command_name_position record
|
163
|
+
CONFIGURATION
|
164
|
+
end
|
165
|
+
|
166
|
+
def test_not_load
|
167
|
+
@real_response["Content-Type"] = "application/json"
|
168
|
+
@real_response.body = JSON.generate([[0, 0.0, 0.0], true])
|
169
|
+
@driver.expect_emit("groonga.command",
|
170
|
+
@now,
|
171
|
+
{
|
172
|
+
"name" => "table_create",
|
173
|
+
"arguments" => {
|
174
|
+
"name" => "Users",
|
175
|
+
"flags" => "TABLE_NO_KEY",
|
176
|
+
}
|
177
|
+
})
|
178
|
+
@driver.run do
|
179
|
+
get("/d/table_create", "name" => "Users", "flags" => "TABLE_NO_KEY")
|
180
|
+
assert_equal("200", @last_response.code)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def test_load
|
185
|
+
@real_response["Content-Type"] = "application/json"
|
186
|
+
@real_response.body = JSON.generate([[0, 0.0, 0.0], true])
|
187
|
+
json = <<-JSON
|
188
|
+
[
|
189
|
+
{"name": "Alice"},
|
190
|
+
{"name": "Bob"}
|
191
|
+
]
|
192
|
+
JSON
|
193
|
+
@driver.expect_emit("groonga.command",
|
194
|
+
@now,
|
195
|
+
{
|
196
|
+
"name" => "load",
|
197
|
+
"arguments" => {
|
198
|
+
"table" => "Users",
|
199
|
+
"values" => json,
|
200
|
+
}
|
201
|
+
})
|
202
|
+
@driver.run do
|
203
|
+
post("/d/load", json, "table" => "Users")
|
204
|
+
assert_equal("200", @last_response.code)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
156
210
|
private
|
157
211
|
def get(path, parameters={})
|
158
212
|
http = Net::HTTP.new(@host, @port)
|
data/test/test_output.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C)
|
3
|
+
# Copyright (C) 2018 Yasuhiro Horimoto <horimoto@clear-code.com>
|
4
|
+
# Copyright (C) 2012-2017 Kouhei Sutou <kou@clear-code.com>
|
4
5
|
#
|
5
6
|
# This library is free software; you can redistribute it and/or
|
6
7
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -96,7 +97,7 @@ EOC
|
|
96
97
|
end
|
97
98
|
|
98
99
|
class CommandTest < self
|
99
|
-
def
|
100
|
+
def test_command_name_position_tag
|
100
101
|
@response_body = JSON.generate([[0, 0.0, 0.0], true])
|
101
102
|
driver = create_driver("groonga.command.table_create")
|
102
103
|
time = Time.parse("2012-10-26T08:45:42Z").to_i
|
@@ -105,6 +106,21 @@ EOC
|
|
105
106
|
assert_equal("/d/table_create?name=Users",
|
106
107
|
@request_parser.request_url)
|
107
108
|
end
|
109
|
+
|
110
|
+
def test_command_name_position_record
|
111
|
+
@response_body = JSON.generate([[0, 0.0, 0.0], true])
|
112
|
+
driver = create_driver("groonga.command")
|
113
|
+
time = Time.parse("2012-10-26T08:45:42Z").to_i
|
114
|
+
driver.emit({
|
115
|
+
"name" => "table_create",
|
116
|
+
"arguments" => {
|
117
|
+
"name" => "Users"
|
118
|
+
}
|
119
|
+
}, time)
|
120
|
+
driver.run
|
121
|
+
assert_equal("/d/table_create?name=Users",
|
122
|
+
@request_parser.request_url)
|
123
|
+
end
|
108
124
|
end
|
109
125
|
|
110
126
|
class StoreTest < self
|
@@ -228,7 +244,7 @@ EOC
|
|
228
244
|
end
|
229
245
|
|
230
246
|
class CommandTest < self
|
231
|
-
def
|
247
|
+
def test_command_name_position_tag
|
232
248
|
driver = create_driver("groonga.command.table_create")
|
233
249
|
time = Time.parse("2012-10-26T08:45:42Z").to_i
|
234
250
|
driver.emit({"name" => "Users"}, time)
|
@@ -246,6 +262,28 @@ EOC
|
|
246
262
|
actual_input,
|
247
263
|
])
|
248
264
|
end
|
265
|
+
|
266
|
+
def test_command_name_position_record
|
267
|
+
driver = create_driver("groonga.command")
|
268
|
+
time = Time.parse("2012-10-26T08:45:42Z").to_i
|
269
|
+
driver.emit({
|
270
|
+
"name" => "table_create",
|
271
|
+
"arguments" => {"name" => "Users"}
|
272
|
+
}, time)
|
273
|
+
driver.run
|
274
|
+
assert_equal([
|
275
|
+
[
|
276
|
+
"--input-fd", actual_input_fd,
|
277
|
+
"--output-fd", actual_output_fd,
|
278
|
+
"-n", @database_path,
|
279
|
+
],
|
280
|
+
"/d/table_create?name=Users\n",
|
281
|
+
],
|
282
|
+
[
|
283
|
+
actual_command_line,
|
284
|
+
actual_input,
|
285
|
+
])
|
286
|
+
end
|
249
287
|
end
|
250
288
|
end
|
251
289
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-groonga
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,14 +16,20 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.12.20
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 0.14.0
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
29
|
+
version: 0.12.20
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 0.14.0
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: groonga-client
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -186,15 +192,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
186
192
|
version: '0'
|
187
193
|
requirements: []
|
188
194
|
rubyforge_project:
|
189
|
-
rubygems_version: 2.
|
195
|
+
rubygems_version: 2.5.2.1
|
190
196
|
signing_key:
|
191
197
|
specification_version: 4
|
192
198
|
summary: Fluentd plugin to store data into Groonga and implement Groonga replication
|
193
199
|
system.
|
194
200
|
test_files:
|
195
201
|
- test/test_output.rb
|
196
|
-
- test/output/test_table_index_definition.rb
|
197
|
-
- test/output/test_type_guesser.rb
|
198
|
-
- test/output/test_table_definition.rb
|
199
202
|
- test/run-test.rb
|
203
|
+
- test/output/test_table_definition.rb
|
204
|
+
- test/output/test_type_guesser.rb
|
205
|
+
- test/output/test_table_index_definition.rb
|
200
206
|
- test/test_input.rb
|