fluent-plugin-groonga 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/configuration.md +19 -0
- data/doc/text/constitution.md +23 -19
- data/doc/text/news.md +12 -0
- data/fluent-plugin-groonga.gemspec +1 -1
- data/lib/fluent/plugin/in_groonga.rb +15 -3
- data/lib/fluent/plugin/out_groonga.rb +23 -5
- data/sample/command.conf +3 -1
- data/sample/gqtp.conf +3 -1
- data/sample/http.conf +3 -1
- data/test/test_input.rb +64 -1
- data/test/test_output.rb +430 -10
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f26f1229bbc925451a26e486e53dded226d90e4
|
4
|
+
data.tar.gz: dedef4594068fddbcda0866e44f1e16ecbb2902c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50e0c5d734a4e1d601486ad33814f313363612f9a0e853d4128f41c502d5a38a5fa73183f3c6a64bbda8e4307173e57d1e67506e75115288f1888002f95f97f9
|
7
|
+
data.tar.gz: d203a60ffc2832960b909e89c8106d1d0e038be03681f1c769feed19871d3ba55f6f5dbf3db1aed9a84eedf7631799080ed54283fa06e6f334a9e3baf60994b1
|
data/doc/text/configuration.md
CHANGED
@@ -50,6 +50,24 @@ Here are available parameters in `source` directive:
|
|
50
50
|
|
51
51
|
* default: `10041`
|
52
52
|
|
53
|
+
* `command_name_position`: It specifies where Groonga command's
|
54
|
+
name.
|
55
|
+
|
56
|
+
If `tag` is specified, the plugin puts Groonga command's name to
|
57
|
+
tag as `groonga.command.#{COMMAND_NAME}` format and record
|
58
|
+
is the arguments of the command.
|
59
|
+
|
60
|
+
If `record` is specified, the plugin puts both Groonga command's
|
61
|
+
name and arguments to record as `{"name": "#{COMMAND_NAME}",
|
62
|
+
"arguments": {...}}` format. Tag is always `groonga.command`.
|
63
|
+
|
64
|
+
`record` is suitable when you want to implement replication. If
|
65
|
+
you `tag` for replication, Groonga command's order may be changed.
|
66
|
+
|
67
|
+
* Available values: `tag`, `record`
|
68
|
+
|
69
|
+
* default: `tag`
|
70
|
+
|
53
71
|
* `emit_commands`: TODO
|
54
72
|
|
55
73
|
Here is an example:
|
@@ -65,6 +83,7 @@ Here is an example:
|
|
65
83
|
port 10041
|
66
84
|
real_host 192.168.0.1
|
67
85
|
real_port 10041
|
86
|
+
command_name_position record
|
68
87
|
</source>
|
69
88
|
|
70
89
|
## The `groonga` output plugin
|
data/doc/text/constitution.md
CHANGED
@@ -66,19 +66,21 @@ Here is an example configuration file:
|
|
66
66
|
# For master Groonga server
|
67
67
|
<source>
|
68
68
|
@type groonga
|
69
|
-
protocol gqtp
|
69
|
+
protocol gqtp # Or use the below line
|
70
70
|
# protocol http
|
71
|
-
bind 127.0.0.1
|
72
|
-
# bind 192.168.0.1
|
71
|
+
bind 127.0.0.1 # For client side Fluentd
|
72
|
+
# bind 192.168.0.1 # For master Groonga server side Fluentd
|
73
73
|
port 10041
|
74
|
-
real_host 192.168.29.1
|
75
|
-
real_port 10041
|
76
|
-
# real_port 20041
|
77
|
-
|
74
|
+
real_host 192.168.29.1 # IP address of master Groonga server
|
75
|
+
real_port 10041 # Port number of master Groonga server
|
76
|
+
# real_port 20041 # Use different port number
|
77
|
+
# for master Groonga server side Fluentd
|
78
|
+
|
79
|
+
command_name_position record # To keep command order
|
78
80
|
</source>
|
79
81
|
|
80
82
|
# For slave Groonga server
|
81
|
-
<match groonga.command
|
83
|
+
<match groonga.command.**>
|
82
84
|
@type groonga
|
83
85
|
protocol gqtp # Or use the below line
|
84
86
|
# protocol http # You can use different protocol for
|
@@ -126,7 +128,7 @@ You cannot update data until fluentd is up.
|
|
126
128
|
Here are recover steps when master Groonga server is down:
|
127
129
|
|
128
130
|
1. Stop fluentd.
|
129
|
-
2. Run `
|
131
|
+
2. Run `groonga /PATH/TO/SLAVE/GROONGA/SERVER/DB dump >
|
130
132
|
SLAVE_GROONGA_DUMP.grn` on slave Groonga server host.
|
131
133
|
3. Run `groonga -n /PATH/TO/MASTER/GROONGA/SERVER/DB <
|
132
134
|
SLAVE_GROONGA_DUMP.grn` on master Groonga server.
|
@@ -139,7 +141,7 @@ You cannot update data until you finish to recover.
|
|
139
141
|
|
140
142
|
Here are recover steps when slave Groonga server is down:
|
141
143
|
|
142
|
-
1. Run `
|
144
|
+
1. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
|
143
145
|
MASTER_GROONGA_DUMP.grn` on master Groonga server host.
|
144
146
|
2. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
|
145
147
|
MASTER_GROONGA_DUMP.grn` on slave Groonga server.
|
@@ -155,7 +157,7 @@ is full (see `buffer_queue_limit`) or fluentd gives up retrying (see
|
|
155
157
|
Groonga server before those situations:
|
156
158
|
|
157
159
|
1. Stop fluentd.
|
158
|
-
2. Run `
|
160
|
+
2. Run `groonga /PATH/TO/MASTER/GROONGA/SERVER/DB dump >
|
159
161
|
MASTER_GROONGA_DUMP.grn` on master Groonga server host.
|
160
162
|
3. Run `groonga -n /PATH/TO/SLAVE/GROONGA/SERVER/DB <
|
161
163
|
MASTER_GROONGA_DUMP.grn` on slave Groonga server host.
|
@@ -194,19 +196,21 @@ Here is an example configuration file:
|
|
194
196
|
# For master Groonga server
|
195
197
|
<source>
|
196
198
|
@type groonga
|
197
|
-
protocol gqtp
|
199
|
+
protocol gqtp # Or use the below line
|
198
200
|
# protocol http
|
199
|
-
bind 127.0.0.1
|
200
|
-
# bind 192.168.0.1
|
201
|
+
bind 127.0.0.1 # For client side Fluentd
|
202
|
+
# bind 192.168.0.1 # For master Groonga server side Fluentd
|
201
203
|
port 10041
|
202
|
-
real_host 192.168.29.1
|
203
|
-
real_port 10041
|
204
|
-
# real_port 20041
|
205
|
-
|
204
|
+
real_host 192.168.29.1 # IP address of master Groonga server
|
205
|
+
real_port 10041 # Port number of master Groonga server
|
206
|
+
# real_port 20041 # Use different port number
|
207
|
+
# for master Groonga server side fluentd
|
208
|
+
|
209
|
+
command_name_position record # To keep command order
|
206
210
|
</source>
|
207
211
|
|
208
212
|
# For slave Groonga servers
|
209
|
-
<match groonga.command
|
213
|
+
<match groonga.command.**>
|
210
214
|
@type copy
|
211
215
|
|
212
216
|
# The first slave Groonga server
|
data/doc/text/news.md
CHANGED
@@ -2,6 +2,18 @@
|
|
2
2
|
|
3
3
|
# News
|
4
4
|
|
5
|
+
## 1.2.3: 2018-01-16
|
6
|
+
|
7
|
+
### Improvements
|
8
|
+
|
9
|
+
* in: `command_name_position`: Added a new parameter to control
|
10
|
+
command format. The default behavior isn't changed.
|
11
|
+
|
12
|
+
### Fixes
|
13
|
+
|
14
|
+
* out: Fixed a bug that existing column may be tried to create when
|
15
|
+
Groonga command messages and data load messages are mixed.
|
16
|
+
|
5
17
|
## 1.2.2: 2017-11-22
|
6
18
|
|
7
19
|
### Fixes
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |spec|
|
19
19
|
spec.name = "fluent-plugin-groonga"
|
20
|
-
spec.version = "1.2.
|
20
|
+
spec.version = "1.2.3"
|
21
21
|
spec.authors = ["Kouhei Sutou"]
|
22
22
|
spec.email = ["kou@clear-code.com"]
|
23
23
|
spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2012-
|
1
|
+
# Copyright (C) 2012-2018 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -36,6 +36,7 @@ module Fluent
|
|
36
36
|
end
|
37
37
|
|
38
38
|
config_param :protocol, :enum, :list => [:http, :gqtp], :default => :http
|
39
|
+
config_param :command_name_position, :enum, :list => [:tag, :record], :default => :tag
|
39
40
|
|
40
41
|
def configure(conf)
|
41
42
|
super
|
@@ -157,9 +158,20 @@ module Fluent
|
|
157
158
|
def emit(command, params)
|
158
159
|
normalized_command = command.split(".")[0]
|
159
160
|
return unless emit_command?(normalized_command)
|
160
|
-
@input_plugin.
|
161
|
+
case @input_plugin.command_name_position
|
162
|
+
when :tag
|
163
|
+
tag = "groonga.command.#{normalized_command}"
|
164
|
+
record = params
|
165
|
+
else
|
166
|
+
tag = "groonga.command"
|
167
|
+
record = {
|
168
|
+
"name" => normalized_command,
|
169
|
+
"arguments" => params
|
170
|
+
}
|
171
|
+
end
|
172
|
+
@input_plugin.router.emit(tag,
|
161
173
|
Engine.now,
|
162
|
-
|
174
|
+
record)
|
163
175
|
end
|
164
176
|
|
165
177
|
def log
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2012-
|
1
|
+
# Copyright (C) 2012-2018 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -314,6 +314,11 @@ module Fluent
|
|
314
314
|
end
|
315
315
|
end
|
316
316
|
|
317
|
+
def clear_cache
|
318
|
+
@target_table = nil
|
319
|
+
@columns = nil
|
320
|
+
end
|
321
|
+
|
317
322
|
private
|
318
323
|
def ensure_table
|
319
324
|
return if @target_table
|
@@ -344,8 +349,7 @@ module Fluent
|
|
344
349
|
@columns = {}
|
345
350
|
column_list.each do |column|
|
346
351
|
name = column.name
|
347
|
-
|
348
|
-
@columns[name] = Column.new(name, column.range, vector_p)
|
352
|
+
@columns[name] = Column.new(name, column.range, column.vector?)
|
349
353
|
ensure_column_indexes(name)
|
350
354
|
end
|
351
355
|
end
|
@@ -579,13 +583,27 @@ module Fluent
|
|
579
583
|
records = []
|
580
584
|
chunk.msgpack_each do |message|
|
581
585
|
tag, _, record = message
|
582
|
-
|
586
|
+
name = nil
|
587
|
+
arguments = nil
|
588
|
+
case tag
|
589
|
+
when /\Agroonga\.command\./
|
583
590
|
name = $POSTMATCH
|
591
|
+
arguments = record
|
592
|
+
when "groonga.command"
|
593
|
+
name = record["name"]
|
594
|
+
arguments = record["arguments"]
|
595
|
+
end
|
596
|
+
|
597
|
+
if name
|
584
598
|
unless records.empty?
|
585
599
|
store_records(records)
|
586
600
|
records.clear
|
587
601
|
end
|
588
|
-
@client.execute(name,
|
602
|
+
@client.execute(name, arguments)
|
603
|
+
case name
|
604
|
+
when /\A(?:table|column)_(?:create|remove)/
|
605
|
+
@schema.clear_cache
|
606
|
+
end
|
589
607
|
else
|
590
608
|
records << record
|
591
609
|
end
|
data/sample/command.conf
CHANGED
data/sample/gqtp.conf
CHANGED
data/sample/http.conf
CHANGED
data/test/test_input.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2012-
|
1
|
+
# Copyright (C) 2012-2018 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -159,6 +159,69 @@ EOJ
|
|
159
159
|
end
|
160
160
|
end
|
161
161
|
|
162
|
+
sub_test_case("command_name_position") do
|
163
|
+
sub_test_case("record") do
|
164
|
+
def configuration
|
165
|
+
<<-CONFIGURATION
|
166
|
+
#{super}
|
167
|
+
command_name_position record
|
168
|
+
CONFIGURATION
|
169
|
+
end
|
170
|
+
|
171
|
+
def test_not_load
|
172
|
+
@real_response["Content-Type"] = "application/json"
|
173
|
+
@real_response.body = JSON.generate([[0, 0.0, 0.0], true])
|
174
|
+
@driver.run do
|
175
|
+
get("/d/table_create", "name" => "Users", "flags" => "TABLE_NO_KEY")
|
176
|
+
assert_equal("200", @last_response.code)
|
177
|
+
end
|
178
|
+
assert_equal([
|
179
|
+
[
|
180
|
+
"groonga.command",
|
181
|
+
@now,
|
182
|
+
{
|
183
|
+
"name" => "table_create",
|
184
|
+
"arguments" => {
|
185
|
+
"name" => "Users",
|
186
|
+
"flags" => "TABLE_NO_KEY",
|
187
|
+
}
|
188
|
+
},
|
189
|
+
]
|
190
|
+
],
|
191
|
+
@driver.events)
|
192
|
+
end
|
193
|
+
|
194
|
+
def test_load
|
195
|
+
@real_response["Content-Type"] = "application/json"
|
196
|
+
@real_response.body = JSON.generate([[0, 0.0, 0.0], true])
|
197
|
+
json = <<-JSON
|
198
|
+
[
|
199
|
+
{"name": "Alice"},
|
200
|
+
{"name": "Bob"}
|
201
|
+
]
|
202
|
+
JSON
|
203
|
+
@driver.run do
|
204
|
+
post("/d/load", json, "table" => "Users")
|
205
|
+
assert_equal("200", @last_response.code)
|
206
|
+
end
|
207
|
+
assert_equal([
|
208
|
+
[
|
209
|
+
"groonga.command",
|
210
|
+
@now,
|
211
|
+
{
|
212
|
+
"name" => "load",
|
213
|
+
"arguments" => {
|
214
|
+
"table" => "Users",
|
215
|
+
"values" => json,
|
216
|
+
}
|
217
|
+
},
|
218
|
+
]
|
219
|
+
],
|
220
|
+
@driver.events)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
162
225
|
private
|
163
226
|
def get(path, parameters={})
|
164
227
|
http = Net::HTTP.new(@host, @port)
|
data/test/test_output.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2012-
|
1
|
+
# Copyright (C) 2012-2018 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -44,9 +44,10 @@ EOC
|
|
44
44
|
class HTTPTest < self
|
45
45
|
setup :before => :append
|
46
46
|
def setup_real_server
|
47
|
+
@request_urls = []
|
47
48
|
@request_parser = HTTP::Parser.new
|
48
49
|
@request_body = ""
|
49
|
-
@
|
50
|
+
@response_bodies = []
|
50
51
|
|
51
52
|
@real_host = "127.0.0.1"
|
52
53
|
@real_port = 29292
|
@@ -55,16 +56,15 @@ EOC
|
|
55
56
|
loop do
|
56
57
|
response_config = WEBrick::Config::HTTP.dup.update(:Logger => $log)
|
57
58
|
real_response = WEBrick::HTTPResponse.new(response_config)
|
58
|
-
request_headers = nil
|
59
|
-
request_body = ""
|
60
59
|
client = @real_server.accept
|
61
60
|
@request_parser.on_body = lambda do |chunk|
|
62
61
|
@request_body << chunk
|
63
62
|
end
|
64
63
|
@request_parser.on_message_complete = lambda do
|
65
|
-
real_response.body = @
|
64
|
+
real_response.body = @response_bodies.shift
|
66
65
|
real_response.send_response(client)
|
67
66
|
client.close
|
67
|
+
@request_urls << @request_parser.request_url
|
68
68
|
end
|
69
69
|
|
70
70
|
loop do
|
@@ -92,8 +92,8 @@ EOC
|
|
92
92
|
end
|
93
93
|
|
94
94
|
class CommandTest < self
|
95
|
-
def
|
96
|
-
@
|
95
|
+
def test_command_name_position_tag
|
96
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], true])
|
97
97
|
driver = create_driver
|
98
98
|
time = event_time("2012-10-26T08:45:42Z")
|
99
99
|
driver.run(default_tag: "groonga.command.table_create") do
|
@@ -102,6 +102,23 @@ EOC
|
|
102
102
|
assert_equal("/d/table_create?name=Users",
|
103
103
|
@request_parser.request_url)
|
104
104
|
end
|
105
|
+
|
106
|
+
def test_command_name_position_record
|
107
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], true])
|
108
|
+
driver = create_driver
|
109
|
+
time = event_time("2012-10-26T08:45:42Z")
|
110
|
+
driver.run(default_tag: "groonga.command") do
|
111
|
+
driver.feed(time,
|
112
|
+
{
|
113
|
+
"name" => "table_create",
|
114
|
+
"arguments" => {
|
115
|
+
"name" => "Users",
|
116
|
+
},
|
117
|
+
})
|
118
|
+
end
|
119
|
+
assert_equal("/d/table_create?name=Users",
|
120
|
+
@request_parser.request_url)
|
121
|
+
end
|
105
122
|
end
|
106
123
|
|
107
124
|
class StoreTest < self
|
@@ -112,8 +129,126 @@ EOC
|
|
112
129
|
CONFIGURATION
|
113
130
|
end
|
114
131
|
|
132
|
+
def table_list_response_body
|
133
|
+
[
|
134
|
+
[
|
135
|
+
[
|
136
|
+
"id",
|
137
|
+
"UInt32"
|
138
|
+
],
|
139
|
+
[
|
140
|
+
"name",
|
141
|
+
"ShortText"
|
142
|
+
],
|
143
|
+
[
|
144
|
+
"path",
|
145
|
+
"ShortText"
|
146
|
+
],
|
147
|
+
[
|
148
|
+
"flags",
|
149
|
+
"ShortText"
|
150
|
+
],
|
151
|
+
[
|
152
|
+
"domain",
|
153
|
+
"ShortText"
|
154
|
+
],
|
155
|
+
[
|
156
|
+
"range",
|
157
|
+
"ShortText"
|
158
|
+
],
|
159
|
+
[
|
160
|
+
"default_tokenizer",
|
161
|
+
"ShortText"
|
162
|
+
],
|
163
|
+
[
|
164
|
+
"normalizer",
|
165
|
+
"ShortText"
|
166
|
+
]
|
167
|
+
],
|
168
|
+
[
|
169
|
+
256,
|
170
|
+
"Logs",
|
171
|
+
"//tmp/db/db.0000100",
|
172
|
+
"TABLE_HASH_KEY|PERSISTENT",
|
173
|
+
"ShortText",
|
174
|
+
nil,
|
175
|
+
nil,
|
176
|
+
nil,
|
177
|
+
],
|
178
|
+
]
|
179
|
+
end
|
180
|
+
|
181
|
+
def column_list_response_body
|
182
|
+
[
|
183
|
+
[
|
184
|
+
[
|
185
|
+
"id",
|
186
|
+
"UInt32"
|
187
|
+
],
|
188
|
+
[
|
189
|
+
"name",
|
190
|
+
"ShortText"
|
191
|
+
],
|
192
|
+
[
|
193
|
+
"path",
|
194
|
+
"ShortText"
|
195
|
+
],
|
196
|
+
[
|
197
|
+
"type",
|
198
|
+
"ShortText"
|
199
|
+
],
|
200
|
+
[
|
201
|
+
"flags",
|
202
|
+
"ShortText"
|
203
|
+
],
|
204
|
+
[
|
205
|
+
"domain",
|
206
|
+
"ShortText"
|
207
|
+
],
|
208
|
+
[
|
209
|
+
"range",
|
210
|
+
"ShortText"
|
211
|
+
],
|
212
|
+
[
|
213
|
+
"source",
|
214
|
+
"ShortText"
|
215
|
+
]
|
216
|
+
],
|
217
|
+
[
|
218
|
+
256,
|
219
|
+
"_key",
|
220
|
+
"",
|
221
|
+
"",
|
222
|
+
"COLUMN_SCALAR",
|
223
|
+
"Logs",
|
224
|
+
"ShortText",
|
225
|
+
[
|
226
|
+
]
|
227
|
+
],
|
228
|
+
[
|
229
|
+
257,
|
230
|
+
"message",
|
231
|
+
"/tmp/db/db.0000101",
|
232
|
+
"scalar",
|
233
|
+
"COLUMN_SCALAR",
|
234
|
+
"Logs",
|
235
|
+
"ShortText",
|
236
|
+
[
|
237
|
+
]
|
238
|
+
],
|
239
|
+
]
|
240
|
+
end
|
241
|
+
|
115
242
|
def test_one_message
|
116
|
-
@
|
243
|
+
@response_bodies << JSON.generate([
|
244
|
+
[0, 0.0, 0.0],
|
245
|
+
table_list_response_body,
|
246
|
+
])
|
247
|
+
@response_bodies << JSON.generate([
|
248
|
+
[0, 0.0, 0.0],
|
249
|
+
column_list_response_body,
|
250
|
+
])
|
251
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], [1]])
|
117
252
|
driver = create_driver
|
118
253
|
time = event_time("2012-10-26T08:45:42Z")
|
119
254
|
driver.run(default_tag: "log") do
|
@@ -126,7 +261,15 @@ EOC
|
|
126
261
|
end
|
127
262
|
|
128
263
|
def test_multiple_messages
|
129
|
-
@
|
264
|
+
@response_bodies << JSON.generate([
|
265
|
+
[0, 0.0, 0.0],
|
266
|
+
table_list_response_body,
|
267
|
+
])
|
268
|
+
@response_bodies << JSON.generate([
|
269
|
+
[0, 0.0, 0.0],
|
270
|
+
column_list_response_body,
|
271
|
+
])
|
272
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], [2]])
|
130
273
|
driver = create_driver
|
131
274
|
time = event_time("2012-10-26T08:45:42Z")
|
132
275
|
driver.run(default_tag: "log") do
|
@@ -142,6 +285,257 @@ EOC
|
|
142
285
|
JSON.parse(@request_body))
|
143
286
|
end
|
144
287
|
end
|
288
|
+
|
289
|
+
class MixTest < self
|
290
|
+
def configuration
|
291
|
+
<<-CONFIGURATION
|
292
|
+
#{super}
|
293
|
+
store_table Logs
|
294
|
+
CONFIGURATION
|
295
|
+
end
|
296
|
+
|
297
|
+
|
298
|
+
def table_list_response_body
|
299
|
+
[
|
300
|
+
[
|
301
|
+
[
|
302
|
+
"id",
|
303
|
+
"UInt32"
|
304
|
+
],
|
305
|
+
[
|
306
|
+
"name",
|
307
|
+
"ShortText"
|
308
|
+
],
|
309
|
+
[
|
310
|
+
"path",
|
311
|
+
"ShortText"
|
312
|
+
],
|
313
|
+
[
|
314
|
+
"flags",
|
315
|
+
"ShortText"
|
316
|
+
],
|
317
|
+
[
|
318
|
+
"domain",
|
319
|
+
"ShortText"
|
320
|
+
],
|
321
|
+
[
|
322
|
+
"range",
|
323
|
+
"ShortText"
|
324
|
+
],
|
325
|
+
[
|
326
|
+
"default_tokenizer",
|
327
|
+
"ShortText"
|
328
|
+
],
|
329
|
+
[
|
330
|
+
"normalizer",
|
331
|
+
"ShortText"
|
332
|
+
]
|
333
|
+
],
|
334
|
+
[
|
335
|
+
256,
|
336
|
+
"Logs",
|
337
|
+
"//tmp/db/db.0000100",
|
338
|
+
"TABLE_HASH_KEY|PERSISTENT",
|
339
|
+
"ShortText",
|
340
|
+
nil,
|
341
|
+
nil,
|
342
|
+
nil,
|
343
|
+
],
|
344
|
+
]
|
345
|
+
end
|
346
|
+
|
347
|
+
def column_list_response_body
|
348
|
+
[
|
349
|
+
[
|
350
|
+
[
|
351
|
+
"id",
|
352
|
+
"UInt32"
|
353
|
+
],
|
354
|
+
[
|
355
|
+
"name",
|
356
|
+
"ShortText"
|
357
|
+
],
|
358
|
+
[
|
359
|
+
"path",
|
360
|
+
"ShortText"
|
361
|
+
],
|
362
|
+
[
|
363
|
+
"type",
|
364
|
+
"ShortText"
|
365
|
+
],
|
366
|
+
[
|
367
|
+
"flags",
|
368
|
+
"ShortText"
|
369
|
+
],
|
370
|
+
[
|
371
|
+
"domain",
|
372
|
+
"ShortText"
|
373
|
+
],
|
374
|
+
[
|
375
|
+
"range",
|
376
|
+
"ShortText"
|
377
|
+
],
|
378
|
+
[
|
379
|
+
"source",
|
380
|
+
"ShortText"
|
381
|
+
]
|
382
|
+
],
|
383
|
+
[
|
384
|
+
256,
|
385
|
+
"_key",
|
386
|
+
"",
|
387
|
+
"",
|
388
|
+
"COLUMN_SCALAR",
|
389
|
+
"Logs",
|
390
|
+
"ShortText",
|
391
|
+
[
|
392
|
+
]
|
393
|
+
],
|
394
|
+
[
|
395
|
+
257,
|
396
|
+
"message",
|
397
|
+
"/tmp/db/db.0000101",
|
398
|
+
"scalar",
|
399
|
+
"COLUMN_SCALAR",
|
400
|
+
"Logs",
|
401
|
+
"ShortText",
|
402
|
+
[
|
403
|
+
]
|
404
|
+
],
|
405
|
+
]
|
406
|
+
end
|
407
|
+
|
408
|
+
def test_command_name_position_record
|
409
|
+
@response_bodies << JSON.generate([
|
410
|
+
[0, 0.0, 0.0],
|
411
|
+
table_list_response_body,
|
412
|
+
])
|
413
|
+
@response_bodies << JSON.generate([
|
414
|
+
[0, 0.0, 0.0],
|
415
|
+
column_list_response_body,
|
416
|
+
])
|
417
|
+
driver = create_driver
|
418
|
+
time = event_time("2012-10-26T08:45:42Z")
|
419
|
+
driver.run do
|
420
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], 2])
|
421
|
+
driver.feed("log", time + 0, {"message" => "message1"})
|
422
|
+
driver.feed("log", time + 1, {"message" => "message2"})
|
423
|
+
|
424
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], true])
|
425
|
+
driver.feed("groonga.command",
|
426
|
+
time + 2,
|
427
|
+
{
|
428
|
+
"name" => "column_create",
|
429
|
+
"arguments" => {
|
430
|
+
"table" => "Logs",
|
431
|
+
"name" => "new_column",
|
432
|
+
"flags" => "COLUMN_SCALAR",
|
433
|
+
"type" => "ShortText",
|
434
|
+
},
|
435
|
+
})
|
436
|
+
|
437
|
+
@response_bodies << JSON.generate([
|
438
|
+
[0, 0.0, 0.0],
|
439
|
+
table_list_response_body,
|
440
|
+
])
|
441
|
+
new_column_list_response_body = column_list_response_body
|
442
|
+
new_column_list_response_body << [
|
443
|
+
258,
|
444
|
+
"new_column",
|
445
|
+
"/tmp/db/db.0000102",
|
446
|
+
"scalar",
|
447
|
+
"COLUMN_SCALAR",
|
448
|
+
"Logs",
|
449
|
+
"ShortText",
|
450
|
+
[
|
451
|
+
],
|
452
|
+
]
|
453
|
+
@response_bodies << JSON.generate([
|
454
|
+
[0, 0.0, 0.0],
|
455
|
+
new_column_list_response_body,
|
456
|
+
])
|
457
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], 2])
|
458
|
+
driver.feed("log", time + 3,
|
459
|
+
{"message" => "message3", "new_column" => "value1"})
|
460
|
+
driver.feed("log", time + 4,
|
461
|
+
{"message" => "message4", "new_column" => "value2"})
|
462
|
+
end
|
463
|
+
assert_equal([
|
464
|
+
"/d/table_list",
|
465
|
+
"/d/column_list?table=Logs",
|
466
|
+
"/d/load?table=Logs",
|
467
|
+
"/d/column_create?flags=COLUMN_SCALAR&name=new_column&table=Logs&type=ShortText",
|
468
|
+
"/d/table_list",
|
469
|
+
"/d/column_list?table=Logs",
|
470
|
+
"/d/load?table=Logs",
|
471
|
+
],
|
472
|
+
@request_urls)
|
473
|
+
end
|
474
|
+
|
475
|
+
def test_command_name_position_tag
|
476
|
+
@response_bodies << JSON.generate([
|
477
|
+
[0, 0.0, 0.0],
|
478
|
+
table_list_response_body,
|
479
|
+
])
|
480
|
+
@response_bodies << JSON.generate([
|
481
|
+
[0, 0.0, 0.0],
|
482
|
+
column_list_response_body,
|
483
|
+
])
|
484
|
+
driver = create_driver
|
485
|
+
time = event_time("2012-10-26T08:45:42Z")
|
486
|
+
driver.run do
|
487
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], 2])
|
488
|
+
driver.feed("log", time + 0, {"message" => "message1"})
|
489
|
+
driver.feed("log", time + 1, {"message" => "message2"})
|
490
|
+
|
491
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], true])
|
492
|
+
driver.feed("groonga.command.column_create",
|
493
|
+
time + 2,
|
494
|
+
{
|
495
|
+
"table" => "Logs",
|
496
|
+
"name" => "new_column",
|
497
|
+
"flags" => "COLUMN_SCALAR",
|
498
|
+
"type" => "ShortText",
|
499
|
+
})
|
500
|
+
|
501
|
+
@response_bodies << JSON.generate([
|
502
|
+
[0, 0.0, 0.0],
|
503
|
+
table_list_response_body,
|
504
|
+
])
|
505
|
+
new_column_list_response_body = column_list_response_body
|
506
|
+
new_column_list_response_body << [
|
507
|
+
258,
|
508
|
+
"new_column",
|
509
|
+
"/tmp/db/db.0000102",
|
510
|
+
"scalar",
|
511
|
+
"COLUMN_SCALAR",
|
512
|
+
"Logs",
|
513
|
+
"ShortText",
|
514
|
+
[
|
515
|
+
],
|
516
|
+
]
|
517
|
+
@response_bodies << JSON.generate([
|
518
|
+
[0, 0.0, 0.0],
|
519
|
+
new_column_list_response_body,
|
520
|
+
])
|
521
|
+
@response_bodies << JSON.generate([[0, 0.0, 0.0], 2])
|
522
|
+
driver.feed("log", time + 3,
|
523
|
+
{"message" => "message3", "new_column" => "value1"})
|
524
|
+
driver.feed("log", time + 4,
|
525
|
+
{"message" => "message4", "new_column" => "value2"})
|
526
|
+
end
|
527
|
+
assert_equal([
|
528
|
+
"/d/table_list",
|
529
|
+
"/d/column_list?table=Logs",
|
530
|
+
"/d/load?table=Logs",
|
531
|
+
"/d/column_create?flags=COLUMN_SCALAR&name=new_column&table=Logs&type=ShortText",
|
532
|
+
"/d/table_list",
|
533
|
+
"/d/column_list?table=Logs",
|
534
|
+
"/d/load?table=Logs",
|
535
|
+
],
|
536
|
+
@request_urls)
|
537
|
+
end
|
538
|
+
end
|
145
539
|
end
|
146
540
|
|
147
541
|
class CommandLineTest < self
|
@@ -227,7 +621,7 @@ EOC
|
|
227
621
|
end
|
228
622
|
|
229
623
|
class CommandTest < self
|
230
|
-
def
|
624
|
+
def test_command_name_position_tag
|
231
625
|
driver = create_driver
|
232
626
|
time = event_time("2012-10-26T08:45:42Z")
|
233
627
|
driver.run(default_tag: "groonga.command.table_create") do
|
@@ -246,6 +640,32 @@ EOC
|
|
246
640
|
actual_input,
|
247
641
|
])
|
248
642
|
end
|
643
|
+
|
644
|
+
def test_command_name_position_record
|
645
|
+
driver = create_driver
|
646
|
+
time = event_time("2012-10-26T08:45:42Z")
|
647
|
+
driver.run(default_tag: "groonga.command") do
|
648
|
+
driver.feed(time,
|
649
|
+
{
|
650
|
+
"name" => "table_create",
|
651
|
+
"arguments" => {
|
652
|
+
"name" => "Users",
|
653
|
+
},
|
654
|
+
})
|
655
|
+
end
|
656
|
+
assert_equal([
|
657
|
+
[
|
658
|
+
"--input-fd", actual_input_fd,
|
659
|
+
"--output-fd", actual_output_fd,
|
660
|
+
"-n", @database_path,
|
661
|
+
],
|
662
|
+
"/d/table_create?name=Users\n",
|
663
|
+
],
|
664
|
+
[
|
665
|
+
actual_command_line,
|
666
|
+
actual_input,
|
667
|
+
])
|
668
|
+
end
|
249
669
|
end
|
250
670
|
end
|
251
671
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-groonga
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -192,9 +192,9 @@ specification_version: 4
|
|
192
192
|
summary: Fluentd plugin to store data into Groonga and implement Groonga replication
|
193
193
|
system.
|
194
194
|
test_files:
|
195
|
-
- test/test_input.rb
|
196
|
-
- test/run-test.rb
|
197
195
|
- test/test_output.rb
|
198
|
-
- test/
|
199
|
-
- test/output/test_type_guesser.rb
|
196
|
+
- test/run-test.rb
|
200
197
|
- test/output/test_table_definition.rb
|
198
|
+
- test/output/test_type_guesser.rb
|
199
|
+
- test/output/test_table_index_definition.rb
|
200
|
+
- test/test_input.rb
|