fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +2 -0
- data/CONTRIBUTING.md +6 -1
- data/ChangeLog +95 -0
- data/Rakefile +21 -0
- data/appveyor.yml +1 -0
- data/code-of-conduct.md +3 -0
- data/example/out_exec_filter.conf +42 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/agent.rb +2 -2
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cat.rb +15 -4
- data/lib/fluent/compat/output.rb +14 -9
- data/lib/fluent/compat/parser.rb +141 -11
- data/lib/fluent/config/configure_proxy.rb +2 -11
- data/lib/fluent/config/section.rb +8 -1
- data/lib/fluent/configurable.rb +1 -3
- data/lib/fluent/env.rb +1 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/plugin/base.rb +17 -0
- data/lib/fluent/plugin/filter_parser.rb +108 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
- data/lib/fluent/plugin/filter_stdout.rb +1 -1
- data/lib/fluent/plugin/formatter.rb +5 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_exec.rb +48 -93
- data/lib/fluent/plugin/in_forward.rb +66 -265
- data/lib/fluent/plugin/in_http.rb +68 -65
- data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
- data/lib/fluent/plugin/in_syslog.rb +42 -58
- data/lib/fluent/plugin/in_tail.rb +29 -14
- data/lib/fluent/plugin/in_tcp.rb +54 -14
- data/lib/fluent/plugin/in_udp.rb +49 -13
- data/lib/fluent/plugin/multi_output.rb +1 -3
- data/lib/fluent/plugin/out_exec.rb +58 -71
- data/lib/fluent/plugin/out_exec_filter.rb +199 -279
- data/lib/fluent/plugin/out_file.rb +172 -81
- data/lib/fluent/plugin/out_forward.rb +229 -206
- data/lib/fluent/plugin/out_stdout.rb +6 -21
- data/lib/fluent/plugin/output.rb +90 -59
- data/lib/fluent/plugin/parser.rb +121 -61
- data/lib/fluent/plugin/parser_csv.rb +9 -3
- data/lib/fluent/plugin/parser_json.rb +37 -35
- data/lib/fluent/plugin/parser_ltsv.rb +11 -19
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -42
- data/lib/fluent/plugin/parser_tsv.rb +8 -3
- data/lib/fluent/plugin_helper.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +139 -73
- data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
- data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
- data/lib/fluent/plugin_helper/event_loop.rb +24 -6
- data/lib/fluent/plugin_helper/extract.rb +16 -4
- data/lib/fluent/plugin_helper/formatter.rb +9 -11
- data/lib/fluent/plugin_helper/inject.rb +16 -1
- data/lib/fluent/plugin_helper/parser.rb +3 -3
- data/lib/fluent/plugin_helper/server.rb +494 -0
- data/lib/fluent/plugin_helper/socket.rb +101 -0
- data/lib/fluent/plugin_helper/socket_option.rb +84 -0
- data/lib/fluent/plugin_helper/timer.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/driver/base.rb +95 -49
- data/lib/fluent/test/driver/base_owner.rb +18 -8
- data/lib/fluent/test/driver/multi_output.rb +2 -1
- data/lib/fluent/test/driver/output.rb +29 -6
- data/lib/fluent/test/helpers.rb +3 -1
- data/lib/fluent/test/log.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +13 -0
- data/lib/fluent/time.rb +14 -8
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +1 -1
- data/test/command/test_binlog_reader.rb +5 -1
- data/test/compat/test_parser.rb +10 -0
- data/test/config/test_configurable.rb +193 -0
- data/test/config/test_configure_proxy.rb +0 -43
- data/test/helper.rb +36 -1
- data/test/plugin/test_base.rb +16 -0
- data/test/plugin/test_filter_parser.rb +665 -0
- data/test/plugin/test_filter_record_transformer.rb +36 -100
- data/test/plugin/test_filter_stdout.rb +18 -27
- data/test/plugin/test_in_dummy.rb +1 -1
- data/test/plugin/test_in_exec.rb +206 -94
- data/test/plugin/test_in_forward.rb +268 -347
- data/test/plugin/test_in_http.rb +310 -186
- data/test/plugin/test_in_monitor_agent.rb +65 -35
- data/test/plugin/test_in_syslog.rb +39 -3
- data/test/plugin/test_in_tcp.rb +78 -62
- data/test/plugin/test_in_udp.rb +101 -80
- data/test/plugin/test_out_exec.rb +223 -68
- data/test/plugin/test_out_exec_filter.rb +520 -169
- data/test/plugin/test_out_file.rb +637 -177
- data/test/plugin/test_out_forward.rb +242 -234
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_out_secondary_file.rb +4 -2
- data/test/plugin/test_out_stdout.rb +14 -35
- data/test/plugin/test_output_as_buffered.rb +60 -2
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_csv.rb +1 -2
- data/test/plugin/test_parser_json.rb +3 -4
- data/test/plugin/test_parser_labeled_tsv.rb +1 -2
- data/test/plugin/test_parser_none.rb +1 -2
- data/test/plugin/test_parser_regexp.rb +8 -4
- data/test/plugin/test_parser_tsv.rb +4 -3
- data/test/plugin_helper/test_child_process.rb +184 -0
- data/test/plugin_helper/test_compat_parameters.rb +88 -1
- data/test/plugin_helper/test_extract.rb +0 -1
- data/test/plugin_helper/test_formatter.rb +5 -2
- data/test/plugin_helper/test_inject.rb +21 -0
- data/test/plugin_helper/test_parser.rb +6 -5
- data/test/plugin_helper/test_server.rb +905 -0
- data/test/test_event_time.rb +3 -1
- data/test/test_output.rb +53 -2
- data/test/test_plugin_classes.rb +20 -0
- data/test/test_root_agent.rb +139 -0
- data/test/test_test_drivers.rb +135 -0
- metadata +28 -8
- data/test/plugin/test_parser_base.rb +0 -32
@@ -1,14 +1,19 @@
|
|
1
1
|
require_relative '../helper'
|
2
|
-
require 'fluent/test'
|
3
|
-
require 'fluent/test/startup_shutdown'
|
2
|
+
require 'fluent/test/driver/output'
|
4
3
|
require 'fluent/plugin/out_forward'
|
5
4
|
require 'flexmock/test_unit'
|
6
5
|
|
7
|
-
|
8
|
-
|
6
|
+
require 'fluent/test/driver/input'
|
7
|
+
require 'fluent/plugin/in_forward'
|
9
8
|
|
9
|
+
class ForwardOutputTest < Test::Unit::TestCase
|
10
10
|
def setup
|
11
11
|
Fluent::Test.setup
|
12
|
+
@d = nil
|
13
|
+
end
|
14
|
+
|
15
|
+
def teardown
|
16
|
+
@d.instance_shutdown if @d
|
12
17
|
end
|
13
18
|
|
14
19
|
TARGET_HOST = '127.0.0.1'
|
@@ -29,7 +34,7 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
29
34
|
]
|
30
35
|
|
31
36
|
def create_driver(conf=CONFIG)
|
32
|
-
|
37
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
|
33
38
|
attr_reader :responses, :exceptions
|
34
39
|
|
35
40
|
def initialize
|
@@ -38,31 +43,17 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
38
43
|
@exceptions = []
|
39
44
|
end
|
40
45
|
|
41
|
-
def
|
42
|
-
super
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
rescue => e
|
47
|
-
@sender.exceptions << e
|
48
|
-
raise e
|
49
|
-
end
|
50
|
-
end
|
51
|
-
@nodes.each do |node|
|
52
|
-
node.singleton_class.prepend(m)
|
53
|
-
end
|
46
|
+
def read_ack_from_sock(sock, unpacker)
|
47
|
+
@responses << super
|
48
|
+
rescue => e
|
49
|
+
@exceptions << e
|
50
|
+
raise e
|
54
51
|
end
|
55
52
|
}.configure(conf)
|
56
|
-
router = Object.new
|
57
|
-
def router.method_missing(name, *args, **kw_args, &block)
|
58
|
-
Engine.root_agent.event_router.__send__(name, *args, **kw_args, &block)
|
59
|
-
end
|
60
|
-
d.instance.router = router
|
61
|
-
d
|
62
53
|
end
|
63
54
|
|
64
|
-
|
65
|
-
d = create_driver(%[
|
55
|
+
test 'configure' do
|
56
|
+
@d = d = create_driver(%[
|
66
57
|
self_hostname localhost
|
67
58
|
<server>
|
68
59
|
name test
|
@@ -80,44 +71,71 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
80
71
|
assert_equal TARGET_PORT, node.port
|
81
72
|
end
|
82
73
|
|
83
|
-
|
84
|
-
d = create_driver(
|
74
|
+
test 'configure_traditional' do
|
75
|
+
@d = d = create_driver(<<EOL)
|
76
|
+
self_hostname localhost
|
77
|
+
<server>
|
78
|
+
name test
|
79
|
+
host #{TARGET_HOST}
|
80
|
+
port #{TARGET_PORT}
|
81
|
+
</server>
|
82
|
+
buffer_chunk_limit 10m
|
83
|
+
EOL
|
84
|
+
instance = d.instance
|
85
|
+
assert instance.chunk_key_tag
|
86
|
+
assert !instance.chunk_key_time
|
87
|
+
assert_equal [], instance.chunk_keys
|
88
|
+
assert{ instance.buffer.is_a?(Fluent::Plugin::MemoryBuffer) }
|
89
|
+
assert_equal( 10*1024*1024, instance.buffer.chunk_limit_size )
|
90
|
+
end
|
91
|
+
|
92
|
+
test 'configure_udp_heartbeat' do
|
93
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type udp")
|
85
94
|
assert_equal :udp, d.instance.heartbeat_type
|
86
95
|
end
|
87
96
|
|
88
|
-
|
89
|
-
d = create_driver(CONFIG + "\nheartbeat_type none")
|
97
|
+
test 'configure_none_heartbeat' do
|
98
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type none")
|
90
99
|
assert_equal :none, d.instance.heartbeat_type
|
91
100
|
end
|
92
101
|
|
93
|
-
|
102
|
+
test 'configure_expire_dns_cache' do
|
103
|
+
@d = d = create_driver(CONFIG + "\nexpire_dns_cache 5")
|
104
|
+
assert_equal 5, d.instance.expire_dns_cache
|
105
|
+
end
|
106
|
+
|
107
|
+
test 'configure_dns_round_robin udp' do
|
94
108
|
assert_raise(Fluent::ConfigError) do
|
95
109
|
create_driver(CONFIG + "\nheartbeat_type udp\ndns_round_robin true")
|
96
110
|
end
|
111
|
+
end
|
97
112
|
|
98
|
-
|
113
|
+
test 'configure_dns_round_robin tcp' do
|
114
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type tcp\ndns_round_robin true")
|
99
115
|
assert_equal true, d.instance.dns_round_robin
|
116
|
+
end
|
100
117
|
|
101
|
-
|
118
|
+
test 'configure_dns_round_robin none' do
|
119
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type none\ndns_round_robin true")
|
102
120
|
assert_equal true, d.instance.dns_round_robin
|
103
121
|
end
|
104
122
|
|
105
|
-
|
123
|
+
test 'configure_no_server' do
|
106
124
|
assert_raise(Fluent::ConfigError, 'forward output plugin requires at least one <server> is required') do
|
107
125
|
create_driver('')
|
108
126
|
end
|
109
127
|
end
|
110
128
|
|
111
|
-
|
112
|
-
d = create_driver
|
129
|
+
test 'compress_default_value' do
|
130
|
+
@d = d = create_driver
|
113
131
|
assert_equal :text, d.instance.compress
|
114
132
|
|
115
133
|
node = d.instance.nodes.first
|
116
134
|
assert_equal :text, node.instance_variable_get(:@compress)
|
117
135
|
end
|
118
136
|
|
119
|
-
|
120
|
-
d = create_driver(CONFIG + %[compress gzip])
|
137
|
+
test 'set_compress_is_gzip' do
|
138
|
+
@d = d = create_driver(CONFIG + %[compress gzip])
|
121
139
|
assert_equal :gzip, d.instance.compress
|
122
140
|
assert_equal :gzip, d.instance.buffer.compress
|
123
141
|
|
@@ -125,11 +143,11 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
125
143
|
assert_equal :gzip, node.instance_variable_get(:@compress)
|
126
144
|
end
|
127
145
|
|
128
|
-
|
146
|
+
test 'set_compress_is_gzip_in_buffer_section' do
|
129
147
|
mock = flexmock($log)
|
130
148
|
mock.should_receive(:log).with("buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>")
|
131
149
|
|
132
|
-
d = create_driver(CONFIG + %[
|
150
|
+
@d = d = create_driver(CONFIG + %[
|
133
151
|
<buffer>
|
134
152
|
type memory
|
135
153
|
compress gzip
|
@@ -142,25 +160,29 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
142
160
|
assert_equal :text, node.instance_variable_get(:@compress)
|
143
161
|
end
|
144
162
|
|
145
|
-
|
146
|
-
d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
|
163
|
+
test 'phi_failure_detector disabled' do
|
164
|
+
@d = d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
|
147
165
|
node = d.instance.nodes.first
|
148
166
|
stub(node.failure).phi { raise 'Should not be called' }
|
149
167
|
node.tick
|
150
168
|
assert_equal node.available, true
|
169
|
+
end
|
151
170
|
|
152
|
-
|
171
|
+
test 'phi_failure_detector enabled' do
|
172
|
+
@d = d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
|
153
173
|
node = d.instance.nodes.first
|
154
174
|
node.tick
|
155
175
|
assert_equal node.available, false
|
156
176
|
end
|
157
177
|
|
158
|
-
|
159
|
-
d = create_driver(CONFIG)
|
178
|
+
test 'require_ack_response is disabled in default' do
|
179
|
+
@d = d = create_driver(CONFIG)
|
160
180
|
assert_equal false, d.instance.require_ack_response
|
161
181
|
assert_equal 190, d.instance.ack_response_timeout
|
182
|
+
end
|
162
183
|
|
163
|
-
|
184
|
+
test 'require_ack_response can be enabled' do
|
185
|
+
@d = d = create_driver(CONFIG + %[
|
164
186
|
require_ack_response true
|
165
187
|
ack_response_timeout 2s
|
166
188
|
])
|
@@ -168,79 +190,69 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
168
190
|
assert_equal 2, d.instance.ack_response_timeout
|
169
191
|
end
|
170
192
|
|
171
|
-
|
193
|
+
test 'send_with_time_as_integer' do
|
172
194
|
target_input_driver = create_target_input_driver
|
173
195
|
|
174
|
-
d = create_driver(CONFIG + %[flush_interval 1s])
|
196
|
+
@d = d = create_driver(CONFIG + %[flush_interval 1s])
|
175
197
|
|
176
|
-
time =
|
198
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
177
199
|
|
178
200
|
records = [
|
179
201
|
{"a" => 1},
|
180
202
|
{"a" => 2}
|
181
203
|
]
|
182
|
-
|
183
|
-
d.
|
184
|
-
end
|
185
|
-
|
186
|
-
target_input_driver.run do
|
187
|
-
d.run do
|
204
|
+
target_input_driver.run(expect_records: 2) do
|
205
|
+
d.run(default_tag: 'test') do
|
188
206
|
records.each do |record|
|
189
|
-
d.
|
207
|
+
d.feed(time, record)
|
190
208
|
end
|
191
209
|
end
|
192
210
|
end
|
193
211
|
|
194
|
-
|
195
|
-
|
196
|
-
assert_equal ['test', time, records[
|
197
|
-
|
198
|
-
|
212
|
+
events = target_input_driver.events
|
213
|
+
assert_equal_event_time(time, events[0][1])
|
214
|
+
assert_equal ['test', time, records[0]], events[0]
|
215
|
+
assert_equal_event_time(time, events[1][1])
|
216
|
+
assert_equal ['test', time, records[1]], events[1]
|
199
217
|
|
200
|
-
assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
|
201
218
|
assert_empty d.instance.exceptions
|
202
219
|
end
|
203
220
|
|
204
|
-
|
221
|
+
test 'send_without_time_as_integer' do
|
205
222
|
target_input_driver = create_target_input_driver
|
206
223
|
|
207
|
-
d = create_driver(CONFIG + %[
|
224
|
+
@d = d = create_driver(CONFIG + %[
|
208
225
|
flush_interval 1s
|
209
226
|
time_as_integer false
|
210
227
|
])
|
211
228
|
|
212
|
-
time =
|
229
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
213
230
|
|
214
231
|
records = [
|
215
232
|
{"a" => 1},
|
216
233
|
{"a" => 2}
|
217
234
|
]
|
218
|
-
|
219
|
-
d.
|
220
|
-
end
|
221
|
-
|
222
|
-
target_input_driver.run do
|
223
|
-
d.run do
|
235
|
+
target_input_driver.run(expect_records: 2) do
|
236
|
+
d.run(default_tag: 'test') do
|
224
237
|
records.each do |record|
|
225
|
-
d.
|
238
|
+
d.feed(time, record)
|
226
239
|
end
|
227
240
|
end
|
228
241
|
end
|
229
242
|
|
230
|
-
|
231
|
-
|
232
|
-
assert_equal ['test', time, records[
|
233
|
-
assert_equal_event_time(time,
|
234
|
-
|
243
|
+
events = target_input_driver.events
|
244
|
+
assert_equal_event_time(time, events[0][1])
|
245
|
+
assert_equal ['test', time, records[0]], events[0]
|
246
|
+
assert_equal_event_time(time, events[1][1])
|
247
|
+
assert_equal ['test', time, records[1]], events[1]
|
235
248
|
|
236
|
-
assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
|
237
249
|
assert_empty d.instance.exceptions
|
238
250
|
end
|
239
251
|
|
240
|
-
|
252
|
+
test 'send_comprssed_message_pack_stream_if_compress_is_gzip' do
|
241
253
|
target_input_driver = create_target_input_driver
|
242
254
|
|
243
|
-
d = create_driver(CONFIG + %[
|
255
|
+
@d = d = create_driver(CONFIG + %[
|
244
256
|
flush_interval 1s
|
245
257
|
compress gzip
|
246
258
|
])
|
@@ -251,209 +263,202 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
251
263
|
{"a" => 1},
|
252
264
|
{"a" => 2}
|
253
265
|
]
|
254
|
-
|
255
|
-
d.
|
256
|
-
end
|
257
|
-
|
258
|
-
target_input_driver.run do
|
259
|
-
d.run do
|
266
|
+
target_input_driver.run(expect_records: 2) do
|
267
|
+
d.run(default_tag: 'test') do
|
260
268
|
records.each do |record|
|
261
|
-
d.
|
269
|
+
d.feed(time, record)
|
262
270
|
end
|
263
271
|
end
|
264
272
|
end
|
265
273
|
|
266
274
|
event_streams = target_input_driver.event_streams
|
267
|
-
assert_true event_streams[0].is_a?(Fluent::CompressedMessagePackEventStream)
|
275
|
+
assert_true event_streams[0][1].is_a?(Fluent::CompressedMessagePackEventStream)
|
268
276
|
|
269
|
-
|
270
|
-
assert_equal ['test', time, records[0]],
|
271
|
-
assert_equal ['test', time, records[1]],
|
277
|
+
events = target_input_driver.events
|
278
|
+
assert_equal ['test', time, records[0]], events[0]
|
279
|
+
assert_equal ['test', time, records[1]], events[1]
|
272
280
|
end
|
273
281
|
|
274
|
-
|
275
|
-
target_input_driver = create_target_input_driver
|
282
|
+
test 'send_to_a_node_supporting_responses' do
|
283
|
+
target_input_driver = create_target_input_driver
|
276
284
|
|
277
|
-
d = create_driver(CONFIG + %[flush_interval 1s])
|
285
|
+
@d = d = create_driver(CONFIG + %[flush_interval 1s])
|
278
286
|
|
279
|
-
time =
|
287
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
280
288
|
|
281
289
|
records = [
|
282
290
|
{"a" => 1},
|
283
291
|
{"a" => 2}
|
284
292
|
]
|
285
|
-
|
286
|
-
d.
|
287
|
-
end
|
288
|
-
|
289
|
-
target_input_driver.run do
|
290
|
-
d.run do
|
293
|
+
target_input_driver.run(expect_records: 2) do
|
294
|
+
d.run(default_tag: 'test') do
|
291
295
|
records.each do |record|
|
292
|
-
d.
|
296
|
+
d.feed(time, record)
|
293
297
|
end
|
294
298
|
end
|
295
299
|
end
|
296
300
|
|
297
|
-
|
298
|
-
assert_equal ['test', time, records[0]],
|
299
|
-
assert_equal ['test', time, records[1]],
|
301
|
+
events = target_input_driver.events
|
302
|
+
assert_equal ['test', time, records[0]], events[0]
|
303
|
+
assert_equal ['test', time, records[1]], events[1]
|
300
304
|
|
301
|
-
|
305
|
+
assert_empty d.instance.responses # not attempt to receive responses, so it's empty
|
302
306
|
assert_empty d.instance.exceptions
|
303
307
|
end
|
304
308
|
|
305
|
-
|
309
|
+
test 'send_to_a_node_not_supporting_responses' do
|
306
310
|
target_input_driver = create_target_input_driver
|
307
311
|
|
308
|
-
d = create_driver(CONFIG + %[flush_interval 1s])
|
312
|
+
@d = d = create_driver(CONFIG + %[flush_interval 1s])
|
309
313
|
|
310
|
-
time =
|
314
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
311
315
|
|
312
316
|
records = [
|
313
317
|
{"a" => 1},
|
314
318
|
{"a" => 2}
|
315
319
|
]
|
316
|
-
|
317
|
-
d.
|
318
|
-
end
|
319
|
-
|
320
|
-
target_input_driver.run do
|
321
|
-
d.run do
|
320
|
+
target_input_driver.run(expect_records: 2) do
|
321
|
+
d.run(default_tag: 'test') do
|
322
322
|
records.each do |record|
|
323
|
-
d.
|
323
|
+
d.feed(time, record)
|
324
324
|
end
|
325
325
|
end
|
326
326
|
end
|
327
327
|
|
328
|
-
|
329
|
-
assert_equal ['test', time, records[0]],
|
330
|
-
assert_equal ['test', time, records[1]],
|
328
|
+
events = target_input_driver.events
|
329
|
+
assert_equal ['test', time, records[0]], events[0]
|
330
|
+
assert_equal ['test', time, records[1]], events[1]
|
331
331
|
|
332
|
-
|
332
|
+
assert_empty d.instance.responses # not attempt to receive responses, so it's empty
|
333
333
|
assert_empty d.instance.exceptions
|
334
334
|
end
|
335
335
|
|
336
|
-
|
336
|
+
test 'a node supporting responses' do
|
337
337
|
target_input_driver = create_target_input_driver
|
338
338
|
|
339
|
-
d = create_driver(CONFIG + %[
|
340
|
-
flush_interval 1s
|
339
|
+
@d = d = create_driver(CONFIG + %[
|
341
340
|
require_ack_response true
|
342
341
|
ack_response_timeout 1s
|
342
|
+
<buffer tag>
|
343
|
+
flush_mode immediate
|
344
|
+
retry_type periodic
|
345
|
+
retry_wait 30s
|
346
|
+
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
347
|
+
</buffer>
|
343
348
|
])
|
344
349
|
|
345
|
-
time =
|
350
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
346
351
|
|
347
352
|
records = [
|
348
353
|
{"a" => 1},
|
349
354
|
{"a" => 2}
|
350
355
|
]
|
351
|
-
|
352
|
-
d.instance.responses.length
|
353
|
-
|
354
|
-
|
355
|
-
target_input_driver.run do
|
356
|
-
d.run do
|
357
|
-
records.each do |record|
|
358
|
-
d.emit record, time
|
359
|
-
end
|
356
|
+
target_input_driver.run(expect_records: 2) do
|
357
|
+
d.end_if{ d.instance.responses.length > 0 }
|
358
|
+
d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
|
359
|
+
d.feed([[time, records[0]], [time,records[1]]])
|
360
360
|
end
|
361
361
|
end
|
362
362
|
|
363
|
-
|
364
|
-
assert_equal ['test', time, records[0]],
|
365
|
-
assert_equal ['test', time, records[1]],
|
363
|
+
events = target_input_driver.events
|
364
|
+
assert_equal ['test', time, records[0]], events[0]
|
365
|
+
assert_equal ['test', time, records[1]], events[1]
|
366
366
|
|
367
367
|
assert_equal 1, d.instance.responses.length
|
368
|
-
assert d.instance.responses[0].has_key?('ack')
|
369
368
|
assert_empty d.instance.exceptions
|
370
369
|
end
|
371
370
|
|
372
|
-
|
373
|
-
target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect:
|
371
|
+
test 'a destination node not supporting responses by just ignoring' do
|
372
|
+
target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: false)
|
374
373
|
|
375
|
-
d = create_driver(CONFIG + %[
|
376
|
-
flush_interval 1s
|
374
|
+
@d = d = create_driver(CONFIG + %[
|
377
375
|
require_ack_response true
|
378
376
|
ack_response_timeout 1s
|
377
|
+
<buffer tag>
|
378
|
+
flush_mode immediate
|
379
|
+
retry_type periodic
|
380
|
+
retry_wait 30s
|
381
|
+
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
382
|
+
</buffer>
|
379
383
|
])
|
380
384
|
|
381
|
-
|
385
|
+
node = d.instance.nodes.first
|
386
|
+
delayed_commit_timeout_value = nil
|
387
|
+
|
388
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
382
389
|
|
383
390
|
records = [
|
384
391
|
{"a" => 1},
|
385
392
|
{"a" => 2}
|
386
393
|
]
|
387
|
-
d.
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
target_input_driver.run do
|
394
|
-
d.run do
|
395
|
-
records.each do |record|
|
396
|
-
d.emit record, time
|
397
|
-
end
|
398
|
-
end
|
394
|
+
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
395
|
+
target_input_driver.end_if{ !node.available }
|
396
|
+
target_input_driver.run(expect_records: 2, timeout: 25) do
|
397
|
+
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
|
398
|
+
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
399
|
+
d.feed([[time, records[0]], [time,records[1]]])
|
399
400
|
end
|
400
401
|
end
|
401
402
|
|
402
|
-
|
403
|
-
assert_equal ['test', time, records[0]], emits[0]
|
404
|
-
assert_equal ['test', time, records[1]], emits[1]
|
403
|
+
assert_equal 1, delayed_commit_timeout_value
|
405
404
|
|
406
|
-
|
407
|
-
assert_equal
|
405
|
+
events = target_input_driver.events
|
406
|
+
assert_equal ['test', time, records[0]], events[0]
|
407
|
+
assert_equal ['test', time, records[1]], events[1]
|
408
408
|
|
409
|
-
|
410
|
-
|
409
|
+
assert{ d.instance.rollback_count > 0 }
|
410
|
+
|
411
|
+
logs = d.instance.log.logs
|
412
|
+
assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
|
413
|
+
assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
|
411
414
|
end
|
412
415
|
|
413
|
-
|
414
|
-
def test_require_a_node_not_supporting_responses_2_to_respond_with_ack
|
415
|
-
# in_forward, that doesn't support ack feature, and disconnect immediately
|
416
|
+
test 'a destination node not supporting responses by disconnection' do
|
416
417
|
target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: true)
|
417
418
|
|
418
|
-
d = create_driver(CONFIG + %[
|
419
|
-
flush_interval 1s
|
419
|
+
@d = d = create_driver(CONFIG + %[
|
420
420
|
require_ack_response true
|
421
421
|
ack_response_timeout 5s
|
422
|
+
<buffer tag>
|
423
|
+
flush_mode immediate
|
424
|
+
retry_type periodic
|
425
|
+
retry_wait 30s
|
426
|
+
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
427
|
+
</buffer>
|
422
428
|
])
|
423
429
|
|
424
|
-
|
430
|
+
node = d.instance.nodes.first
|
431
|
+
delayed_commit_timeout_value = nil
|
432
|
+
|
433
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
425
434
|
|
426
435
|
records = [
|
427
436
|
{"a" => 1},
|
428
437
|
{"a" => 2}
|
429
438
|
]
|
430
|
-
d.
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
target_input_driver.run do
|
437
|
-
d.run do
|
438
|
-
records.each do |record|
|
439
|
-
d.emit record, time
|
440
|
-
end
|
441
|
-
end
|
439
|
+
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
440
|
+
target_input_driver.end_if{ !node.available }
|
441
|
+
target_input_driver.run(expect_records: 2, timeout: 25) do
|
442
|
+
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
|
443
|
+
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
444
|
+
d.feed([[time, records[0]], [time,records[1]]])
|
442
445
|
end
|
443
446
|
end
|
444
447
|
|
445
|
-
|
446
|
-
assert_equal ['test', time, records[0]], emits[0]
|
447
|
-
assert_equal ['test', time, records[1]], emits[1]
|
448
|
+
assert_equal 5, delayed_commit_timeout_value
|
448
449
|
|
449
|
-
|
450
|
-
assert_equal
|
450
|
+
events = target_input_driver.events
|
451
|
+
assert_equal ['test', time, records[0]], events[0]
|
452
|
+
assert_equal ['test', time, records[1]], events[1]
|
451
453
|
|
452
|
-
|
453
|
-
|
454
|
+
assert{ d.instance.rollback_count > 0 }
|
455
|
+
|
456
|
+
logs = d.instance.log.logs
|
457
|
+
assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
|
458
|
+
assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
|
454
459
|
end
|
455
460
|
|
456
|
-
|
461
|
+
test 'authentication_with_shared_key' do
|
457
462
|
input_conf = TARGET_CONFIG + %[
|
458
463
|
<security>
|
459
464
|
self_hostname in.localhost
|
@@ -478,37 +483,29 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
478
483
|
shared_key fluentd-sharedkey
|
479
484
|
</server>
|
480
485
|
]
|
481
|
-
d = create_driver(output_conf)
|
486
|
+
@d = d = create_driver(output_conf)
|
482
487
|
|
483
|
-
time =
|
488
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
484
489
|
records = [
|
485
490
|
{"a" => 1},
|
486
491
|
{"a" => 2}
|
487
492
|
]
|
488
493
|
|
489
|
-
target_input_driver.run do
|
490
|
-
|
491
|
-
|
492
|
-
d.run do
|
494
|
+
target_input_driver.run(expect_records: 2, timeout: 15) do
|
495
|
+
d.run(default_tag: 'test') do
|
493
496
|
records.each do |record|
|
494
|
-
d.
|
495
|
-
end
|
496
|
-
|
497
|
-
# run_post_condition of Fluent::Test::*Driver are buggy:
|
498
|
-
t = Time.now
|
499
|
-
while Time.now < t + 15 && target_input_driver.emits.size < 2
|
500
|
-
sleep 0.1
|
497
|
+
d.feed(time, record)
|
501
498
|
end
|
502
499
|
end
|
503
500
|
end
|
504
501
|
|
505
|
-
|
506
|
-
assert{
|
507
|
-
assert_equal(['test', time, records[0]],
|
508
|
-
assert_equal(['test', time, records[1]],
|
502
|
+
events = target_input_driver.events
|
503
|
+
assert{ events != [] }
|
504
|
+
assert_equal(['test', time, records[0]], events[0])
|
505
|
+
assert_equal(['test', time, records[1]], events[1])
|
509
506
|
end
|
510
507
|
|
511
|
-
|
508
|
+
test 'authentication_with_user_auth' do
|
512
509
|
input_conf = TARGET_CONFIG + %[
|
513
510
|
<security>
|
514
511
|
self_hostname in.localhost
|
@@ -540,41 +537,33 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
540
537
|
password fluentd
|
541
538
|
</server>
|
542
539
|
]
|
543
|
-
d = create_driver(output_conf)
|
540
|
+
@d = d = create_driver(output_conf)
|
544
541
|
|
545
|
-
time =
|
542
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
546
543
|
records = [
|
547
544
|
{"a" => 1},
|
548
545
|
{"a" => 2}
|
549
546
|
]
|
550
547
|
|
551
|
-
target_input_driver.run do
|
552
|
-
|
553
|
-
|
554
|
-
d.run do
|
548
|
+
target_input_driver.run(expect_records: 2, timeout: 15) do
|
549
|
+
d.run(default_tag: 'test') do
|
555
550
|
records.each do |record|
|
556
|
-
d.
|
557
|
-
end
|
558
|
-
|
559
|
-
# run_post_condition of Fluent::Test::*Driver are buggy:
|
560
|
-
t = Time.now
|
561
|
-
while Time.now < t + 15 && target_input_driver.emits.size < 2
|
562
|
-
sleep 0.1
|
551
|
+
d.feed(time, record)
|
563
552
|
end
|
564
553
|
end
|
565
554
|
end
|
566
555
|
|
567
|
-
|
568
|
-
assert{
|
569
|
-
assert_equal(['test', time, records[0]],
|
570
|
-
assert_equal(['test', time, records[1]],
|
556
|
+
events = target_input_driver.events
|
557
|
+
assert{ events != [] }
|
558
|
+
assert_equal(['test', time, records[0]], events[0])
|
559
|
+
assert_equal(['test', time, records[1]], events[1])
|
571
560
|
end
|
572
561
|
|
573
562
|
def create_target_input_driver(response_stub: nil, disconnect: false, conf: TARGET_CONFIG)
|
574
563
|
require 'fluent/plugin/in_forward'
|
575
564
|
|
576
565
|
# TODO: Support actual TCP heartbeat test
|
577
|
-
Fluent::Test::
|
566
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput) {
|
578
567
|
if response_stub.nil?
|
579
568
|
# do nothing because in_forward responds for ack option in default
|
580
569
|
else
|
@@ -585,10 +574,10 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
585
574
|
}.configure(conf)
|
586
575
|
end
|
587
576
|
|
588
|
-
|
589
|
-
d = create_driver(CONFIG + "\nheartbeat_type none")
|
577
|
+
test 'heartbeat_type_none' do
|
578
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type none")
|
590
579
|
node = d.instance.nodes.first
|
591
|
-
assert_equal Fluent::ForwardOutput::NoneHeartbeatNode, node.class
|
580
|
+
assert_equal Fluent::Plugin::ForwardOutput::NoneHeartbeatNode, node.class
|
592
581
|
|
593
582
|
d.instance.start
|
594
583
|
assert_nil d.instance.instance_variable_get(:@loop) # no HeartbeatHandler, or HeartbeatRequestTimer
|
@@ -599,19 +588,38 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
599
588
|
assert_equal node.available, true
|
600
589
|
end
|
601
590
|
|
602
|
-
|
603
|
-
d = create_driver(CONFIG + "\nheartbeat_type udp")
|
591
|
+
test 'heartbeat_type_udp' do
|
592
|
+
@d = d = create_driver(CONFIG + "\nheartbeat_type udp")
|
604
593
|
|
605
594
|
d.instance.start
|
606
595
|
usock = d.instance.instance_variable_get(:@usock)
|
607
|
-
|
608
|
-
|
596
|
+
servers = d.instance.instance_variable_get(:@_servers)
|
597
|
+
timers = d.instance.instance_variable_get(:@_timers)
|
609
598
|
assert_equal UDPSocket, usock.class
|
610
|
-
|
611
|
-
|
599
|
+
assert servers.find{|s| s.title == :out_forward_heartbeat_receiver }
|
600
|
+
assert timers.include?(:out_forward_heartbeat_request)
|
612
601
|
|
613
602
|
mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
|
614
|
-
timer.disable # call send_heartbeat at just once
|
615
|
-
|
603
|
+
# timer.disable # call send_heartbeat at just once
|
604
|
+
d.instance.send(:on_timer)
|
605
|
+
end
|
606
|
+
|
607
|
+
test 'acts_as_secondary' do
|
608
|
+
i = Fluent::Plugin::ForwardOutput.new
|
609
|
+
conf = config_element(
|
610
|
+
'match',
|
611
|
+
'primary.**',
|
612
|
+
{'@type' => 'forward'},
|
613
|
+
[
|
614
|
+
config_element('server', '', {'host' => '127.0.0.1'}),
|
615
|
+
config_element('secondary', '', {}, [
|
616
|
+
config_element('server', '', {'host' => '192.168.1.2'}),
|
617
|
+
config_element('server', '', {'host' => '192.168.1.3'})
|
618
|
+
]),
|
619
|
+
]
|
620
|
+
)
|
621
|
+
assert_nothing_raised do
|
622
|
+
i.configure(conf)
|
623
|
+
end
|
616
624
|
end
|
617
625
|
end
|