fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -1,14 +1,19 @@
1
1
  require_relative '../helper'
2
- require 'fluent/test'
3
- require 'fluent/test/startup_shutdown'
2
+ require 'fluent/test/driver/output'
4
3
  require 'fluent/plugin/out_forward'
5
4
  require 'flexmock/test_unit'
6
5
 
7
- class ForwardOutputTest < Test::Unit::TestCase
8
- extend Fluent::Test::StartupShutdown
6
+ require 'fluent/test/driver/input'
7
+ require 'fluent/plugin/in_forward'
9
8
 
9
+ class ForwardOutputTest < Test::Unit::TestCase
10
10
  def setup
11
11
  Fluent::Test.setup
12
+ @d = nil
13
+ end
14
+
15
+ def teardown
16
+ @d.instance_shutdown if @d
12
17
  end
13
18
 
14
19
  TARGET_HOST = '127.0.0.1'
@@ -29,7 +34,7 @@ class ForwardOutputTest < Test::Unit::TestCase
29
34
  ]
30
35
 
31
36
  def create_driver(conf=CONFIG)
32
- d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ForwardOutput) {
37
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
33
38
  attr_reader :responses, :exceptions
34
39
 
35
40
  def initialize
@@ -38,31 +43,17 @@ class ForwardOutputTest < Test::Unit::TestCase
38
43
  @exceptions = []
39
44
  end
40
45
 
41
- def configure(conf)
42
- super
43
- m = Module.new do
44
- def send_data(tag, chunk)
45
- @sender.responses << super
46
- rescue => e
47
- @sender.exceptions << e
48
- raise e
49
- end
50
- end
51
- @nodes.each do |node|
52
- node.singleton_class.prepend(m)
53
- end
46
+ def read_ack_from_sock(sock, unpacker)
47
+ @responses << super
48
+ rescue => e
49
+ @exceptions << e
50
+ raise e
54
51
  end
55
52
  }.configure(conf)
56
- router = Object.new
57
- def router.method_missing(name, *args, **kw_args, &block)
58
- Engine.root_agent.event_router.__send__(name, *args, **kw_args, &block)
59
- end
60
- d.instance.router = router
61
- d
62
53
  end
63
54
 
64
- def test_configure
65
- d = create_driver(%[
55
+ test 'configure' do
56
+ @d = d = create_driver(%[
66
57
  self_hostname localhost
67
58
  <server>
68
59
  name test
@@ -80,44 +71,71 @@ class ForwardOutputTest < Test::Unit::TestCase
80
71
  assert_equal TARGET_PORT, node.port
81
72
  end
82
73
 
83
- def test_configure_udp_heartbeat
84
- d = create_driver(CONFIG + "\nheartbeat_type udp")
74
+ test 'configure_traditional' do
75
+ @d = d = create_driver(<<EOL)
76
+ self_hostname localhost
77
+ <server>
78
+ name test
79
+ host #{TARGET_HOST}
80
+ port #{TARGET_PORT}
81
+ </server>
82
+ buffer_chunk_limit 10m
83
+ EOL
84
+ instance = d.instance
85
+ assert instance.chunk_key_tag
86
+ assert !instance.chunk_key_time
87
+ assert_equal [], instance.chunk_keys
88
+ assert{ instance.buffer.is_a?(Fluent::Plugin::MemoryBuffer) }
89
+ assert_equal( 10*1024*1024, instance.buffer.chunk_limit_size )
90
+ end
91
+
92
+ test 'configure_udp_heartbeat' do
93
+ @d = d = create_driver(CONFIG + "\nheartbeat_type udp")
85
94
  assert_equal :udp, d.instance.heartbeat_type
86
95
  end
87
96
 
88
- def test_configure_none_heartbeat
89
- d = create_driver(CONFIG + "\nheartbeat_type none")
97
+ test 'configure_none_heartbeat' do
98
+ @d = d = create_driver(CONFIG + "\nheartbeat_type none")
90
99
  assert_equal :none, d.instance.heartbeat_type
91
100
  end
92
101
 
93
- def test_configure_dns_round_robin
102
+ test 'configure_expire_dns_cache' do
103
+ @d = d = create_driver(CONFIG + "\nexpire_dns_cache 5")
104
+ assert_equal 5, d.instance.expire_dns_cache
105
+ end
106
+
107
+ test 'configure_dns_round_robin udp' do
94
108
  assert_raise(Fluent::ConfigError) do
95
109
  create_driver(CONFIG + "\nheartbeat_type udp\ndns_round_robin true")
96
110
  end
111
+ end
97
112
 
98
- d = create_driver(CONFIG + "\nheartbeat_type tcp\ndns_round_robin true")
113
+ test 'configure_dns_round_robin tcp' do
114
+ @d = d = create_driver(CONFIG + "\nheartbeat_type tcp\ndns_round_robin true")
99
115
  assert_equal true, d.instance.dns_round_robin
116
+ end
100
117
 
101
- d = create_driver(CONFIG + "\nheartbeat_type none\ndns_round_robin true")
118
+ test 'configure_dns_round_robin none' do
119
+ @d = d = create_driver(CONFIG + "\nheartbeat_type none\ndns_round_robin true")
102
120
  assert_equal true, d.instance.dns_round_robin
103
121
  end
104
122
 
105
- def test_configure_no_server
123
+ test 'configure_no_server' do
106
124
  assert_raise(Fluent::ConfigError, 'forward output plugin requires at least one <server> is required') do
107
125
  create_driver('')
108
126
  end
109
127
  end
110
128
 
111
- def test_compress_default_value
112
- d = create_driver
129
+ test 'compress_default_value' do
130
+ @d = d = create_driver
113
131
  assert_equal :text, d.instance.compress
114
132
 
115
133
  node = d.instance.nodes.first
116
134
  assert_equal :text, node.instance_variable_get(:@compress)
117
135
  end
118
136
 
119
- def test_set_compress_is_gzip
120
- d = create_driver(CONFIG + %[compress gzip])
137
+ test 'set_compress_is_gzip' do
138
+ @d = d = create_driver(CONFIG + %[compress gzip])
121
139
  assert_equal :gzip, d.instance.compress
122
140
  assert_equal :gzip, d.instance.buffer.compress
123
141
 
@@ -125,11 +143,11 @@ class ForwardOutputTest < Test::Unit::TestCase
125
143
  assert_equal :gzip, node.instance_variable_get(:@compress)
126
144
  end
127
145
 
128
- def test_set_compress_is_gzip_in_buffer_section
146
+ test 'set_compress_is_gzip_in_buffer_section' do
129
147
  mock = flexmock($log)
130
148
  mock.should_receive(:log).with("buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>")
131
149
 
132
- d = create_driver(CONFIG + %[
150
+ @d = d = create_driver(CONFIG + %[
133
151
  <buffer>
134
152
  type memory
135
153
  compress gzip
@@ -142,25 +160,29 @@ class ForwardOutputTest < Test::Unit::TestCase
142
160
  assert_equal :text, node.instance_variable_get(:@compress)
143
161
  end
144
162
 
145
- def test_phi_failure_detector
146
- d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
163
+ test 'phi_failure_detector disabled' do
164
+ @d = d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
147
165
  node = d.instance.nodes.first
148
166
  stub(node.failure).phi { raise 'Should not be called' }
149
167
  node.tick
150
168
  assert_equal node.available, true
169
+ end
151
170
 
152
- d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
171
+ test 'phi_failure_detector enabled' do
172
+ @d = d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
153
173
  node = d.instance.nodes.first
154
174
  node.tick
155
175
  assert_equal node.available, false
156
176
  end
157
177
 
158
- def test_wait_response_timeout_config
159
- d = create_driver(CONFIG)
178
+ test 'require_ack_response is disabled in default' do
179
+ @d = d = create_driver(CONFIG)
160
180
  assert_equal false, d.instance.require_ack_response
161
181
  assert_equal 190, d.instance.ack_response_timeout
182
+ end
162
183
 
163
- d = create_driver(CONFIG + %[
184
+ test 'require_ack_response can be enabled' do
185
+ @d = d = create_driver(CONFIG + %[
164
186
  require_ack_response true
165
187
  ack_response_timeout 2s
166
188
  ])
@@ -168,79 +190,69 @@ class ForwardOutputTest < Test::Unit::TestCase
168
190
  assert_equal 2, d.instance.ack_response_timeout
169
191
  end
170
192
 
171
- def test_send_with_time_as_integer
193
+ test 'send_with_time_as_integer' do
172
194
  target_input_driver = create_target_input_driver
173
195
 
174
- d = create_driver(CONFIG + %[flush_interval 1s])
196
+ @d = d = create_driver(CONFIG + %[flush_interval 1s])
175
197
 
176
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
198
+ time = event_time("2011-01-02 13:14:15 UTC")
177
199
 
178
200
  records = [
179
201
  {"a" => 1},
180
202
  {"a" => 2}
181
203
  ]
182
- d.register_run_post_condition do
183
- d.instance.responses.length == 1
184
- end
185
-
186
- target_input_driver.run do
187
- d.run do
204
+ target_input_driver.run(expect_records: 2) do
205
+ d.run(default_tag: 'test') do
188
206
  records.each do |record|
189
- d.emit record, time
207
+ d.feed(time, record)
190
208
  end
191
209
  end
192
210
  end
193
211
 
194
- emits = target_input_driver.emits
195
- assert_equal ['test', time, records[0]], emits[0]
196
- assert_equal ['test', time, records[1]], emits[1]
197
- assert(emits[0][1].is_a?(Integer))
198
- assert(emits[1][1].is_a?(Integer))
212
+ events = target_input_driver.events
213
+ assert_equal_event_time(time, events[0][1])
214
+ assert_equal ['test', time, records[0]], events[0]
215
+ assert_equal_event_time(time, events[1][1])
216
+ assert_equal ['test', time, records[1]], events[1]
199
217
 
200
- assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
201
218
  assert_empty d.instance.exceptions
202
219
  end
203
220
 
204
- def test_send_without_time_as_integer
221
+ test 'send_without_time_as_integer' do
205
222
  target_input_driver = create_target_input_driver
206
223
 
207
- d = create_driver(CONFIG + %[
224
+ @d = d = create_driver(CONFIG + %[
208
225
  flush_interval 1s
209
226
  time_as_integer false
210
227
  ])
211
228
 
212
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
229
+ time = event_time("2011-01-02 13:14:15 UTC")
213
230
 
214
231
  records = [
215
232
  {"a" => 1},
216
233
  {"a" => 2}
217
234
  ]
218
- d.register_run_post_condition do
219
- d.instance.responses.length == 1
220
- end
221
-
222
- target_input_driver.run do
223
- d.run do
235
+ target_input_driver.run(expect_records: 2) do
236
+ d.run(default_tag: 'test') do
224
237
  records.each do |record|
225
- d.emit record, time
238
+ d.feed(time, record)
226
239
  end
227
240
  end
228
241
  end
229
242
 
230
- emits = target_input_driver.emits
231
- assert_equal ['test', time, records[0]], emits[0]
232
- assert_equal ['test', time, records[1]], emits[1]
233
- assert_equal_event_time(time, emits[0][1])
234
- assert_equal_event_time(time, emits[1][1])
243
+ events = target_input_driver.events
244
+ assert_equal_event_time(time, events[0][1])
245
+ assert_equal ['test', time, records[0]], events[0]
246
+ assert_equal_event_time(time, events[1][1])
247
+ assert_equal ['test', time, records[1]], events[1]
235
248
 
236
- assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
237
249
  assert_empty d.instance.exceptions
238
250
  end
239
251
 
240
- def test_send_comprssed_message_pack_stream_if_compress_is_gzip
252
+ test 'send_comprssed_message_pack_stream_if_compress_is_gzip' do
241
253
  target_input_driver = create_target_input_driver
242
254
 
243
- d = create_driver(CONFIG + %[
255
+ @d = d = create_driver(CONFIG + %[
244
256
  flush_interval 1s
245
257
  compress gzip
246
258
  ])
@@ -251,209 +263,202 @@ class ForwardOutputTest < Test::Unit::TestCase
251
263
  {"a" => 1},
252
264
  {"a" => 2}
253
265
  ]
254
- d.register_run_post_condition do
255
- d.instance.responses.length == 1
256
- end
257
-
258
- target_input_driver.run do
259
- d.run do
266
+ target_input_driver.run(expect_records: 2) do
267
+ d.run(default_tag: 'test') do
260
268
  records.each do |record|
261
- d.emit record, time
269
+ d.feed(time, record)
262
270
  end
263
271
  end
264
272
  end
265
273
 
266
274
  event_streams = target_input_driver.event_streams
267
- assert_true event_streams[0].is_a?(Fluent::CompressedMessagePackEventStream)
275
+ assert_true event_streams[0][1].is_a?(Fluent::CompressedMessagePackEventStream)
268
276
 
269
- emits = target_input_driver.emits
270
- assert_equal ['test', time, records[0]], emits[0]
271
- assert_equal ['test', time, records[1]], emits[1]
277
+ events = target_input_driver.events
278
+ assert_equal ['test', time, records[0]], events[0]
279
+ assert_equal ['test', time, records[1]], events[1]
272
280
  end
273
281
 
274
- def test_send_to_a_node_supporting_responses
275
- target_input_driver = create_target_input_driver(response_stub: true)
282
+ test 'send_to_a_node_supporting_responses' do
283
+ target_input_driver = create_target_input_driver
276
284
 
277
- d = create_driver(CONFIG + %[flush_interval 1s])
285
+ @d = d = create_driver(CONFIG + %[flush_interval 1s])
278
286
 
279
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
287
+ time = event_time("2011-01-02 13:14:15 UTC")
280
288
 
281
289
  records = [
282
290
  {"a" => 1},
283
291
  {"a" => 2}
284
292
  ]
285
- d.register_run_post_condition do
286
- d.instance.responses.length == 1
287
- end
288
-
289
- target_input_driver.run do
290
- d.run do
293
+ target_input_driver.run(expect_records: 2) do
294
+ d.run(default_tag: 'test') do
291
295
  records.each do |record|
292
- d.emit record, time
296
+ d.feed(time, record)
293
297
  end
294
298
  end
295
299
  end
296
300
 
297
- emits = target_input_driver.emits
298
- assert_equal ['test', time, records[0]], emits[0]
299
- assert_equal ['test', time, records[1]], emits[1]
301
+ events = target_input_driver.events
302
+ assert_equal ['test', time, records[0]], events[0]
303
+ assert_equal ['test', time, records[1]], events[1]
300
304
 
301
- assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
305
+ assert_empty d.instance.responses # not attempt to receive responses, so it's empty
302
306
  assert_empty d.instance.exceptions
303
307
  end
304
308
 
305
- def test_send_to_a_node_not_supporting_responses
309
+ test 'send_to_a_node_not_supporting_responses' do
306
310
  target_input_driver = create_target_input_driver
307
311
 
308
- d = create_driver(CONFIG + %[flush_interval 1s])
312
+ @d = d = create_driver(CONFIG + %[flush_interval 1s])
309
313
 
310
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
314
+ time = event_time("2011-01-02 13:14:15 UTC")
311
315
 
312
316
  records = [
313
317
  {"a" => 1},
314
318
  {"a" => 2}
315
319
  ]
316
- d.register_run_post_condition do
317
- d.instance.responses.length == 1
318
- end
319
-
320
- target_input_driver.run do
321
- d.run do
320
+ target_input_driver.run(expect_records: 2) do
321
+ d.run(default_tag: 'test') do
322
322
  records.each do |record|
323
- d.emit record, time
323
+ d.feed(time, record)
324
324
  end
325
325
  end
326
326
  end
327
327
 
328
- emits = target_input_driver.emits
329
- assert_equal ['test', time, records[0]], emits[0]
330
- assert_equal ['test', time, records[1]], emits[1]
328
+ events = target_input_driver.events
329
+ assert_equal ['test', time, records[0]], events[0]
330
+ assert_equal ['test', time, records[1]], events[1]
331
331
 
332
- assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
332
+ assert_empty d.instance.responses # not attempt to receive responses, so it's empty
333
333
  assert_empty d.instance.exceptions
334
334
  end
335
335
 
336
- def test_require_a_node_supporting_responses_to_respond_with_ack
336
+ test 'a node supporting responses' do
337
337
  target_input_driver = create_target_input_driver
338
338
 
339
- d = create_driver(CONFIG + %[
340
- flush_interval 1s
339
+ @d = d = create_driver(CONFIG + %[
341
340
  require_ack_response true
342
341
  ack_response_timeout 1s
342
+ <buffer tag>
343
+ flush_mode immediate
344
+ retry_type periodic
345
+ retry_wait 30s
346
+ flush_at_shutdown false # suppress errors in d.instance_shutdown
347
+ </buffer>
343
348
  ])
344
349
 
345
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
350
+ time = event_time("2011-01-02 13:14:15 UTC")
346
351
 
347
352
  records = [
348
353
  {"a" => 1},
349
354
  {"a" => 2}
350
355
  ]
351
- d.register_run_post_condition do
352
- d.instance.responses.length == 1
353
- end
354
-
355
- target_input_driver.run do
356
- d.run do
357
- records.each do |record|
358
- d.emit record, time
359
- end
356
+ target_input_driver.run(expect_records: 2) do
357
+ d.end_if{ d.instance.responses.length > 0 }
358
+ d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
359
+ d.feed([[time, records[0]], [time,records[1]]])
360
360
  end
361
361
  end
362
362
 
363
- emits = target_input_driver.emits
364
- assert_equal ['test', time, records[0]], emits[0]
365
- assert_equal ['test', time, records[1]], emits[1]
363
+ events = target_input_driver.events
364
+ assert_equal ['test', time, records[0]], events[0]
365
+ assert_equal ['test', time, records[1]], events[1]
366
366
 
367
367
  assert_equal 1, d.instance.responses.length
368
- assert d.instance.responses[0].has_key?('ack')
369
368
  assert_empty d.instance.exceptions
370
369
  end
371
370
 
372
- def test_require_a_node_not_supporting_responses_to_respond_with_ack
373
- target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: true)
371
+ test 'a destination node not supporting responses by just ignoring' do
372
+ target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: false)
374
373
 
375
- d = create_driver(CONFIG + %[
376
- flush_interval 1s
374
+ @d = d = create_driver(CONFIG + %[
377
375
  require_ack_response true
378
376
  ack_response_timeout 1s
377
+ <buffer tag>
378
+ flush_mode immediate
379
+ retry_type periodic
380
+ retry_wait 30s
381
+ flush_at_shutdown false # suppress errors in d.instance_shutdown
382
+ </buffer>
379
383
  ])
380
384
 
381
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
385
+ node = d.instance.nodes.first
386
+ delayed_commit_timeout_value = nil
387
+
388
+ time = event_time("2011-01-02 13:14:15 UTC")
382
389
 
383
390
  records = [
384
391
  {"a" => 1},
385
392
  {"a" => 2}
386
393
  ]
387
- d.register_run_post_condition do
388
- d.instance.responses.length == 1
389
- end
390
- d.run_timeout = 2
391
-
392
- assert_raise Fluent::ForwardOutputACKTimeoutError do
393
- target_input_driver.run do
394
- d.run do
395
- records.each do |record|
396
- d.emit record, time
397
- end
398
- end
394
+ target_input_driver.end_if{ d.instance.rollback_count > 0 }
395
+ target_input_driver.end_if{ !node.available }
396
+ target_input_driver.run(expect_records: 2, timeout: 25) do
397
+ d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
398
+ delayed_commit_timeout_value = d.instance.delayed_commit_timeout
399
+ d.feed([[time, records[0]], [time,records[1]]])
399
400
  end
400
401
  end
401
402
 
402
- emits = target_input_driver.emits
403
- assert_equal ['test', time, records[0]], emits[0]
404
- assert_equal ['test', time, records[1]], emits[1]
403
+ assert_equal 1, delayed_commit_timeout_value
405
404
 
406
- node = d.instance.nodes.first
407
- assert_equal false, node.available # node is regarded as unavailable when timeout
405
+ events = target_input_driver.events
406
+ assert_equal ['test', time, records[0]], events[0]
407
+ assert_equal ['test', time, records[1]], events[1]
408
408
 
409
- assert_empty d.instance.responses # send_data() raises exception, so response is missing
410
- assert_equal 1, d.instance.exceptions.size
409
+ assert{ d.instance.rollback_count > 0 }
410
+
411
+ logs = d.instance.log.logs
412
+ assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
413
+ assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
411
414
  end
412
415
 
413
- # bdf1f4f104c00a791aa94dc20087fe2011e1fd83
414
- def test_require_a_node_not_supporting_responses_2_to_respond_with_ack
415
- # in_forward, that doesn't support ack feature, and disconnect immediately
416
+ test 'a destination node not supporting responses by disconnection' do
416
417
  target_input_driver = create_target_input_driver(response_stub: ->(_option) { nil }, disconnect: true)
417
418
 
418
- d = create_driver(CONFIG + %[
419
- flush_interval 1s
419
+ @d = d = create_driver(CONFIG + %[
420
420
  require_ack_response true
421
421
  ack_response_timeout 5s
422
+ <buffer tag>
423
+ flush_mode immediate
424
+ retry_type periodic
425
+ retry_wait 30s
426
+ flush_at_shutdown false # suppress errors in d.instance_shutdown
427
+ </buffer>
422
428
  ])
423
429
 
424
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
430
+ node = d.instance.nodes.first
431
+ delayed_commit_timeout_value = nil
432
+
433
+ time = event_time("2011-01-02 13:14:15 UTC")
425
434
 
426
435
  records = [
427
436
  {"a" => 1},
428
437
  {"a" => 2}
429
438
  ]
430
- d.register_run_post_condition do
431
- d.instance.responses.length == 1
432
- end
433
- d.run_timeout = 2
434
-
435
- assert_raise Fluent::ForwardOutputACKTimeoutError do
436
- target_input_driver.run do
437
- d.run do
438
- records.each do |record|
439
- d.emit record, time
440
- end
441
- end
439
+ target_input_driver.end_if{ d.instance.rollback_count > 0 }
440
+ target_input_driver.end_if{ !node.available }
441
+ target_input_driver.run(expect_records: 2, timeout: 25) do
442
+ d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
443
+ delayed_commit_timeout_value = d.instance.delayed_commit_timeout
444
+ d.feed([[time, records[0]], [time,records[1]]])
442
445
  end
443
446
  end
444
447
 
445
- emits = target_input_driver.emits
446
- assert_equal ['test', time, records[0]], emits[0]
447
- assert_equal ['test', time, records[1]], emits[1]
448
+ assert_equal 5, delayed_commit_timeout_value
448
449
 
449
- assert_equal 0, d.instance.responses.size
450
- assert_equal 1, d.instance.exceptions.size # send_data() fails and to be retried
450
+ events = target_input_driver.events
451
+ assert_equal ['test', time, records[0]], events[0]
452
+ assert_equal ['test', time, records[1]], events[1]
451
453
 
452
- node = d.instance.nodes.first
453
- assert_equal false, node.available # node is regarded as unavailable when unexpected EOF
454
+ assert{ d.instance.rollback_count > 0 }
455
+
456
+ logs = d.instance.log.logs
457
+ assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
458
+ assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
454
459
  end
455
460
 
456
- def test_authentication_with_shared_key
461
+ test 'authentication_with_shared_key' do
457
462
  input_conf = TARGET_CONFIG + %[
458
463
  <security>
459
464
  self_hostname in.localhost
@@ -478,37 +483,29 @@ class ForwardOutputTest < Test::Unit::TestCase
478
483
  shared_key fluentd-sharedkey
479
484
  </server>
480
485
  ]
481
- d = create_driver(output_conf)
486
+ @d = d = create_driver(output_conf)
482
487
 
483
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
488
+ time = event_time("2011-01-02 13:14:15 UTC")
484
489
  records = [
485
490
  {"a" => 1},
486
491
  {"a" => 2}
487
492
  ]
488
493
 
489
- target_input_driver.run do
490
- sleep 0.1 until target_input_driver.instance.instance_eval{ @thread } && target_input_driver.instance.instance_eval{ @thread }.status
491
-
492
- d.run do
494
+ target_input_driver.run(expect_records: 2, timeout: 15) do
495
+ d.run(default_tag: 'test') do
493
496
  records.each do |record|
494
- d.emit(record, time)
495
- end
496
-
497
- # run_post_condition of Fluent::Test::*Driver are buggy:
498
- t = Time.now
499
- while Time.now < t + 15 && target_input_driver.emits.size < 2
500
- sleep 0.1
497
+ d.feed(time, record)
501
498
  end
502
499
  end
503
500
  end
504
501
 
505
- emits = target_input_driver.emits
506
- assert{ emits != [] }
507
- assert_equal(['test', time, records[0]], emits[0])
508
- assert_equal(['test', time, records[1]], emits[1])
502
+ events = target_input_driver.events
503
+ assert{ events != [] }
504
+ assert_equal(['test', time, records[0]], events[0])
505
+ assert_equal(['test', time, records[1]], events[1])
509
506
  end
510
507
 
511
- def test_authentication_with_user_auth
508
+ test 'authentication_with_user_auth' do
512
509
  input_conf = TARGET_CONFIG + %[
513
510
  <security>
514
511
  self_hostname in.localhost
@@ -540,41 +537,33 @@ class ForwardOutputTest < Test::Unit::TestCase
540
537
  password fluentd
541
538
  </server>
542
539
  ]
543
- d = create_driver(output_conf)
540
+ @d = d = create_driver(output_conf)
544
541
 
545
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
542
+ time = event_time("2011-01-02 13:14:15 UTC")
546
543
  records = [
547
544
  {"a" => 1},
548
545
  {"a" => 2}
549
546
  ]
550
547
 
551
- target_input_driver.run do
552
- sleep 0.1 until target_input_driver.instance.instance_eval{ @thread } && target_input_driver.instance.instance_eval{ @thread }.status
553
-
554
- d.run do
548
+ target_input_driver.run(expect_records: 2, timeout: 15) do
549
+ d.run(default_tag: 'test') do
555
550
  records.each do |record|
556
- d.emit(record, time)
557
- end
558
-
559
- # run_post_condition of Fluent::Test::*Driver are buggy:
560
- t = Time.now
561
- while Time.now < t + 15 && target_input_driver.emits.size < 2
562
- sleep 0.1
551
+ d.feed(time, record)
563
552
  end
564
553
  end
565
554
  end
566
555
 
567
- emits = target_input_driver.emits
568
- assert{ emits != [] }
569
- assert_equal(['test', time, records[0]], emits[0])
570
- assert_equal(['test', time, records[1]], emits[1])
556
+ events = target_input_driver.events
557
+ assert{ events != [] }
558
+ assert_equal(['test', time, records[0]], events[0])
559
+ assert_equal(['test', time, records[1]], events[1])
571
560
  end
572
561
 
573
562
  def create_target_input_driver(response_stub: nil, disconnect: false, conf: TARGET_CONFIG)
574
563
  require 'fluent/plugin/in_forward'
575
564
 
576
565
  # TODO: Support actual TCP heartbeat test
577
- Fluent::Test::InputTestDriver.new(Fluent::ForwardInput) {
566
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput) {
578
567
  if response_stub.nil?
579
568
  # do nothing because in_forward responds for ack option in default
580
569
  else
@@ -585,10 +574,10 @@ class ForwardOutputTest < Test::Unit::TestCase
585
574
  }.configure(conf)
586
575
  end
587
576
 
588
- def test_heartbeat_type_none
589
- d = create_driver(CONFIG + "\nheartbeat_type none")
577
+ test 'heartbeat_type_none' do
578
+ @d = d = create_driver(CONFIG + "\nheartbeat_type none")
590
579
  node = d.instance.nodes.first
591
- assert_equal Fluent::ForwardOutput::NoneHeartbeatNode, node.class
580
+ assert_equal Fluent::Plugin::ForwardOutput::NoneHeartbeatNode, node.class
592
581
 
593
582
  d.instance.start
594
583
  assert_nil d.instance.instance_variable_get(:@loop) # no HeartbeatHandler, or HeartbeatRequestTimer
@@ -599,19 +588,38 @@ class ForwardOutputTest < Test::Unit::TestCase
599
588
  assert_equal node.available, true
600
589
  end
601
590
 
602
- def test_heartbeat_type_udp
603
- d = create_driver(CONFIG + "\nheartbeat_type udp")
591
+ test 'heartbeat_type_udp' do
592
+ @d = d = create_driver(CONFIG + "\nheartbeat_type udp")
604
593
 
605
594
  d.instance.start
606
595
  usock = d.instance.instance_variable_get(:@usock)
607
- timer = d.instance.instance_variable_get(:@timer)
608
- hb = d.instance.instance_variable_get(:@hb)
596
+ servers = d.instance.instance_variable_get(:@_servers)
597
+ timers = d.instance.instance_variable_get(:@_timers)
609
598
  assert_equal UDPSocket, usock.class
610
- assert_equal Fluent::ForwardOutput::HeartbeatRequestTimer, timer.class
611
- assert_equal Fluent::ForwardOutput::HeartbeatHandler, hb.class
599
+ assert servers.find{|s| s.title == :out_forward_heartbeat_receiver }
600
+ assert timers.include?(:out_forward_heartbeat_request)
612
601
 
613
602
  mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
614
- timer.disable # call send_heartbeat at just once
615
- timer.on_timer
603
+ # timer.disable # call send_heartbeat at just once
604
+ d.instance.send(:on_timer)
605
+ end
606
+
607
+ test 'acts_as_secondary' do
608
+ i = Fluent::Plugin::ForwardOutput.new
609
+ conf = config_element(
610
+ 'match',
611
+ 'primary.**',
612
+ {'@type' => 'forward'},
613
+ [
614
+ config_element('server', '', {'host' => '127.0.0.1'}),
615
+ config_element('secondary', '', {}, [
616
+ config_element('server', '', {'host' => '192.168.1.2'}),
617
+ config_element('server', '', {'host' => '192.168.1.3'})
618
+ ]),
619
+ ]
620
+ )
621
+ assert_nothing_raised do
622
+ i.configure(conf)
623
+ end
616
624
  end
617
625
  end