fluentd 1.6.3 → 1.7.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +35 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
  4. data/CHANGELOG.md +58 -0
  5. data/README.md +5 -1
  6. data/fluentd.gemspec +1 -1
  7. data/lib/fluent/clock.rb +4 -0
  8. data/lib/fluent/compat/output.rb +3 -3
  9. data/lib/fluent/compat/socket_util.rb +1 -1
  10. data/lib/fluent/config/element.rb +3 -3
  11. data/lib/fluent/config/literal_parser.rb +1 -1
  12. data/lib/fluent/config/section.rb +4 -1
  13. data/lib/fluent/error.rb +4 -0
  14. data/lib/fluent/event.rb +28 -24
  15. data/lib/fluent/event_router.rb +2 -1
  16. data/lib/fluent/log.rb +1 -1
  17. data/lib/fluent/msgpack_factory.rb +8 -0
  18. data/lib/fluent/plugin/bare_output.rb +4 -4
  19. data/lib/fluent/plugin/buf_file_single.rb +211 -0
  20. data/lib/fluent/plugin/buffer.rb +62 -63
  21. data/lib/fluent/plugin/buffer/chunk.rb +21 -3
  22. data/lib/fluent/plugin/buffer/file_chunk.rb +37 -12
  23. data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
  24. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
  25. data/lib/fluent/plugin/compressable.rb +10 -6
  26. data/lib/fluent/plugin/filter_grep.rb +2 -2
  27. data/lib/fluent/plugin/formatter_csv.rb +10 -6
  28. data/lib/fluent/plugin/in_syslog.rb +10 -3
  29. data/lib/fluent/plugin/in_tail.rb +7 -2
  30. data/lib/fluent/plugin/in_tcp.rb +34 -7
  31. data/lib/fluent/plugin/multi_output.rb +4 -4
  32. data/lib/fluent/plugin/out_exec_filter.rb +1 -0
  33. data/lib/fluent/plugin/out_file.rb +13 -3
  34. data/lib/fluent/plugin/out_forward.rb +126 -588
  35. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  36. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  37. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  38. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  39. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
  40. data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
  41. data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
  42. data/lib/fluent/plugin/out_http.rb +231 -0
  43. data/lib/fluent/plugin/output.rb +29 -35
  44. data/lib/fluent/plugin/parser.rb +77 -0
  45. data/lib/fluent/plugin/parser_csv.rb +75 -0
  46. data/lib/fluent/plugin_helper/server.rb +1 -1
  47. data/lib/fluent/plugin_helper/thread.rb +1 -0
  48. data/lib/fluent/root_agent.rb +1 -1
  49. data/lib/fluent/time.rb +4 -2
  50. data/lib/fluent/timezone.rb +21 -7
  51. data/lib/fluent/version.rb +1 -1
  52. data/test/command/test_fluentd.rb +1 -1
  53. data/test/command/test_plugin_generator.rb +18 -2
  54. data/test/config/test_configurable.rb +78 -40
  55. data/test/counter/test_store.rb +1 -1
  56. data/test/helper.rb +1 -0
  57. data/test/helpers/process_extenstion.rb +33 -0
  58. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  59. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  60. data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
  61. data/test/plugin/out_forward/test_load_balancer.rb +60 -0
  62. data/test/plugin/out_forward/test_socket_cache.rb +139 -0
  63. data/test/plugin/test_buf_file.rb +118 -2
  64. data/test/plugin/test_buf_file_single.rb +734 -0
  65. data/test/plugin/test_buffer.rb +4 -48
  66. data/test/plugin/test_buffer_file_chunk.rb +19 -1
  67. data/test/plugin/test_buffer_file_single_chunk.rb +620 -0
  68. data/test/plugin/test_formatter_csv.rb +16 -0
  69. data/test/plugin/test_in_syslog.rb +56 -6
  70. data/test/plugin/test_in_tail.rb +1 -1
  71. data/test/plugin/test_in_tcp.rb +25 -0
  72. data/test/plugin/test_out_forward.rb +75 -201
  73. data/test/plugin/test_out_http.rb +352 -0
  74. data/test/plugin/test_output_as_buffered.rb +27 -24
  75. data/test/plugin/test_parser.rb +40 -0
  76. data/test/plugin/test_parser_csv.rb +83 -0
  77. data/test/plugin_helper/test_record_accessor.rb +1 -1
  78. data/test/test_time_formatter.rb +140 -121
  79. metadata +33 -4
@@ -0,0 +1,352 @@
1
+ require_relative "../helper"
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_http'
4
+
5
+ require 'webrick'
6
+ require 'webrick/https'
7
+ require 'net/http'
8
+ require 'uri'
9
+ require 'json'
10
+
11
+ # WEBrick's ProcHandler doesn't handle PUT by default
12
+ module WEBrick::HTTPServlet
13
+ class ProcHandler < AbstractServlet
14
+ alias do_PUT do_GET
15
+ end
16
+ end
17
+
18
+ class HTTPOutputTest < Test::Unit::TestCase
19
+ include Fluent::Test::Helpers
20
+
21
+ TMP_DIR = File.join(__dir__, "../tmp/out_http#{ENV['TEST_ENV_NUMBER']}")
22
+ DEFAULT_LOGGER = ::WEBrick::Log.new(::STDOUT, ::WEBrick::BasicLog::FATAL)
23
+
24
+ class << self
25
+ # Use class variable to reduce server start/shutdown time
26
+ def startup
27
+ @@result = nil
28
+ @@auth_handler = nil
29
+ @@http_server_thread = nil
30
+ end
31
+
32
+ def shutdown
33
+ @@http_server_thread.kill
34
+ @@http_server_thread.join
35
+ rescue
36
+ end
37
+ end
38
+
39
+ def server_port
40
+ 19880
41
+ end
42
+
43
+ def base_endpoint
44
+ "http://127.0.0.1:#{server_port}"
45
+ end
46
+
47
+ def server_config
48
+ config = {BindAddress: '127.0.0.1', Port: server_port}
49
+ # Suppress webrick logs
50
+ config[:Logger] = DEFAULT_LOGGER
51
+ config[:AccessLog] = []
52
+ config
53
+ end
54
+
55
+ def http_client(**opts, &block)
56
+ opts = opts.merge(open_timeout: 1, read_timeout: 1)
57
+ if block_given?
58
+ Net::HTTP.start('127.0.0.1', server_port, **opts, &block)
59
+ else
60
+ Net::HTTP.start('127.0.0.1', server_port, **opts)
61
+ end
62
+ end
63
+
64
+ def run_http_server
65
+ server = ::WEBrick::HTTPServer.new(server_config)
66
+ server.mount_proc('/test') { |req, res|
67
+ if @@auth_handler
68
+ @@auth_handler.call(req, res)
69
+ end
70
+
71
+ @@result.method = req.request_method
72
+ @@result.content_type = req.content_type
73
+ req.each do |key, value|
74
+ @@result.headers[key] = value
75
+ end
76
+
77
+ data = []
78
+ case req.content_type
79
+ when 'application/x-ndjson'
80
+ req.body.each_line { |l|
81
+ data << JSON.parse(l)
82
+ }
83
+ when 'text/plain'
84
+ # Use single_value in this test
85
+ req.body.each_line { |line|
86
+ data << line.chomp
87
+ }
88
+ else
89
+ data << req.body
90
+ end
91
+ @@result.data = data
92
+
93
+ res.status = 200
94
+ res.body = "success"
95
+ }
96
+ server.mount_proc('/503') { |_, res|
97
+ res.status = 503
98
+ res.body = 'Service Unavailable'
99
+ }
100
+ server.mount_proc('/404') { |_, res|
101
+ res.status = 404
102
+ res.body = 'Not Found'
103
+ }
104
+ # For start check
105
+ server.mount_proc('/') { |_, res|
106
+ res.status = 200
107
+ res.body = 'Hello Fluentd!'
108
+ }
109
+ server.start
110
+ ensure
111
+ server.shutdown rescue nil
112
+ end
113
+
114
+ Result = Struct.new("Result", :method, :content_type, :headers, :data)
115
+
116
+ setup do
117
+ Fluent::Test.setup
118
+ FileUtils.rm_rf(TMP_DIR)
119
+
120
+ @@result = Result.new(nil, nil, {}, nil)
121
+ @@http_server_thread ||= Thread.new do
122
+ run_http_server
123
+ end
124
+
125
+ now = Time.now
126
+ started = false
127
+ until started
128
+ raise "Server not started" if (now - Time.now > 10.0)
129
+ begin
130
+ http_client { |c| c.request_get('/') }
131
+ started = true
132
+ rescue
133
+ sleep 0.5
134
+ end
135
+ end
136
+ end
137
+
138
+ teardown do
139
+ @@result = nil
140
+ @@auth_handler = nil
141
+ end
142
+
143
+ def create_driver(conf)
144
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::HTTPOutput).configure(conf)
145
+ end
146
+
147
+ def config
148
+ %[
149
+ endpoint #{base_endpoint}/test
150
+ ]
151
+ end
152
+
153
+ def test_events
154
+ [
155
+ {"message" => "hello", "num" => 10, "bool" => true},
156
+ {"message" => "hello", "num" => 11, "bool" => false}
157
+ ]
158
+ end
159
+
160
+ def test_configure
161
+ d = create_driver(config)
162
+ assert_equal "http://127.0.0.1:#{server_port}/test", d.instance.endpoint
163
+ assert_equal :post, d.instance.http_method
164
+ assert_equal 'application/x-ndjson', d.instance.content_type
165
+ assert_equal [503], d.instance.retryable_response_codes
166
+ assert_true d.instance.error_response_as_unrecoverable
167
+ assert_nil d.instance.proxy
168
+ assert_nil d.instance.headers
169
+ end
170
+
171
+ data('json' => ['json', 'application/x-ndjson'],
172
+ 'ltsv' => ['ltsv', 'text/tab-separated-values'],
173
+ 'msgpack' => ['msgpack', 'application/x-msgpack'],
174
+ 'single_value' => ['single_value', 'text/plain'])
175
+ def test_configure_content_type(types)
176
+ format_type, content_type = types
177
+ d = create_driver(config + %[
178
+ <format>
179
+ @type #{format_type}
180
+ </format>
181
+ ])
182
+ assert_equal content_type, d.instance.content_type
183
+ end
184
+
185
+ data('PUT' => 'put', 'POST' => 'post')
186
+ def test_write_with_method(method)
187
+ d = create_driver(config + "http_method #{method}")
188
+ d.run(default_tag: 'test.http') do
189
+ test_events.each { |event|
190
+ d.feed(event)
191
+ }
192
+ end
193
+
194
+ result = @@result
195
+ assert_equal method.upcase, result.method
196
+ assert_equal 'application/x-ndjson', result.content_type
197
+ assert_equal test_events, result.data
198
+ assert_not_empty result.headers
199
+ end
200
+
201
+ def test_write_with_single_value_format
202
+ d = create_driver(config + %[
203
+ <format>
204
+ @type single_value
205
+ </format>
206
+ ])
207
+ d.run(default_tag: 'test.http') do
208
+ test_events.each { |event|
209
+ d.feed(event)
210
+ }
211
+ end
212
+
213
+ result = @@result
214
+ assert_equal 'text/plain', result.content_type
215
+ assert_equal (test_events.map { |e| e['message'] }), result.data
216
+ assert_not_empty result.headers
217
+ end
218
+
219
+ def test_write_with_headers
220
+ d = create_driver(config + 'headers {"test_header":"fluentd!"}')
221
+ d.run(default_tag: 'test.http') do
222
+ test_events.each { |event|
223
+ d.feed(event)
224
+ }
225
+ end
226
+
227
+ result = @@result
228
+ assert_true result.headers.has_key?('test_header')
229
+ assert_equal "fluentd!", result.headers['test_header']
230
+ end
231
+
232
+ def test_write_with_retryable_response
233
+ d = create_driver("endpoint #{base_endpoint}/503")
234
+ assert_raise(Fluent::Plugin::HTTPOutput::RetryableResponse) do
235
+ d.run(default_tag: 'test.http', shutdown: false) do
236
+ test_events.each { |event|
237
+ d.feed(event)
238
+ }
239
+ end
240
+ end
241
+ d.instance_shutdown
242
+ end
243
+
244
+ def test_write_with_disabled_unrecoverable
245
+ d = create_driver(%[
246
+ endpoint #{base_endpoint}/404
247
+ error_response_as_unrecoverable false
248
+ ])
249
+ d.run(default_tag: 'test.http', shutdown: false) do
250
+ test_events.each { |event|
251
+ d.feed(event)
252
+ }
253
+ end
254
+ assert_match(/got error response from.*404 Not Found Not Found/, d.instance.log.out.logs.first)
255
+ d.instance_shutdown
256
+ end
257
+
258
+ sub_test_case 'basic auth' do
259
+ setup do
260
+ FileUtils.mkdir_p(TMP_DIR)
261
+ htpd = WEBrick::HTTPAuth::Htpasswd.new(File.join(TMP_DIR, 'dot.htpasswd'))
262
+ htpd.set_passwd(nil, 'test', 'hey')
263
+ authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpd, :Realm => 'test', :Logger => DEFAULT_LOGGER)
264
+ @@auth_handler = Proc.new { |req, res| authenticator.authenticate(req, res) }
265
+ end
266
+
267
+ teardown do
268
+ FileUtils.rm_rf(TMP_DIR)
269
+ end
270
+
271
+ def server_port
272
+ 19881
273
+ end
274
+
275
+ def test_basic_auth
276
+ d = create_driver(config + %[
277
+ <auth>
278
+ method basic
279
+ username test
280
+ password hey
281
+ </auth>
282
+ ])
283
+ d.run(default_tag: 'test.http') do
284
+ test_events.each { |event|
285
+ d.feed(event)
286
+ }
287
+ end
288
+
289
+ result = @@result
290
+ assert_equal 'POST', result.method
291
+ assert_equal 'application/x-ndjson', result.content_type
292
+ assert_equal test_events, result.data
293
+ assert_not_empty result.headers
294
+ end
295
+
296
+ # This test includes `error_response_as_unrecoverable true` behaviour check
297
+ def test_basic_auth_with_invalid_auth
298
+ d = create_driver(config + %[
299
+ <auth>
300
+ method basic
301
+ username ayaya
302
+ password hello?
303
+ </auth>
304
+ ])
305
+ d.run(default_tag: 'test.http', shutdown: false) do
306
+ test_events.each { |event|
307
+ d.feed(event)
308
+ }
309
+ end
310
+ assert_match(/got unrecoverable error/, d.instance.log.out.logs.first)
311
+
312
+ d.instance_shutdown
313
+ end
314
+ end
315
+
316
+ sub_test_case 'HTTPS' do
317
+ def server_port
318
+ 19882
319
+ end
320
+
321
+ def server_config
322
+ config = super
323
+ # WEBrick supports self-generated self-signed certificate
324
+ config[:SSLEnable] = true
325
+ config[:SSLCertName] = [["CN", WEBrick::Utils::getservername]]
326
+ config
327
+ end
328
+
329
+ def http_client(&block)
330
+ super(use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE, &block)
331
+ end
332
+
333
+ def test_write_with_https
334
+ d = create_driver(%[
335
+ endpoint https://127.0.0.1:#{server_port}/test
336
+ tls_verify_mode none
337
+ ssl_timeout 2s
338
+ ])
339
+ d.run(default_tag: 'test.http') do
340
+ test_events.each { |event|
341
+ d.feed(event)
342
+ }
343
+ end
344
+
345
+ result = @@result
346
+ assert_equal 'POST', result.method
347
+ assert_equal 'application/x-ndjson', result.content_type
348
+ assert_equal test_events, result.data
349
+ assert_not_empty result.headers
350
+ end
351
+ end
352
+ end
@@ -1079,40 +1079,43 @@ class BufferedOutputTest < Test::Unit::TestCase
1079
1079
  end
1080
1080
 
1081
1081
  sub_test_case 'buffered output with large timekey and small timekey_wait' do
1082
- setup do
1082
+ test 'writes event in proper interval' do
1083
1083
  chunk_key = 'time'
1084
1084
  hash = {
1085
+ 'timekey_zone' => '+0900',
1085
1086
  'timekey' => 86400, # per 1 day
1086
1087
  'timekey_wait' => 10, # 10 seconds delay for flush
1087
1088
  'flush_thread_count' => 1,
1088
1089
  'flush_thread_burst_interval' => 0.01,
1089
1090
  }
1090
- @i = create_output(:buffered)
1091
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1092
- @i.start
1093
- @i.after_start
1094
- end
1095
1091
 
1096
- test '#configure raises config error if timekey is not specified' do
1097
- Timecop.freeze( Time.parse('2019-02-08 00:01:00 +0900') )
1098
- ary = []
1099
- @i.register(:write){|chunk| ary << chunk.read }
1100
- @i.thread_wait_until_start
1101
- events = [
1102
- [event_time('2019-02-08 00:02:00 +0900'), {"message" => "foobar"}]
1103
- ]
1104
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
1105
- @i.enqueue_thread_wait
1106
- assert{ @i.write_count == 0 }
1092
+ with_timezone("UTC-9") do
1093
+ Timecop.freeze(Time.parse('2019-02-08 00:01:00 +0900'))
1094
+ @i = create_output(:buffered)
1095
+ # timezone is set
1096
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer',chunk_key,hash)]))
1097
+ @i.start
1098
+ @i.after_start
1099
+ @i.thread_wait_until_start
1100
+ assert_equal(0, @i.write_count)
1101
+ @i.interrupt_flushes
1107
1102
 
1108
- Timecop.freeze( Time.parse('2019-02-09 00:00:08 +0900') )
1109
- @i.enqueue_thread_wait
1110
- assert{ @i.write_count == 0 }
1103
+ events = [
1104
+ [event_time('2019-02-08 00:02:00 +0900'), { "message" => "foobar" }]
1105
+ ]
1106
+ @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
1107
+ @i.enqueue_thread_wait
1108
+ assert_equal(0, @i.write_count)
1111
1109
 
1112
- Timecop.freeze( Time.parse('2019-02-09 00:00:12 +0900') )
1113
- # wirte should be called in few seconds since
1114
- # running interval of enque thread is timekey_wait / 11.0.
1115
- waiting(5){ sleep 0.1 until @i.write_count == 1 }
1110
+ Timecop.freeze(Time.parse('2019-02-09 00:00:08 +0900'))
1111
+ @i.enqueue_thread_wait
1112
+ assert_equal(0, @i.write_count)
1113
+
1114
+ Timecop.freeze(Time.parse('2019-02-09 00:00:12 +0900'))
1115
+ # wirte should be called in few seconds since
1116
+ # running interval of enque thread is timekey_wait / 11.0.
1117
+ waiting(5){ sleep 0.1 until @i.write_count == 1 }
1118
+ end
1116
1119
  end
1117
1120
  end
1118
1121
 
@@ -356,4 +356,44 @@ class ParserTest < ::Test::Unit::TestCase
356
356
  end
357
357
  end
358
358
  end
359
+
360
+ sub_test_case 'timeout' do
361
+ class SleepParser < Fluent::Plugin::Parser
362
+ attr :test_value
363
+
364
+ def configure(conf)
365
+ super
366
+
367
+ @test_value = nil
368
+ end
369
+
370
+ def parse(data)
371
+ sleep 10
372
+ @test_value = :passed
373
+ yield JSON.parse(data), Fluent::EventTime.now
374
+ end
375
+ end
376
+
377
+ setup do
378
+ @i = SleepParser.new
379
+ @i.instance_variable_set(:@log, Fluent::Test::TestLogger.new)
380
+ @i.configure(config_element('parse', '', {'timeout' => '1.0'}))
381
+ @i.start
382
+ end
383
+
384
+ teardown do
385
+ @i.stop
386
+ end
387
+
388
+ test 'stop longer processing and return nil' do
389
+ waiting(10) {
390
+ @i.parse('{"k":"v"}') do |time, record|
391
+ assert_nil @i.test_value
392
+ assert_nil time
393
+ assert_nil record
394
+ end
395
+ assert_true @i.log.out.logs.first.include?('parsing timed out')
396
+ }
397
+ end
398
+ end
359
399
  end