fluentd 1.13.0-x64-mingw32 → 1.14.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/linux-test.yaml +1 -1
  5. data/.github/workflows/windows-test.yaml +4 -4
  6. data/.gitlab-ci.yml +0 -22
  7. data/CHANGELOG.md +131 -0
  8. data/README.md +2 -2
  9. data/example/v0_12_filter.conf +2 -2
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/command/fluentd.rb +8 -0
  12. data/lib/fluent/command/plugin_generator.rb +15 -5
  13. data/lib/fluent/compat/output.rb +9 -6
  14. data/lib/fluent/config/section.rb +5 -0
  15. data/lib/fluent/config/types.rb +15 -0
  16. data/lib/fluent/config/v1_parser.rb +3 -2
  17. data/lib/fluent/config.rb +1 -1
  18. data/lib/fluent/env.rb +2 -1
  19. data/lib/fluent/event_router.rb +28 -1
  20. data/lib/fluent/oj_options.rb +62 -0
  21. data/lib/fluent/plugin/bare_output.rb +49 -8
  22. data/lib/fluent/plugin/buffer.rb +84 -22
  23. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  24. data/lib/fluent/plugin/filter.rb +35 -1
  25. data/lib/fluent/plugin/formatter.rb +1 -0
  26. data/lib/fluent/plugin/formatter_json.rb +9 -7
  27. data/lib/fluent/plugin/in_http.rb +21 -2
  28. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  29. data/lib/fluent/plugin/in_syslog.rb +13 -1
  30. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  31. data/lib/fluent/plugin/in_tail.rb +45 -4
  32. data/lib/fluent/plugin/input.rb +39 -1
  33. data/lib/fluent/plugin/metrics.rb +119 -0
  34. data/lib/fluent/plugin/metrics_local.rb +96 -0
  35. data/lib/fluent/plugin/multi_output.rb +43 -6
  36. data/lib/fluent/plugin/out_forward.rb +1 -3
  37. data/lib/fluent/plugin/output.rb +74 -33
  38. data/lib/fluent/plugin/parser_json.rb +2 -3
  39. data/lib/fluent/plugin/service_discovery.rb +0 -15
  40. data/lib/fluent/plugin.rb +10 -1
  41. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  42. data/lib/fluent/plugin_helper/http_server/router.rb +1 -1
  43. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  44. data/lib/fluent/plugin_helper/server.rb +4 -2
  45. data/lib/fluent/plugin_helper.rb +1 -0
  46. data/lib/fluent/root_agent.rb +6 -0
  47. data/lib/fluent/supervisor.rb +2 -0
  48. data/lib/fluent/system_config.rb +9 -1
  49. data/lib/fluent/test/driver/storage.rb +30 -0
  50. data/lib/fluent/version.rb +1 -1
  51. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  52. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  53. data/test/command/test_cat.rb +11 -8
  54. data/test/command/test_plugin_generator.rb +2 -1
  55. data/test/config/test_section.rb +9 -0
  56. data/test/config/test_system_config.rb +6 -0
  57. data/test/config/test_types.rb +7 -0
  58. data/test/plugin/in_tail/test_position_file.rb +48 -8
  59. data/test/plugin/test_bare_output.rb +13 -0
  60. data/test/plugin/test_buffer.rb +8 -2
  61. data/test/plugin/test_file_wrapper.rb +11 -0
  62. data/test/plugin/test_filter.rb +11 -0
  63. data/test/plugin/test_in_forward.rb +59 -83
  64. data/test/plugin/test_in_http.rb +86 -43
  65. data/test/plugin/test_in_monitor_agent.rb +214 -8
  66. data/test/plugin/test_in_syslog.rb +101 -56
  67. data/test/plugin/test_in_tail.rb +149 -48
  68. data/test/plugin/test_in_tcp.rb +45 -32
  69. data/test/plugin/test_in_udp.rb +47 -33
  70. data/test/plugin/test_input.rb +11 -0
  71. data/test/plugin/test_metrics.rb +294 -0
  72. data/test/plugin/test_metrics_local.rb +96 -0
  73. data/test/plugin/test_multi_output.rb +25 -1
  74. data/test/plugin/test_out_forward.rb +103 -89
  75. data/test/plugin/test_out_stream.rb +18 -8
  76. data/test/plugin/test_output.rb +16 -0
  77. data/test/plugin_helper/http_server/test_route.rb +1 -1
  78. data/test/plugin_helper/test_child_process.rb +1 -1
  79. data/test/plugin_helper/test_event_emitter.rb +29 -0
  80. data/test/plugin_helper/test_http_server_helper.rb +33 -26
  81. data/test/plugin_helper/test_metrics.rb +137 -0
  82. data/test/plugin_helper/test_server.rb +137 -138
  83. data/test/plugin_helper/test_socket.rb +16 -9
  84. data/test/test_event_time.rb +2 -2
  85. data/test/test_oj_options.rb +55 -0
  86. data/test/test_plugin_classes.rb +102 -0
  87. data/test/test_root_agent.rb +30 -1
  88. metadata +22 -7
  89. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  90. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -5,42 +5,51 @@ require 'fluent/plugin/in_syslog'
5
5
  class SyslogInputTest < Test::Unit::TestCase
6
6
  def setup
7
7
  Fluent::Test.setup
8
+ @port = unused_port
8
9
  end
9
10
 
10
- PORT = unused_port
11
- CONFIG = %[
12
- port #{PORT}
13
- bind 127.0.0.1
14
- tag syslog
15
- ]
11
+ def teardown
12
+ @port = nil
13
+ end
14
+
15
+ def ipv4_config
16
+ %[
17
+ port #{@port}
18
+ bind 127.0.0.1
19
+ tag syslog
20
+ ]
21
+ end
16
22
 
17
- IPv6_CONFIG = %[
18
- port #{PORT}
19
- bind ::1
20
- tag syslog
21
- ]
23
+ def ipv6_config
24
+ %[
25
+ port #{@port}
26
+ bind ::1
27
+ tag syslog
28
+ ]
29
+ end
22
30
 
23
- def create_driver(conf=CONFIG)
31
+ def create_driver(conf=ipv4_config)
24
32
  Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)
25
33
  end
26
34
 
27
35
  data(
28
- ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
29
- ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
36
+ ipv4: ['127.0.0.1', :ipv4, ::Socket::AF_INET],
37
+ ipv6: ['::1', :ipv6, ::Socket::AF_INET6],
30
38
  )
31
39
  def test_configure(data)
32
- bind_addr, config, family = data
40
+ bind_addr, protocol, family = data
41
+ config = send("#{protocol}_config")
33
42
  omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
34
43
 
35
44
  d = create_driver(config)
36
- assert_equal PORT, d.instance.port
45
+ assert_equal @port, d.instance.port
37
46
  assert_equal bind_addr, d.instance.bind
38
47
  end
39
48
 
40
49
  sub_test_case 'source_hostname_key and source_address_key features' do
41
50
  test 'resolve_hostname must be true with source_hostname_key' do
42
51
  assert_raise(Fluent::ConfigError) {
43
- create_driver(CONFIG + <<EOS)
52
+ create_driver(ipv4_config + <<EOS)
44
53
  resolve_hostname false
45
54
  source_hostname_key hostname
46
55
  EOS
@@ -50,7 +59,7 @@ EOS
50
59
  data('resolve_hostname' => 'resolve_hostname true',
51
60
  'source_hostname_key' => 'source_hostname_key source_host')
52
61
  def test_configure_resolve_hostname(param)
53
- d = create_driver([CONFIG, param].join("\n"))
62
+ d = create_driver([ipv4_config, param].join("\n"))
54
63
  assert_true d.instance.resolve_hostname
55
64
  end
56
65
  end
@@ -60,7 +69,7 @@ EOS
60
69
  'Use transport and protocol' => ["protocol_type udp\n<transport tcp>\n </transport>", :udp, :tcp])
61
70
  def test_configure_protocol(param)
62
71
  conf, proto_type, transport_proto_type = *param
63
- d = create_driver([CONFIG, conf].join("\n"))
72
+ d = create_driver([ipv4_config, conf].join("\n"))
64
73
 
65
74
  assert_equal(d.instance.protocol_type, proto_type)
66
75
  assert_equal(d.instance.transport_config.protocol, transport_proto_type)
@@ -68,12 +77,12 @@ EOS
68
77
 
69
78
  # For backward compat
70
79
  def test_respect_protocol_type_than_transport
71
- d = create_driver([CONFIG, "<transport tcp> \n</transport>", "protocol_type udp"].join("\n"))
80
+ d = create_driver([ipv4_config, "<transport tcp> \n</transport>", "protocol_type udp"].join("\n"))
72
81
  tests = create_test_case
73
82
 
74
83
  d.run(expect_emits: 2) do
75
84
  u = UDPSocket.new
76
- u.connect('127.0.0.1', PORT)
85
+ u.connect('127.0.0.1', @port)
77
86
  tests.each {|test|
78
87
  u.send(test['msg'], 0)
79
88
  }
@@ -85,11 +94,12 @@ EOS
85
94
 
86
95
 
87
96
  data(
88
- ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
89
- ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
97
+ ipv4: ['127.0.0.1', :ipv4, ::Socket::AF_INET],
98
+ ipv6: ['::1', :ipv6, ::Socket::AF_INET6],
90
99
  )
91
100
  def test_time_format(data)
92
- bind_addr, config, family = data
101
+ bind_addr, protocol, family = data
102
+ config = send("#{protocol}_config")
93
103
  omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
94
104
 
95
105
  d = create_driver(config)
@@ -100,7 +110,7 @@ EOS
100
110
  ]
101
111
  d.run(expect_emits: 2) do
102
112
  u = UDPSocket.new(family)
103
- u.connect(bind_addr, PORT)
113
+ u.connect(bind_addr, @port)
104
114
  tests.each {|test|
105
115
  u.send(test['msg'], 0)
106
116
  }
@@ -119,7 +129,7 @@ EOS
119
129
 
120
130
  d.run(expect_emits: 2) do
121
131
  u = UDPSocket.new
122
- u.connect('127.0.0.1', PORT)
132
+ u.connect('127.0.0.1', @port)
123
133
  tests.each {|test|
124
134
  u.send(test['msg'], 0)
125
135
  }
@@ -130,14 +140,14 @@ EOS
130
140
  end
131
141
 
132
142
  def test_msg_size_udp_for_large_msg
133
- d = create_driver(CONFIG + %[
143
+ d = create_driver(ipv4_config + %[
134
144
  message_length_limit 5k
135
145
  ])
136
146
  tests = create_test_case(large_message: true)
137
147
 
138
148
  d.run(expect_emits: 3) do
139
149
  u = UDPSocket.new
140
- u.connect('127.0.0.1', PORT)
150
+ u.connect('127.0.0.1', @port)
141
151
  tests.each {|test|
142
152
  u.send(test['msg'], 0)
143
153
  }
@@ -148,12 +158,12 @@ EOS
148
158
  end
149
159
 
150
160
  def test_msg_size_with_tcp
151
- d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
161
+ d = create_driver([ipv4_config, "<transport tcp> \n</transport>"].join("\n"))
152
162
  tests = create_test_case
153
163
 
154
164
  d.run(expect_emits: 2) do
155
165
  tests.each {|test|
156
- TCPSocket.open('127.0.0.1', PORT) do |s|
166
+ TCPSocket.open('127.0.0.1', @port) do |s|
157
167
  s.send(test['msg'], 0)
158
168
  end
159
169
  }
@@ -164,12 +174,12 @@ EOS
164
174
  end
165
175
 
166
176
  def test_emit_rfc5452
167
- d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
177
+ d = create_driver([ipv4_config, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
168
178
  msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
169
179
 
170
180
  d.run(expect_emits: 1, timeout: 2) do
171
181
  u = UDPSocket.new
172
- u.connect('127.0.0.1', PORT)
182
+ u.connect('127.0.0.1', @port)
173
183
  u.send(msg, 0)
174
184
  end
175
185
 
@@ -179,11 +189,11 @@ EOS
179
189
  end
180
190
 
181
191
  def test_msg_size_with_same_tcp_connection
182
- d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
192
+ d = create_driver([ipv4_config, "<transport tcp> \n</transport>"].join("\n"))
183
193
  tests = create_test_case
184
194
 
185
195
  d.run(expect_emits: 2) do
186
- TCPSocket.open('127.0.0.1', PORT) do |s|
196
+ TCPSocket.open('127.0.0.1', @port) do |s|
187
197
  tests.each {|test|
188
198
  s.send(test['msg'], 0)
189
199
  }
@@ -195,7 +205,7 @@ EOS
195
205
  end
196
206
 
197
207
  def test_msg_size_with_json_format
198
- d = create_driver([CONFIG, 'format json'].join("\n"))
208
+ d = create_driver([ipv4_config, 'format json'].join("\n"))
199
209
  time = Time.parse('2013-09-18 12:00:00 +0900').to_i
200
210
  tests = ['Hello!', 'Syslog!'].map { |msg|
201
211
  event = {'time' => time, 'message' => msg}
@@ -204,7 +214,7 @@ EOS
204
214
 
205
215
  d.run(expect_emits: 2) do
206
216
  u = UDPSocket.new
207
- u.connect('127.0.0.1', PORT)
217
+ u.connect('127.0.0.1', @port)
208
218
  tests.each {|test|
209
219
  u.send(test['msg'], 0)
210
220
  }
@@ -215,13 +225,13 @@ EOS
215
225
  end
216
226
 
217
227
  def test_msg_size_with_include_source_host
218
- d = create_driver([CONFIG, 'include_source_host true'].join("\n"))
228
+ d = create_driver([ipv4_config, 'include_source_host true'].join("\n"))
219
229
  tests = create_test_case
220
230
 
221
231
  host = nil
222
232
  d.run(expect_emits: 2) do
223
233
  u = UDPSocket.new
224
- u.connect('127.0.0.1', PORT)
234
+ u.connect('127.0.0.1', @port)
225
235
  host = u.peeraddr[2]
226
236
  tests.each {|test|
227
237
  u.send(test['msg'], 0)
@@ -237,13 +247,13 @@ EOS
237
247
  priority_key: 'priority_key',
238
248
  )
239
249
  def test_msg_size_with_severity_key(param_name)
240
- d = create_driver([CONFIG, "#{param_name} severity"].join("\n"))
250
+ d = create_driver([ipv4_config, "#{param_name} severity"].join("\n"))
241
251
  tests = create_test_case
242
252
 
243
253
  severity = 'info'
244
254
  d.run(expect_emits: 2) do
245
255
  u = UDPSocket.new
246
- u.connect('127.0.0.1', PORT)
256
+ u.connect('127.0.0.1', @port)
247
257
  tests.each {|test|
248
258
  u.send(test['msg'], 0)
249
259
  }
@@ -254,13 +264,13 @@ EOS
254
264
  end
255
265
 
256
266
  def test_msg_size_with_facility_key
257
- d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
267
+ d = create_driver([ipv4_config, 'facility_key facility'].join("\n"))
258
268
  tests = create_test_case
259
269
 
260
270
  facility = 'kern'
261
271
  d.run(expect_emits: 2) do
262
272
  u = UDPSocket.new
263
- u.connect('127.0.0.1', PORT)
273
+ u.connect('127.0.0.1', @port)
264
274
  tests.each {|test|
265
275
  u.send(test['msg'], 0)
266
276
  }
@@ -271,13 +281,13 @@ EOS
271
281
  end
272
282
 
273
283
  def test_msg_size_with_source_address_key
274
- d = create_driver([CONFIG, 'source_address_key source_address'].join("\n"))
284
+ d = create_driver([ipv4_config, 'source_address_key source_address'].join("\n"))
275
285
  tests = create_test_case
276
286
 
277
287
  address = nil
278
288
  d.run(expect_emits: 2) do
279
289
  u = UDPSocket.new
280
- u.connect('127.0.0.1', PORT)
290
+ u.connect('127.0.0.1', @port)
281
291
  address = u.peeraddr[3]
282
292
  tests.each {|test|
283
293
  u.send(test['msg'], 0)
@@ -289,14 +299,14 @@ EOS
289
299
  end
290
300
 
291
301
  def test_msg_size_with_source_hostname_key
292
- d = create_driver([CONFIG, 'source_hostname_key source_hostname'].join("\n"))
302
+ d = create_driver([ipv4_config, 'source_hostname_key source_hostname'].join("\n"))
293
303
  tests = create_test_case
294
304
 
295
305
  hostname = nil
296
306
  d.run(expect_emits: 2) do
297
307
  u = UDPSocket.new
298
308
  u.do_not_reverse_lookup = false
299
- u.connect('127.0.0.1', PORT)
309
+ u.connect('127.0.0.1', @port)
300
310
  hostname = u.peeraddr[2]
301
311
  tests.each {|test|
302
312
  u.send(test['msg'], 0)
@@ -337,12 +347,12 @@ EOS
337
347
 
338
348
  sub_test_case 'octet counting frame' do
339
349
  def test_msg_size_with_tcp
340
- d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
350
+ d = create_driver([ipv4_config, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
341
351
  tests = create_test_case
342
352
 
343
353
  d.run(expect_emits: 2) do
344
354
  tests.each {|test|
345
- TCPSocket.open('127.0.0.1', PORT) do |s|
355
+ TCPSocket.open('127.0.0.1', @port) do |s|
346
356
  s.send(test['msg'], 0)
347
357
  end
348
358
  }
@@ -353,11 +363,11 @@ EOS
353
363
  end
354
364
 
355
365
  def test_msg_size_with_same_tcp_connection
356
- d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
366
+ d = create_driver([ipv4_config, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
357
367
  tests = create_test_case
358
368
 
359
369
  d.run(expect_emits: 2) do
360
- TCPSocket.open('127.0.0.1', PORT) do |s|
370
+ TCPSocket.open('127.0.0.1', @port) do |s|
361
371
  tests.each {|test|
362
372
  s.send(test['msg'], 0)
363
373
  }
@@ -404,13 +414,13 @@ EOS
404
414
  end
405
415
 
406
416
  def test_emit_unmatched_lines
407
- d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
417
+ d = create_driver([ipv4_config, 'emit_unmatched_lines true'].join("\n"))
408
418
  tests = create_unmatched_lines_test_case
409
419
 
410
420
  d.run(expect_emits: 3) do
411
421
  u = UDPSocket.new
412
422
  u.do_not_reverse_lookup = false
413
- u.connect('127.0.0.1', PORT)
423
+ u.connect('127.0.0.1', @port)
414
424
  tests.each {|test|
415
425
  u.send(test['msg'], 0)
416
426
  }
@@ -421,14 +431,14 @@ EOS
421
431
  end
422
432
 
423
433
  def test_emit_unmatched_lines_with_hostname
424
- d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join("\n"))
434
+ d = create_driver([ipv4_config, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join("\n"))
425
435
  tests = create_unmatched_lines_test_case
426
436
 
427
437
  hostname = nil
428
438
  d.run(expect_emits: 3) do
429
439
  u = UDPSocket.new
430
440
  u.do_not_reverse_lookup = false
431
- u.connect('127.0.0.1', PORT)
441
+ u.connect('127.0.0.1', @port)
432
442
  hostname = u.peeraddr[2]
433
443
  tests.each {|test|
434
444
  u.send(test['msg'], 0)
@@ -440,14 +450,14 @@ EOS
440
450
  end
441
451
 
442
452
  def test_emit_unmatched_lines_with_address
443
- d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_address_key source_address'].join("\n"))
453
+ d = create_driver([ipv4_config, 'emit_unmatched_lines true', 'source_address_key source_address'].join("\n"))
444
454
  tests = create_unmatched_lines_test_case
445
455
 
446
456
  address = nil
447
457
  d.run(expect_emits: 3) do
448
458
  u = UDPSocket.new
449
459
  u.do_not_reverse_lookup = false
450
- u.connect('127.0.0.1', PORT)
460
+ u.connect('127.0.0.1', @port)
451
461
  address = u.peeraddr[3]
452
462
  tests.each {|test|
453
463
  u.send(test['msg'], 0)
@@ -457,4 +467,39 @@ EOS
457
467
  assert_equal tests.size, d.events.size
458
468
  compare_unmatched_lines_test_result(d.events, tests, {address: address})
459
469
  end
470
+
471
+ def test_send_keepalive_packet_is_disabled_by_default
472
+ d = create_driver(ipv4_config + %[
473
+ <transport tcp>
474
+ </transport>
475
+ protocol tcp
476
+ ])
477
+ assert_false d.instance.send_keepalive_packet
478
+ end
479
+
480
+ def test_send_keepalive_packet_can_be_enabled
481
+ addr = "127.0.0.1"
482
+ d = create_driver(ipv4_config + %[
483
+ <transport tcp>
484
+ </transport>
485
+ send_keepalive_packet true
486
+ ])
487
+ assert_true d.instance.send_keepalive_packet
488
+ mock.proxy(d.instance).server_create_connection(
489
+ :in_syslog_tcp_server, @port,
490
+ bind: addr,
491
+ resolve_name: nil,
492
+ send_keepalive_packet: true)
493
+ d.run do
494
+ TCPSocket.open(addr, @port)
495
+ end
496
+ end
497
+
498
+ def test_send_keepalive_packet_can_not_be_enabled_for_udp
499
+ assert_raise(Fluent::ConfigError) do
500
+ d = create_driver(ipv4_config + %[
501
+ send_keepalive_packet true
502
+ ])
503
+ end
504
+ end
460
505
  end
@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
109
109
  "refresh_interval" => "1s",
110
110
  "read_from_head" => "true",
111
111
  "format" => "none",
112
+ "rotate_wait" => "1s",
112
113
  "follow_inodes" => "true"
113
114
  })
114
115
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -346,23 +347,15 @@ class TailInputTest < Test::Unit::TestCase
346
347
  end
347
348
 
348
349
  sub_test_case "reads_bytes_per_second w/o throttled" do
349
- data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2, false],
350
- "flat 8192 bytes, 2 events already read limit reached" => [:flat, 100, 8192, 2, true],
351
- "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2, false],
350
+ data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
351
+ "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2],
352
352
  "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
353
353
  "flat #{8192*10} bytes, 20 events w/o stat watcher" => [:flat_without_stat, 100, (8192 * 10), 20],
354
354
  "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
355
355
  "parse #{8192*4} bytes, 8 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 4), 8],
356
356
  "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
357
357
  "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
358
- "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2],
359
- "flat 8k bytes with unit, 2 events w/o stat watcher" => [:flat_without_stat, 100, "8k", 2],
360
- "flat #{8*10}k bytes with unit, 20 events" => [:flat, 100, "#{8*10}k", 20],
361
- "flat #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:flat_without_stat, 100, "#{8*10}k", 20],
362
- "parse #{8*4}k bytes with unit, 8 events" => [:parse, 100, "#{8*4}k", 8],
363
- "parse #{8*4}k bytes with unit, 8 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*4}k", 8],
364
- "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20],
365
- "parse #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*10}k", 20])
358
+ "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2])
366
359
  def test_emit_with_read_bytes_limit_per_second(data)
367
360
  config_style, limit, limit_bytes, num_events = data
368
361
  case config_style
@@ -418,16 +411,10 @@ class TailInputTest < Test::Unit::TestCase
418
411
  end
419
412
 
420
413
  sub_test_case "reads_bytes_per_second w/ throttled already" do
421
- data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
422
- "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
423
- "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
424
- "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
425
- "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2],
426
- "flat #{8*10}k bytes with unit, 20 events" => [:flat, 100, "#{8*10}k", 20],
427
- "parse #{8*4}k bytes with unit, 8 events" => [:parse, 100, "#{8*4}k", 8],
428
- "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20])
414
+ data("flat 8192 bytes" => [:flat, 100, 8192],
415
+ "parse 8192 bytes" => [:parse, 100, 8192])
429
416
  def test_emit_with_read_bytes_limit_per_second(data)
430
- config_style, limit, limit_bytes, num_events = data
417
+ config_style, limit, limit_bytes = data
431
418
  case config_style
432
419
  when :flat
433
420
  config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
@@ -470,6 +457,91 @@ class TailInputTest < Test::Unit::TestCase
470
457
  assert_equal([], d.events)
471
458
  end
472
459
  end
460
+
461
+ sub_test_case "EOF with reads_bytes_per_second" do
462
+ def test_longer_than_rotate_wait
463
+ limit_bytes = 8192
464
+ num_lines = 1024 * 3
465
+ msg = "08bytes"
466
+
467
+ File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
468
+ f.write("#{msg}\n" * num_lines)
469
+ end
470
+
471
+ config = CONFIG_READ_FROM_HEAD +
472
+ SINGLE_LINE_CONFIG +
473
+ config_element("", "", {
474
+ "read_bytes_limit_per_second" => limit_bytes,
475
+ "rotate_wait" => 0.1,
476
+ "refresh_interval" => 0.5,
477
+ })
478
+
479
+ rotated = false
480
+ d = create_driver(config)
481
+ d.run(timeout: 10) do
482
+ while d.events.size < num_lines do
483
+ if d.events.size > 0 && !rotated
484
+ cleanup_file("#{TMP_DIR}/tail.txt")
485
+ FileUtils.touch("#{TMP_DIR}/tail.txt")
486
+ rotated = true
487
+ end
488
+ sleep 0.3
489
+ end
490
+ end
491
+
492
+ assert_equal(num_lines,
493
+ d.events.count do |event|
494
+ event[2]["message"] == msg
495
+ end)
496
+ end
497
+
498
+ def test_shorter_than_rotate_wait
499
+ limit_bytes = 8192
500
+ num_lines = 1024 * 2
501
+ msg = "08bytes"
502
+
503
+ File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
504
+ f.write("#{msg}\n" * num_lines)
505
+ end
506
+
507
+ config = CONFIG_READ_FROM_HEAD +
508
+ SINGLE_LINE_CONFIG +
509
+ config_element("", "", {
510
+ "read_bytes_limit_per_second" => limit_bytes,
511
+ "rotate_wait" => 2,
512
+ "refresh_interval" => 0.5,
513
+ })
514
+
515
+ start_time = Fluent::Clock.now
516
+ rotated = false
517
+ detached = false
518
+ d = create_driver(config)
519
+ mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
520
+ mock.proxy(tw).detach(anything) do |v|
521
+ detached = true
522
+ v
523
+ end
524
+ tw
525
+ end.twice
526
+
527
+ d.run(timeout: 10) do
528
+ until detached do
529
+ if d.events.size > 0 && !rotated
530
+ cleanup_file("#{TMP_DIR}/tail.txt")
531
+ FileUtils.touch("#{TMP_DIR}/tail.txt")
532
+ rotated = true
533
+ end
534
+ sleep 0.3
535
+ end
536
+ end
537
+
538
+ assert_true(Fluent::Clock.now - start_time > 2)
539
+ assert_equal(num_lines,
540
+ d.events.count do |event|
541
+ event[2]["message"] == msg
542
+ end)
543
+ end
544
+ end
473
545
  end
474
546
 
475
547
  data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -1667,7 +1739,7 @@ class TailInputTest < Test::Unit::TestCase
1667
1739
  d.instance_shutdown
1668
1740
  end
1669
1741
 
1670
- def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1742
+ def test_should_remove_deleted_file
1671
1743
  config = config_element("", "", {"format" => "none"})
1672
1744
 
1673
1745
  path = "#{TMP_DIR}/tail.txt"
@@ -1678,30 +1750,11 @@ class TailInputTest < Test::Unit::TestCase
1678
1750
  }
1679
1751
 
1680
1752
  d = create_driver(config)
1681
- d.run(shutdown: false)
1682
-
1683
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1684
- pos_file.pos = 0
1685
-
1686
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1687
- assert_equal(path, path_pos_ino[1])
1688
- assert_equal(pos, path_pos_ino[2].to_i(16))
1689
- assert_equal(ino, path_pos_ino[3].to_i(16))
1690
-
1691
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1692
- f.puts "test1"
1693
- f.puts "test2"
1694
- }
1695
- Timecop.travel(Time.now + 10) do
1696
- sleep 5
1753
+ d.run do
1754
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1697
1755
  pos_file.pos = 0
1698
- tuple = create_target_info("#{TMP_DIR}/tail.txt")
1699
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1700
- assert_equal(tuple.path, path_pos_ino[1])
1701
- assert_equal(12, path_pos_ino[2].to_i(16))
1702
- assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1756
+ assert_equal([], pos_file.readlines)
1703
1757
  end
1704
- d.instance_shutdown
1705
1758
  end
1706
1759
 
1707
1760
  def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -1913,6 +1966,49 @@ class TailInputTest < Test::Unit::TestCase
1913
1966
  assert_equal({"message" => "test4"}, events[3][2])
1914
1967
  d.instance_shutdown
1915
1968
  end
1969
+
1970
+ # issue #3464
1971
+ def test_should_replace_target_info
1972
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1973
+ f.puts "test1\n"
1974
+ }
1975
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1976
+ inodes = []
1977
+
1978
+ config = config_element("ROOT", "", {
1979
+ "path" => "#{TMP_DIR}/tail.txt*",
1980
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1981
+ "tag" => "t1",
1982
+ "refresh_interval" => "60s",
1983
+ "read_from_head" => "true",
1984
+ "format" => "none",
1985
+ "rotate_wait" => "1s",
1986
+ "follow_inodes" => "true"
1987
+ })
1988
+ d = create_driver(config, false)
1989
+ d.run(timeout: 5) do
1990
+ while d.events.size < 1 do
1991
+ sleep 0.1
1992
+ end
1993
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
1994
+ key.ino
1995
+ end
1996
+ assert_equal([target_info.ino], inodes)
1997
+
1998
+ cleanup_file("#{TMP_DIR}/tail.txt")
1999
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
2000
+
2001
+ while d.events.size < 2 do
2002
+ sleep 0.1
2003
+ end
2004
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2005
+ key.ino
2006
+ end
2007
+ new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
2008
+ assert_not_equal(target_info.ino, new_target_info.ino)
2009
+ assert_equal([new_target_info.ino], inodes)
2010
+ end
2011
+ end
1916
2012
  end
1917
2013
 
1918
2014
  sub_test_case "tail_path" do
@@ -2083,11 +2179,13 @@ class TailInputTest < Test::Unit::TestCase
2083
2179
  assert_nothing_raised do
2084
2180
  d.run(shutdown: false) {}
2085
2181
  end
2086
- d.instance_shutdown
2087
2182
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2183
+ ensure
2184
+ d.instance_shutdown if d && d.instance
2088
2185
  end
2089
2186
 
2090
2187
  def test_EACCES_error_after_setup_watcher
2188
+ omit "Cannot test with root user" if Process::UID.eid == 0
2091
2189
  path = "#{TMP_DIR}/noaccess/tail.txt"
2092
2190
  begin
2093
2191
  FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
@@ -2106,12 +2204,14 @@ class TailInputTest < Test::Unit::TestCase
2106
2204
  assert_nothing_raised do
2107
2205
  d.run(shutdown: false) {}
2108
2206
  end
2109
- d.instance_shutdown
2110
2207
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2111
2208
  end
2112
2209
  ensure
2113
- FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2114
- FileUtils.rm_rf("#{TMP_DIR}/noaccess")
2210
+ d.instance_shutdown if d && d.instance
2211
+ if File.exist?("#{TMP_DIR}/noaccess")
2212
+ FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2213
+ FileUtils.rm_rf("#{TMP_DIR}/noaccess")
2214
+ end
2115
2215
  end unless Fluent.windows?
2116
2216
 
2117
2217
  def test_EACCES
@@ -2127,8 +2227,9 @@ class TailInputTest < Test::Unit::TestCase
2127
2227
  assert_nothing_raised do
2128
2228
  d.run(shutdown: false) {}
2129
2229
  end
2130
- d.instance_shutdown
2131
2230
  assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2231
+ ensure
2232
+ d.instance_shutdown if d && d.instance
2132
2233
  end
2133
2234
 
2134
2235
  def test_shutdown_timeout