fluentd 1.9.3 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/issue-auto-closer.yml +12 -0
  3. data/CHANGELOG.md +182 -0
  4. data/CONTRIBUTING.md +1 -1
  5. data/README.md +4 -0
  6. data/docs/SECURITY_AUDIT.pdf +0 -0
  7. data/lib/fluent/command/debug.rb +1 -0
  8. data/lib/fluent/command/fluentd.rb +25 -1
  9. data/lib/fluent/config.rb +1 -0
  10. data/lib/fluent/daemonizer.rb +88 -0
  11. data/lib/fluent/log.rb +45 -6
  12. data/lib/fluent/match.rb +1 -1
  13. data/lib/fluent/msgpack_factory.rb +13 -6
  14. data/lib/fluent/plugin/buffer.rb +2 -2
  15. data/lib/fluent/plugin/in_dummy.rb +3 -3
  16. data/lib/fluent/plugin/in_forward.rb +2 -2
  17. data/lib/fluent/plugin/in_gc_stat.rb +16 -0
  18. data/lib/fluent/plugin/in_http.rb +146 -75
  19. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  20. data/lib/fluent/plugin/in_syslog.rb +4 -4
  21. data/lib/fluent/plugin/in_tail.rb +40 -31
  22. data/lib/fluent/plugin/in_tail/position_file.rb +23 -6
  23. data/lib/fluent/plugin/in_unix.rb +77 -77
  24. data/lib/fluent/plugin/out_copy.rb +1 -1
  25. data/lib/fluent/plugin/out_file.rb +1 -1
  26. data/lib/fluent/plugin/out_forward.rb +25 -22
  27. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +4 -0
  28. data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
  29. data/lib/fluent/plugin/out_http.rb +15 -2
  30. data/lib/fluent/plugin/parser_multiline.rb +1 -1
  31. data/lib/fluent/plugin/parser_syslog.rb +303 -62
  32. data/lib/fluent/plugin/sd_file.rb +1 -0
  33. data/lib/fluent/plugin/sd_srv.rb +135 -0
  34. data/lib/fluent/plugin_helper/cert_option.rb +15 -2
  35. data/lib/fluent/plugin_helper/child_process.rb +3 -2
  36. data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
  37. data/lib/fluent/plugin_helper/server.rb +3 -1
  38. data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
  39. data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
  40. data/lib/fluent/plugin_helper/socket.rb +20 -2
  41. data/lib/fluent/plugin_helper/socket_option.rb +21 -3
  42. data/lib/fluent/supervisor.rb +21 -9
  43. data/lib/fluent/system_config.rb +2 -1
  44. data/lib/fluent/test/filter_test.rb +2 -2
  45. data/lib/fluent/test/output_test.rb +3 -3
  46. data/lib/fluent/version.rb +1 -1
  47. data/test/command/test_fluentd.rb +71 -12
  48. data/test/config/test_system_config.rb +2 -0
  49. data/test/helper.rb +2 -2
  50. data/test/plugin/in_tail/test_fifo.rb +121 -0
  51. data/test/plugin/in_tail/test_io_handler.rb +132 -0
  52. data/test/plugin/in_tail/test_position_file.rb +25 -1
  53. data/test/plugin/out_forward/test_handshake_protocol.rb +10 -1
  54. data/test/plugin/out_forward/test_load_balancer.rb +46 -0
  55. data/test/plugin/test_buf_file.rb +3 -1
  56. data/test/plugin/test_buffer.rb +20 -0
  57. data/test/plugin/test_compressable.rb +7 -4
  58. data/test/plugin/test_in_dummy.rb +12 -14
  59. data/test/plugin/test_in_forward.rb +2 -2
  60. data/test/plugin/test_in_gc_stat.rb +24 -1
  61. data/test/plugin/test_in_http.rb +57 -0
  62. data/test/plugin/test_in_syslog.rb +16 -1
  63. data/test/plugin/test_in_tail.rb +43 -20
  64. data/test/plugin/test_in_unix.rb +128 -73
  65. data/test/plugin/test_out_forward.rb +39 -3
  66. data/test/plugin/test_out_http.rb +38 -0
  67. data/test/plugin/test_out_null.rb +1 -1
  68. data/test/plugin/test_output_as_buffered_retries.rb +12 -4
  69. data/test/plugin/test_output_as_buffered_secondary.rb +9 -1
  70. data/test/plugin/test_parser_syslog.rb +106 -46
  71. data/test/plugin/test_sd_file.rb +17 -0
  72. data/test/plugin/test_sd_srv.rb +230 -0
  73. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  74. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  75. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  76. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  77. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  78. data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
  79. data/test/plugin_helper/http_server/test_app.rb +1 -1
  80. data/test/plugin_helper/http_server/test_route.rb +1 -1
  81. data/test/plugin_helper/test_cert_option.rb +2 -0
  82. data/test/plugin_helper/test_child_process.rb +20 -3
  83. data/test/plugin_helper/test_http_server_helper.rb +2 -2
  84. data/test/plugin_helper/test_record_accessor.rb +41 -0
  85. data/test/plugin_helper/test_server.rb +1 -1
  86. data/test/plugin_helper/test_service_discovery.rb +37 -4
  87. data/test/plugin_helper/test_socket.rb +131 -0
  88. data/test/test_daemonizer.rb +91 -0
  89. data/test/test_log.rb +44 -0
  90. data/test/test_msgpack_factory.rb +18 -0
  91. metadata +28 -2
@@ -6,7 +6,7 @@ require 'tempfile'
6
6
 
7
7
  class IntailPositionFileTest < Test::Unit::TestCase
8
8
  setup do
9
- @file = Tempfile.new('intail_position_file_test')
9
+ @file = Tempfile.new('intail_position_file_test').binmode
10
10
  end
11
11
 
12
12
  teardown do
@@ -77,6 +77,30 @@ class IntailPositionFileTest < Test::Unit::TestCase
77
77
  lines = @file.readlines
78
78
  assert_equal 5, lines.size
79
79
  end
80
+
81
+ test 'update seek postion of remained position entry' do
82
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
83
+ pf['path1']
84
+ pf['path2']
85
+ pf['path3']
86
+ pf.unwatch('path1')
87
+
88
+ pf.try_compact
89
+
90
+ @file.seek(0)
91
+ lines = @file.readlines
92
+ assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0]
93
+ assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
94
+ assert_equal 2, lines.size
95
+
96
+ pf.unwatch('path2')
97
+ pf.unwatch('path3')
98
+ @file.seek(0)
99
+ lines = @file.readlines
100
+ assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
101
+ assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
102
+ assert_equal 2, lines.size
103
+ end
80
104
  end
81
105
 
82
106
  sub_test_case '#load' do
@@ -81,6 +81,15 @@ class HandshakeProtocolTest < Test::Unit::TestCase
81
81
  assert_equal(ri.state, :established)
82
82
  end
83
83
 
84
+ test 'raises an error when password and username are nil if auth exists' do
85
+ handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
86
+ ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
87
+
88
+ assert_raise(Fluent::Plugin::ForwardOutput::PingpongError.new('username and password are required')) do
89
+ handshake.invoke('', ri, ['HELO', { 'auth' => 'auth' }])
90
+ end
91
+ end
92
+
84
93
  data(
85
94
  lack_of_elem: ['PONG', true, '', 'client_hostname'],
86
95
  wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
@@ -89,7 +98,7 @@ class HandshakeProtocolTest < Test::Unit::TestCase
89
98
  wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],
90
99
  )
91
100
  test 'raises an error when message is' do |msg|
92
- handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
101
+ handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: '', username: '')
93
102
  handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
94
103
 
95
104
  ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
49
49
  end
50
50
  end
51
51
 
52
+ test 'call like round robin without weight=0 node' do
53
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
54
+ n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
55
+ n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
56
+ n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
57
+
58
+ lb.rebuild_weight_array([n1, n2, n3])
59
+
60
+ lb.select_healthy_node do |node|
61
+ # to handle random choice
62
+ if node == n1
63
+ lb.select_healthy_node do |node|
64
+ assert_equal(node, n2)
65
+ end
66
+
67
+ lb.select_healthy_node do |node|
68
+ assert_equal(node, n1)
69
+ end
70
+
71
+ lb.select_healthy_node do |node|
72
+ assert_equal(node, n2)
73
+ end
74
+ else
75
+ lb.select_healthy_node do |node|
76
+ assert_equal(node, n1)
77
+ end
78
+
79
+ lb.select_healthy_node do |node|
80
+ assert_equal(node, n2)
81
+ end
82
+
83
+ lb.select_healthy_node do |node|
84
+ assert_equal(node, n1)
85
+ end
86
+ end
87
+ end
88
+ end
89
+
52
90
  test 'raise an error if all node are unavialble' do
53
91
  lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
54
92
  lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
56
94
  lb.select_healthy_node
57
95
  end
58
96
  end
97
+
98
+ test 'it regards weight=0 node as unavialble' do
99
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
100
+ lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
101
+ assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
102
+ lb.select_healthy_node
103
+ end
104
+ end
59
105
  end
60
106
  end
@@ -1062,7 +1062,9 @@ class FileBufferTest < Test::Unit::TestCase
1062
1062
  if @bufdir
1063
1063
  Dir.glob(File.join(@bufdir, '*')).each do |path|
1064
1064
  next if ['.', '..'].include?(File.basename(path))
1065
- File.delete(path)
1065
+ # Windows does not permit to delete files which are used in another process.
1066
+ # Just ignore for removing failure.
1067
+ File.delete(path) rescue nil
1066
1068
  end
1067
1069
  end
1068
1070
  end
@@ -1202,4 +1202,24 @@ class BufferTest < Test::Unit::TestCase
1202
1202
  assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
1203
1203
  end
1204
1204
  end
1205
+
1206
+ sub_test_case '#statistics' do
1207
+ setup do
1208
+ @p = create_buffer({ "total_limit_size" => 1024 })
1209
+ dm = create_metadata(Time.parse('2020-03-13 16:00:00 +0000').to_i, nil, nil)
1210
+
1211
+ (class << @p; self; end).module_eval do
1212
+ define_method(:resume) {
1213
+ queued = [create_chunk(dm, ["a" * (1024 - 102)]).enqueued!]
1214
+ return {}, queued
1215
+ }
1216
+ end
1217
+
1218
+ @p.start
1219
+ end
1220
+
1221
+ test 'returns available_buffer_space_ratios' do
1222
+ assert_equal 10.0, @p.statistics['buffer']['available_buffer_space_ratios']
1223
+ end
1224
+ end
1205
1225
  end
@@ -4,6 +4,12 @@ require 'fluent/plugin/compressable'
4
4
  class CompressableTest < Test::Unit::TestCase
5
5
  include Fluent::Plugin::Compressable
6
6
 
7
+ def compress_assert_equal(expected, actual)
8
+ e = Zlib::GzipReader.new(StringIO.new(expected)).read
9
+ a = Zlib::GzipReader.new(StringIO.new(actual)).read
10
+ assert_equal(e, a)
11
+ end
12
+
7
13
  sub_test_case '#compress' do
8
14
  setup do
9
15
  @src = 'text data for compressing' * 5
@@ -18,8 +24,7 @@ class CompressableTest < Test::Unit::TestCase
18
24
  test 'write compressed data to IO with output_io option' do
19
25
  io = StringIO.new
20
26
  compress(@src, output_io: io)
21
- waiting(10){ sleep 0.1 until @gzipped_src == io.string }
22
- assert_equal @gzipped_src, io.string
27
+ compress_assert_equal @gzipped_src, io.string
23
28
  end
24
29
  end
25
30
 
@@ -36,7 +41,6 @@ class CompressableTest < Test::Unit::TestCase
36
41
  test 'write decompressed data to IO with output_io option' do
37
42
  io = StringIO.new
38
43
  decompress(@gzipped_src, output_io: io)
39
- waiting(10){ sleep 0.1 until @src == io.string }
40
44
  assert_equal @src, io.string
41
45
  end
42
46
 
@@ -58,7 +62,6 @@ class CompressableTest < Test::Unit::TestCase
58
62
  output_io = StringIO.new
59
63
 
60
64
  decompress(input_io: input_io, output_io: output_io)
61
- waiting(10){ sleep 0.1 until @src == output_io.string }
62
65
  assert_equal @src, output_io.string
63
66
  end
64
67
 
@@ -95,21 +95,20 @@ class DummyTest < Test::Unit::TestCase
95
95
  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
96
96
  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
97
97
 
98
- sub_test_case "doesn't suspend internal counters in default" do
98
+ sub_test_case 'when dummy plugin has storage which is not specified the path' do
99
99
  config1 = {
100
100
  'tag' => 'dummy',
101
- 'rate' => '2',
101
+ 'rate' => '0',
102
102
  'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
103
103
  'auto_increment_key' => 'id',
104
- 'suspend' => false,
105
104
  }
106
105
  conf1 = config_element('ROOT', '', config1, [])
107
- test "value of auto increment key is not suspended after stop-and-start" do
106
+ test "value of auto increment key is not kept after stop-and-start" do
108
107
  assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
109
108
 
110
109
  d1 = create_driver(conf1)
111
110
  d1.run(timeout: 0.5) do
112
- d1.instance.emit(4)
111
+ d1.instance.emit(2)
113
112
  end
114
113
 
115
114
  events = d1.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
@@ -124,7 +123,7 @@ class DummyTest < Test::Unit::TestCase
124
123
 
125
124
  d2 = create_driver(conf1)
126
125
  d2.run(timeout: 0.5) do
127
- d2.instance.emit(4)
126
+ d2.instance.emit(2)
128
127
  end
129
128
 
130
129
  events = d2.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
@@ -136,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
136
135
  end
137
136
  end
138
137
 
139
- sub_test_case "suspend internal counters if suspend is true" do
138
+ sub_test_case 'when dummy plugin has storage which is specified the path' do
140
139
  setup do
141
140
  FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
142
141
  FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -146,10 +145,9 @@ class DummyTest < Test::Unit::TestCase
146
145
  config2 = {
147
146
  '@id' => 'test-02',
148
147
  'tag' => 'dummy',
149
- 'rate' => '2',
148
+ 'rate' => '0',
150
149
  'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
151
150
  'auto_increment_key' => 'id',
152
- 'suspend' => true,
153
151
  }
154
152
  conf2 = config_element('ROOT', '', config2, [
155
153
  config_element(
@@ -161,12 +159,12 @@ class DummyTest < Test::Unit::TestCase
161
159
  'persistent' => true,
162
160
  })
163
161
  ])
164
- test "value of auto increment key is suspended after stop-and-start" do
162
+ test "value of auto increment key is kept after stop-and-start" do
165
163
  assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
166
164
 
167
165
  d1 = create_driver(conf2)
168
- d1.run(timeout: 0.5) do
169
- d1.instance.emit(4)
166
+ d1.run(timeout: 1) do
167
+ d1.instance.emit(2)
170
168
  end
171
169
 
172
170
  first_id1 = d1.events.first[2]['id']
@@ -178,8 +176,8 @@ class DummyTest < Test::Unit::TestCase
178
176
  assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
179
177
 
180
178
  d2 = create_driver(conf2)
181
- d2.run(timeout: 0.5) do
182
- d2.instance.emit(4)
179
+ d2.run(timeout: 1) do
180
+ d2.instance.emit(2)
183
181
  end
184
182
  d2.events
185
183
 
@@ -218,7 +218,7 @@ class ForwardInputTest < Test::Unit::TestCase
218
218
  }
219
219
  end
220
220
 
221
- assert_equal(records, d.events.sort_by {|a| a[1] })
221
+ assert_equal(records, d.events.sort_by {|a| a[0] })
222
222
  end
223
223
 
224
224
  test 'json_with_newline' do
@@ -237,7 +237,7 @@ class ForwardInputTest < Test::Unit::TestCase
237
237
  }
238
238
  end
239
239
 
240
- assert_equal(records, d.events.sort_by {|a| a[1] })
240
+ assert_equal(records, d.events.sort_by {|a| a[0] })
241
241
  end
242
242
  end
243
243
 
@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
22
22
  assert_equal("t1", d.instance.tag)
23
23
  end
24
24
 
25
- def test_emit
25
+ def setup_gc_stat
26
26
  stat = GC.stat
27
27
  stub(GC).stat { stat }
28
+ stat
29
+ end
30
+
31
+ def test_emit
32
+ stat = setup_gc_stat
28
33
 
29
34
  d = create_driver
30
35
  d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
36
41
  assert(events[i][1].is_a?(Fluent::EventTime))
37
42
  }
38
43
  end
44
+
45
+ def test_emit_with_use_symbol_keys_false
46
+ stat = setup_gc_stat
47
+ result = {}
48
+ stat.each_pair { |k, v|
49
+ result[k.to_s] = v
50
+ }
51
+
52
+ d = create_driver(CONFIG + "use_symbol_keys false")
53
+ d.run(expect_emits: 2)
54
+
55
+ events = d.events
56
+ assert(events.length > 0)
57
+ events.each_index {|i|
58
+ assert_equal(result, events[i][2])
59
+ assert(events[i][1].is_a?(Fluent::EventTime))
60
+ }
61
+ end
39
62
  end
@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
116
116
  assert_equal_event_time time, d.events[1][1]
117
117
  end
118
118
 
119
+ data('json' => ['json', :to_json],
120
+ 'msgpack' => ['msgpack', :to_msgpack])
121
+ def test_default_with_time_format(data)
122
+ param, method_name = data
123
+ d = create_driver(CONFIG + %[
124
+ <parse>
125
+ keep_time_key
126
+ time_format %iso8601
127
+ </parse>
128
+ ])
129
+
130
+ time = event_time("2020-06-10T01:14:27+00:00")
131
+ events = [
132
+ ["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
133
+ ["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
134
+ ]
135
+ res_codes = []
136
+
137
+ d.run(expect_records: 2) do
138
+ events.each do |tag, t, record|
139
+ res = post("/#{tag}", {param => record.__send__(method_name)})
140
+ res_codes << res.code
141
+ end
142
+ end
143
+ assert_equal ["200", "200"], res_codes
144
+ assert_equal events, d.events
145
+ assert_equal_event_time time, d.events[0][1]
146
+ assert_equal_event_time time, d.events[1][1]
147
+ end
148
+
119
149
  def test_multi_json
120
150
  d = create_driver
121
151
  time = event_time("2011-01-02 13:14:15 UTC")
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
163
193
  assert_equal_event_time time, d.events[1][1]
164
194
  end
165
195
 
196
+ data('json' => ['json', :to_json],
197
+ 'msgpack' => ['msgpack', :to_msgpack])
198
+ def test_default_multi_with_time_format(data)
199
+ param, method_name = data
200
+ d = create_driver(CONFIG + %[
201
+ <parse>
202
+ keep_time_key
203
+ time_format %iso8601
204
+ </parse>
205
+ ])
206
+ time = event_time("2020-06-10T01:14:27+00:00")
207
+ events = [
208
+ ["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
209
+ ["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
210
+ ]
211
+ tag = "tag1"
212
+ res_codes = []
213
+ d.run(expect_records: 2, timeout: 5) do
214
+ res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
215
+ res_codes << res.code
216
+ end
217
+ assert_equal ["200"], res_codes
218
+ assert_equal events, d.events
219
+ assert_equal_event_time time, d.events[0][1]
220
+ assert_equal_event_time time, d.events[1][1]
221
+ end
222
+
166
223
  def test_multi_json_with_nonexistent_time_key
167
224
  d = create_driver(CONFIG + %[
168
225
  <parse>
@@ -163,6 +163,21 @@ EOS
163
163
  compare_test_result(d.events, tests)
164
164
  end
165
165
 
166
+ def test_emit_rfc5452
167
+ d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
168
+ msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
169
+
170
+ d.run(expect_emits: 1, timeout: 2) do
171
+ u = UDPSocket.new
172
+ u.connect('127.0.0.1', PORT)
173
+ u.send(msg, 0)
174
+ end
175
+
176
+ tag, _, event = d.events[0]
177
+ assert_equal('syslog.kern.alert', tag)
178
+ assert_equal('kern', event['pri'])
179
+ end
180
+
166
181
  def test_msg_size_with_same_tcp_connection
167
182
  d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
168
183
  tests = create_test_case
@@ -360,7 +375,7 @@ EOS
360
375
  ]
361
376
  msgs.each { |msg|
362
377
  m = msg['msg']
363
- msg['msg'] = "#{m.size + 1} #{m}"
378
+ msg['msg'] = "#{m.size} #{m}"
364
379
  }
365
380
  msgs
366
381
  end
@@ -23,9 +23,6 @@ class TailInputTest < Test::Unit::TestCase
23
23
  def cleanup_directory(path)
24
24
  FileUtils.rm_rf(path, secure: true)
25
25
  if File.exist?(path)
26
- # ensure files are closed for Windows, on which deleted files
27
- # are still visible from filesystem
28
- GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
29
26
  FileUtils.remove_entry_secure(path, true)
30
27
  end
31
28
  FileUtils.mkdir_p(path)
@@ -105,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
105
102
  assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
106
103
  end
107
104
 
105
+ test "multi paths with same path configured twice" do
106
+ c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
107
+ d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
108
+ assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
109
+ end
110
+
108
111
  test "multi paths with invaid path_delimiter" do
109
112
  c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
110
113
  assert_raise(Fluent::ConfigError) do
@@ -226,7 +229,7 @@ class TailInputTest < Test::Unit::TestCase
226
229
  d = create_driver(config)
227
230
  msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
228
231
 
229
- d.run(expect_emits: 1) do
232
+ d.run(expect_emits: num_events, timeout: 1) do
230
233
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
231
234
  f.puts msg
232
235
  f.puts msg
@@ -237,7 +240,7 @@ class TailInputTest < Test::Unit::TestCase
237
240
  assert_equal(true, events.length > 0)
238
241
  assert_equal({"message" => msg}, events[0][2])
239
242
  assert_equal({"message" => msg}, events[1][2])
240
- assert_equal(num_events, d.emit_count)
243
+ assert num_events <= d.emit_count
241
244
  end
242
245
 
243
246
  data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -1025,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
1025
1028
  assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
1026
1029
  end
1027
1030
 
1031
+ def test_expand_paths_with_duplicate_configuration
1032
+ expanded_paths = [
1033
+ 'test/plugin/data/log/foo/bar.log',
1034
+ 'test/plugin/data/log/test.log'
1035
+ ]
1036
+ duplicate_config = EX_CONFIG.dup
1037
+ duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1038
+ plugin = create_driver(EX_CONFIG, false).instance
1039
+ assert_equal expanded_paths, plugin.expand_paths.sort
1040
+ end
1041
+
1028
1042
  def test_expand_paths_with_timezone
1029
1043
  ['Asia/Taipei', '+08'].each do |tz_type|
1030
1044
  taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
@@ -1065,9 +1079,6 @@ class TailInputTest < Test::Unit::TestCase
1065
1079
  assert_equal expected_files, plugin.expand_paths.sort
1066
1080
  end
1067
1081
 
1068
- # For https://github.com/fluent/fluentd/issues/1455
1069
- # This test is fragile because test content depends on internal implementation.
1070
- # So if you modify in_tail internal, this test may break.
1071
1082
  def test_unwatched_files_should_be_removed
1072
1083
  config = config_element("", "", {
1073
1084
  "tag" => "tail",
@@ -1078,23 +1089,16 @@ class TailInputTest < Test::Unit::TestCase
1078
1089
  "refresh_interval" => 1,
1079
1090
  })
1080
1091
  d = create_driver(config, false)
1092
+ d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1081
1093
  d.run(expect_emits: 1, shutdown: false) do
1082
1094
  File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
1083
1095
  end
1084
1096
 
1085
- assert_equal 1, d.instance.instance_variable_get(:@tails).keys.size
1086
1097
  cleanup_directory(TMP_DIR)
1087
1098
  waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
1088
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
1099
+ waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1089
1100
 
1090
- # Previous implementation has an infinite watcher creation bug.
1091
- # Following code checks such unexpected bug by counting actual object allocation.
1092
- base_num = count_timer_object
1093
- 2.times {
1094
- sleep 1
1095
- num = count_timer_object
1096
- assert_equal base_num, num
1097
- }
1101
+ assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
1098
1102
 
1099
1103
  d.instance_shutdown
1100
1104
  end
@@ -1164,7 +1168,7 @@ class TailInputTest < Test::Unit::TestCase
1164
1168
 
1165
1169
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1166
1170
  EX_PATHS.each do |path|
1167
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, 1000, anything, anything, anything, anything, false).once
1171
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
1168
1172
  end
1169
1173
 
1170
1174
  plugin.refresh_watchers
@@ -1173,7 +1177,7 @@ class TailInputTest < Test::Unit::TestCase
1173
1177
  mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
1174
1178
 
1175
1179
  Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1176
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, 1000, anything, anything, anything, anything, false).once
1180
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
1177
1181
  plugin.refresh_watchers
1178
1182
 
1179
1183
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1183,6 +1187,25 @@ class TailInputTest < Test::Unit::TestCase
1183
1187
  end
1184
1188
  end
1185
1189
 
1190
+ sub_test_case "refresh of pos file" do
1191
+ test 'type of pos_file_compaction_interval is time' do
1192
+ tail = {
1193
+ "tag" => "tail",
1194
+ "path" => "#{TMP_DIR}/*.txt",
1195
+ "format" => "none",
1196
+ "pos_file" => "#{TMP_DIR}/pos/tail.pos",
1197
+ "refresh_interval" => 1,
1198
+ "read_from_head" => true,
1199
+ 'pos_file_compaction_interval' => '24h',
1200
+ }
1201
+ config = config_element("", "", tail)
1202
+ d = create_driver(config, false)
1203
+ mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
1204
+ mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
1205
+ d.run # call start
1206
+ end
1207
+ end
1208
+
1186
1209
  sub_test_case "receive_lines" do
1187
1210
  DummyWatcher = Struct.new("DummyWatcher", :tag)
1188
1211