fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +151 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/command/ctl.rb +2 -2
  15. data/lib/fluent/command/fluentd.rb +55 -53
  16. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  17. data/lib/fluent/config/dsl.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +2 -2
  19. data/lib/fluent/counter/server.rb +1 -1
  20. data/lib/fluent/counter/validator.rb +3 -3
  21. data/lib/fluent/daemon.rb +2 -4
  22. data/lib/fluent/engine.rb +1 -1
  23. data/lib/fluent/event.rb +8 -4
  24. data/lib/fluent/log/console_adapter.rb +66 -0
  25. data/lib/fluent/log.rb +44 -5
  26. data/lib/fluent/match.rb +1 -1
  27. data/lib/fluent/msgpack_factory.rb +6 -1
  28. data/lib/fluent/plugin/base.rb +6 -8
  29. data/lib/fluent/plugin/buf_file.rb +32 -3
  30. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  32. data/lib/fluent/plugin/buffer.rb +21 -0
  33. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  34. data/lib/fluent/plugin/in_forward.rb +1 -1
  35. data/lib/fluent/plugin/in_http.rb +8 -8
  36. data/lib/fluent/plugin/in_sample.rb +1 -1
  37. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  38. data/lib/fluent/plugin/in_tail.rb +58 -24
  39. data/lib/fluent/plugin/in_tcp.rb +47 -2
  40. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  41. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  42. data/lib/fluent/plugin/out_forward.rb +2 -2
  43. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  44. data/lib/fluent/plugin/output.rb +50 -13
  45. data/lib/fluent/plugin/parser_json.rb +1 -1
  46. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  47. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  48. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  49. data/lib/fluent/plugin_helper/server.rb +8 -0
  50. data/lib/fluent/plugin_helper/thread.rb +3 -3
  51. data/lib/fluent/plugin_id.rb +1 -1
  52. data/lib/fluent/supervisor.rb +157 -251
  53. data/lib/fluent/test/driver/base.rb +11 -5
  54. data/lib/fluent/test/driver/filter.rb +4 -0
  55. data/lib/fluent/test/startup_shutdown.rb +6 -8
  56. data/lib/fluent/version.rb +1 -1
  57. data/templates/new_gem/test/helper.rb.erb +0 -1
  58. data/test/command/test_ctl.rb +1 -1
  59. data/test/command/test_fluentd.rb +137 -6
  60. data/test/command/test_plugin_config_formatter.rb +0 -1
  61. data/test/compat/test_parser.rb +5 -5
  62. data/test/config/test_system_config.rb +0 -8
  63. data/test/log/test_console_adapter.rb +110 -0
  64. data/test/plugin/in_tail/test_position_file.rb +31 -1
  65. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  66. data/test/plugin/test_base.rb +99 -1
  67. data/test/plugin/test_buf_file.rb +62 -23
  68. data/test/plugin/test_buf_file_single.rb +65 -0
  69. data/test/plugin/test_buffer_chunk.rb +11 -0
  70. data/test/plugin/test_in_forward.rb +9 -9
  71. data/test/plugin/test_in_http.rb +2 -3
  72. data/test/plugin/test_in_monitor_agent.rb +2 -3
  73. data/test/plugin/test_in_tail.rb +379 -0
  74. data/test/plugin/test_in_tcp.rb +87 -2
  75. data/test/plugin/test_in_udp.rb +28 -0
  76. data/test/plugin/test_in_unix.rb +2 -2
  77. data/test/plugin/test_multi_output.rb +1 -1
  78. data/test/plugin/test_out_exec_filter.rb +2 -2
  79. data/test/plugin/test_out_file.rb +2 -2
  80. data/test/plugin/test_out_forward.rb +14 -18
  81. data/test/plugin/test_out_http.rb +1 -0
  82. data/test/plugin/test_output.rb +281 -12
  83. data/test/plugin/test_output_as_buffered.rb +44 -44
  84. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  85. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  86. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  87. data/test/plugin/test_parser_regexp.rb +1 -6
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  90. data/test/plugin_helper/test_server.rb +60 -6
  91. data/test/test_config.rb +0 -21
  92. data/test/test_formatter.rb +23 -20
  93. data/test/test_log.rb +108 -36
  94. data/test/test_msgpack_factory.rb +32 -0
  95. data/test/test_supervisor.rb +287 -279
  96. metadata +15 -21
  97. data/.drone.yml +0 -35
  98. data/.gitlab-ci.yml +0 -103
  99. data/test/test_logger_initializer.rb +0 -46
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
35
35
  @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
36
  end
37
37
  end
38
+
39
+ def self.dummy_event_stream
40
+ Fluent::ArrayEventStream.new(
41
+ [
42
+ [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
+ [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
+ [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
+ ]
46
+ )
47
+ end
38
48
  end
39
49
 
40
50
  class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
60
70
  end
61
71
  end
62
72
 
63
- def dummy_event_stream
64
- Fluent::ArrayEventStream.new(
65
- [
66
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
67
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
68
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
69
- ]
70
- )
71
- end
72
-
73
73
  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
74
 
75
75
  setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
89
89
  end
90
90
 
91
91
  data(
92
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
93
- handle_stream_with_standard_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
94
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- handle_stream_with_standard_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
92
+ :buffer_config,
93
+ [
94
+ config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
+ config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
+ config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
+ config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
+ ],
96
99
  )
97
- test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
100
+ data(
101
+ :input_es,
102
+ [
103
+ FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
+ # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
+ # https://github.com/fluent/fluentd/issues/4146
106
+ Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
+ ],
108
+ )
109
+ test 'call a standard format when output plugin adds data to chunk' do |data|
110
+ buffer_config = data[:buffer_config]
111
+ es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
+
98
113
  @i = create_output(:async)
99
114
  @i.configure(config_element('ROOT','', {}, [buffer_config]))
100
115
  @i.start
101
116
  @i.after_start
102
117
 
103
118
  io = StringIO.new
104
- es = dummy_event_stream
105
- expected = es.map { |e| e }
119
+ expected = es.dup.map { |t, r| [t, r] }
106
120
  compressed_data = ''
107
121
 
108
122
  assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
138
152
  @i.after_start
139
153
 
140
154
  io = StringIO.new
141
- es = dummy_event_stream
155
+ es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
142
156
  expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
143
157
  compressed_data = ''
144
158
 
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
93
93
  end
94
94
  def get_log_time(msg, logs)
95
95
  log_time = nil
96
- log = logs.select{|l| l.include?(msg) }.first
96
+ log = logs.find{|l| l.include?(msg) }
97
97
  if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
98
98
  log_time = Time.parse($1)
99
99
  end
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
634
634
 
635
635
  assert @i.retry
636
636
  logs = @i.log.out.logs
637
- waiting(4){ sleep 0.1 until logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
638
- assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
637
+ waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
638
+ assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
639
639
  end
640
640
 
641
641
  test 'retry_wait for secondary is same with one for primary' do
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
28
28
  if initialize_conf
29
29
  Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
30
  else
31
- # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
- instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
33
- instance.configure(conf)
34
- d = Struct.new(:instance).new
35
- d.instance = instance
36
- d
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
37
32
  end
38
33
  end
39
34
 
@@ -559,7 +559,7 @@ class ChildProcessTest < Test::Unit::TestCase
559
559
  unless Fluent.windows?
560
560
  test 'can specify subprocess name' do
561
561
  io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
562
- process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
562
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
563
563
  Process.kill(:TERM, io.pid) rescue nil
564
564
  io.close rescue nil
565
565
 
@@ -584,7 +584,7 @@ class ChildProcessTest < Test::Unit::TestCase
584
584
  m.lock
585
585
  pid = pids.first
586
586
  # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
587
- assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
587
+ assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
588
588
  @d.stop; @d.shutdown; @d.close; @d.terminate
589
589
  end
590
590
  end
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
127
127
  end
128
128
 
129
129
  client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
130
- reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
130
+ reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
131
131
 
132
132
  resp = nil
133
133
  error = nil
@@ -16,14 +16,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
16
16
 
17
17
  setup do
18
18
  @port = unused_port
19
- @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
20
- if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
21
- FileUtils.rm_f @socket_manager_path
19
+ if Fluent.windows?
20
+ @socket_manager_server = ServerEngine::SocketManager::Server.open
21
+ @socket_manager_path = @socket_manager_server.path
22
+ else
23
+ @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
24
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
25
+ FileUtils.rm_f @socket_manager_path
26
+ end
27
+ @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
22
28
  end
23
- @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
24
29
  ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
25
30
 
26
31
  @d = Dummy.new
32
+ @d.under_plugin_development = true
27
33
  @d.start
28
34
  @d.after_start
29
35
  end
@@ -37,7 +43,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
37
43
  (@d.terminated? || @d.terminate) rescue nil
38
44
 
39
45
  @socket_manager_server.close
40
- if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
46
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
41
47
  FileUtils.rm_f @socket_manager_path
42
48
  end
43
49
  end
@@ -789,6 +795,50 @@ class ServerPluginHelperTest < Test::Unit::TestCase
789
795
  end
790
796
  end
791
797
  end
798
+
799
+ sub_test_case 'over max_bytes' do
800
+ data("cut off on Non-Windows", { max_bytes: 32, records: ["a" * 40], expected: ["a" * 32] }, keep: true) unless Fluent.windows?
801
+ data("drop on Windows", { max_bytes: 32, records: ["a" * 40], expected: [] }, keep: true) if Fluent.windows?
802
+ test 'with sock' do |data|
803
+ max_bytes, records, expected = data.values
804
+
805
+ actual_records = []
806
+ @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data, sock|
807
+ actual_records << data
808
+ end
809
+
810
+ open_client(:udp, "127.0.0.1", @port) do |sock|
811
+ records.each do |record|
812
+ sock.send(record, 0)
813
+ end
814
+ end
815
+
816
+ waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
817
+ sleep 1 if expected.size == 0 # To confirm no record recieved.
818
+
819
+ assert_equal expected, actual_records
820
+ end
821
+
822
+ test 'without sock' do |data|
823
+ max_bytes, records, expected = data.values
824
+
825
+ actual_records = []
826
+ @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data|
827
+ actual_records << data
828
+ end
829
+
830
+ open_client(:udp, "127.0.0.1", @port) do |sock|
831
+ records.each do |record|
832
+ sock.send(record, 0)
833
+ end
834
+ end
835
+
836
+ waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
837
+ sleep 1 if expected.size == 0 # To confirm no record recieved.
838
+
839
+ assert_equal expected, actual_records
840
+ end
841
+ end
792
842
  end
793
843
 
794
844
  module CertUtil
@@ -1279,7 +1329,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
1279
1329
  # OpenSSL 1.1.1: "TLSv1.2"
1280
1330
  if tls_version == "TLSv1/SSLv3" || tls_version == "TLSv1.2"
1281
1331
  matched = true
1282
- unless cipher_name.match(/#{conf.ciphers}/)
1332
+ unless cipher_name.match?(/#{conf.ciphers}/)
1283
1333
  matched = false
1284
1334
  break
1285
1335
  end
@@ -1570,6 +1620,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
1570
1620
 
1571
1621
  def open_client(proto, addr, port)
1572
1622
  client = case proto
1623
+ when :udp
1624
+ c = UDPSocket.open
1625
+ c.connect(addr, port)
1626
+ c
1573
1627
  when :tcp
1574
1628
  TCPSocket.open(addr, port)
1575
1629
  when :tls
data/test/test_config.rb CHANGED
@@ -344,27 +344,6 @@ class ConfigTest < Test::Unit::TestCase
344
344
  File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
345
345
  end
346
346
 
347
- def test_inline
348
- prepare_config
349
- opts = {
350
- :config_path => "#{TMP_DIR}/config_test_1.conf",
351
- :inline_config => "<source>\n type http\n port 2222\n </source>",
352
- :use_v1_config => false
353
- }
354
- assert_nothing_raised do
355
- Fluent::Supervisor.new(opts)
356
- end
357
- create_warn_dummy_logger
358
- end
359
-
360
- def create_warn_dummy_logger
361
- dl_opts = {}
362
- dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
363
- logdev = Fluent::Test::DummyLogDevice.new
364
- logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
365
- $log = Fluent::Log.new(logger)
366
- end
367
-
368
347
  sub_test_case '.build' do
369
348
  test 'read config' do
370
349
  write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
@@ -18,7 +18,7 @@ module FormatterTest
18
18
 
19
19
  def test_call
20
20
  formatter = Formatter.new
21
- formatter.configure({})
21
+ formatter.configure(config_element())
22
22
  assert_raise NotImplementedError do
23
23
  formatter.format('tag', Engine.now, {})
24
24
  end
@@ -130,7 +130,7 @@ module FormatterTest
130
130
  include FormatterTest
131
131
 
132
132
  def setup
133
- @formatter = TextFormatter::MessagePackFormatter.new
133
+ @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)
134
134
  @time = Engine.now
135
135
  end
136
136
 
@@ -146,7 +146,7 @@ module FormatterTest
146
146
  include FormatterTest
147
147
 
148
148
  def setup
149
- @formatter = TextFormatter::LabeledTSVFormatter.new
149
+ @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)
150
150
  @time = Engine.now
151
151
  @newline = if Fluent.windows?
152
152
  "\r\n"
@@ -156,16 +156,16 @@ module FormatterTest
156
156
  end
157
157
 
158
158
  def test_config_params
159
- assert_equal "\t", @formatter.delimiter
160
- assert_equal ":", @formatter.label_delimiter
159
+ assert_equal "\t", @formatter.instance.delimiter
160
+ assert_equal ":", @formatter.instance.label_delimiter
161
161
 
162
162
  @formatter.configure(
163
163
  'delimiter' => ',',
164
164
  'label_delimiter' => '=',
165
165
  )
166
166
 
167
- assert_equal ",", @formatter.delimiter
168
- assert_equal "=", @formatter.label_delimiter
167
+ assert_equal ",", @formatter.instance.delimiter
168
+ assert_equal "=", @formatter.instance.label_delimiter
169
169
  end
170
170
 
171
171
  def test_format
@@ -220,14 +220,14 @@ module FormatterTest
220
220
  include FormatterTest
221
221
 
222
222
  def setup
223
- @formatter = TextFormatter::CsvFormatter.new
223
+ @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)
224
224
  @time = Engine.now
225
225
  end
226
226
 
227
227
  def test_config_params
228
- assert_equal ',', @formatter.delimiter
229
- assert_equal true, @formatter.force_quotes
230
- assert_nil @formatter.fields
228
+ assert_equal ',', @formatter.instance.delimiter
229
+ assert_equal true, @formatter.instance.force_quotes
230
+ assert_nil @formatter.instance.fields
231
231
  end
232
232
 
233
233
  data(
@@ -237,8 +237,8 @@ module FormatterTest
237
237
  def test_config_params_with_customized_delimiters(data)
238
238
  expected, target = data
239
239
  @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
240
- assert_equal expected, @formatter.delimiter
241
- assert_equal ['a', 'b', 'c'], @formatter.fields
240
+ assert_equal expected, @formatter.instance.delimiter
241
+ assert_equal ['a', 'b', 'c'], @formatter.instance.fields
242
242
  end
243
243
 
244
244
  def test_format
@@ -299,7 +299,7 @@ module FormatterTest
299
299
  'blank' => 'one,,two,three')
300
300
  def test_config_params_with_fields(data)
301
301
  @formatter.configure('fields' => data)
302
- assert_equal %w(one two three), @formatter.fields
302
+ assert_equal %w(one two three), @formatter.instance.fields
303
303
  end
304
304
  end
305
305
 
@@ -313,31 +313,34 @@ module FormatterTest
313
313
  end
314
314
  end
315
315
 
316
+ def create_driver(klass_or_str)
317
+ Fluent::Test::FormatterTestDriver.new(klass_or_str)
318
+ end
316
319
 
317
320
  def test_config_params
318
- formatter = TextFormatter::SingleValueFormatter.new
319
- assert_equal "message", formatter.message_key
321
+ formatter = create_driver(TextFormatter::SingleValueFormatter)
322
+ assert_equal "message", formatter.instance.message_key
320
323
 
321
324
  formatter.configure('message_key' => 'foobar')
322
- assert_equal "foobar", formatter.message_key
325
+ assert_equal "foobar", formatter.instance.message_key
323
326
  end
324
327
 
325
328
  def test_format
326
- formatter = Fluent::Plugin.new_formatter('single_value')
329
+ formatter = create_driver('single_value')
327
330
  formatter.configure({})
328
331
  formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
329
332
  assert_equal("awesome#{@newline}", formatted)
330
333
  end
331
334
 
332
335
  def test_format_without_newline
333
- formatter = Fluent::Plugin.new_formatter('single_value')
336
+ formatter = create_driver('single_value')
334
337
  formatter.configure('add_newline' => 'false')
335
338
  formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
336
339
  assert_equal("awesome", formatted)
337
340
  end
338
341
 
339
342
  def test_format_with_message_key
340
- formatter = TextFormatter::SingleValueFormatter.new
343
+ formatter = create_driver(TextFormatter::SingleValueFormatter)
341
344
  formatter.configure('message_key' => 'foobar')
342
345
  formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
343
346
 
data/test/test_log.rb CHANGED
@@ -5,6 +5,7 @@ require 'fluent/log'
5
5
  require 'timecop'
6
6
  require 'logger'
7
7
  require 'securerandom'
8
+ require 'pathname'
8
9
 
9
10
  class LogTest < Test::Unit::TestCase
10
11
  def tmp_dir
@@ -33,6 +34,14 @@ class LogTest < Test::Unit::TestCase
33
34
  end
34
35
  end
35
36
 
37
+ def test_per_process_path
38
+ path = Fluent::Log.per_process_path("C:/tmp/test.log", :supervisor, 0)
39
+ assert_equal(path, "C:/tmp/test-supervisor-0.log")
40
+
41
+ path = Fluent::Log.per_process_path("C:/tmp/test.log", :worker, 1)
42
+ assert_equal(path, "C:/tmp/test-1.log")
43
+ end
44
+
36
45
  sub_test_case "log level" do
37
46
  data(
38
47
  trace: [Fluent::Log::LEVEL_TRACE, 0],
@@ -463,6 +472,43 @@ class LogTest < Test::Unit::TestCase
463
472
  ]
464
473
  assert_equal(expected, log.out.logs)
465
474
  end
475
+
476
+ def test_reject_on_max_size
477
+ ignore_same_log_interval = 10
478
+
479
+ logger = Fluent::Log.new(
480
+ ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
481
+ ignore_same_log_interval: ignore_same_log_interval,
482
+ )
483
+
484
+ # Output unique log every second.
485
+ Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
486
+ logger.info "Test #{i}"
487
+ Timecop.freeze(@timestamp + i)
488
+ end
489
+ logger.info "Over max size!"
490
+
491
+ # The newest cache and the latest caches in `ignore_same_log_interval` should exist.
492
+ assert { Thread.current[:last_same_log].size == ignore_same_log_interval + 1 }
493
+ end
494
+
495
+ def test_clear_on_max_size
496
+ ignore_same_log_interval = 10
497
+
498
+ logger = Fluent::Log.new(
499
+ ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
500
+ ignore_same_log_interval: ignore_same_log_interval,
501
+ )
502
+
503
+ # Output unique log at the same time.
504
+ Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
505
+ logger.info "Test #{i}"
506
+ end
507
+ logger.info "Over max size!"
508
+
509
+ # Can't reject old logs, so all cache should be cleared and only the newest should exist.
510
+ assert { Thread.current[:last_same_log].size == 1 }
511
+ end
466
512
  end
467
513
 
468
514
  def test_dup
@@ -593,45 +639,71 @@ class LogTest < Test::Unit::TestCase
593
639
 
594
640
  def test_log_rotates_specified_size_with_logdevio
595
641
  with_timezone('utc') do
596
- rotate_age = 2
597
- rotate_size = 100
598
- path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
599
- path0 = path + '.0'
600
- path1 = path + '.1'
642
+ begin
643
+ rotate_age = 2
644
+ rotate_size = 100
645
+ path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
646
+ path0 = path + '.0'
647
+ path1 = path + '.1'
648
+
649
+ logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
650
+ logger = ServerEngine::DaemonLogger.new(logdev)
651
+ log = Fluent::Log.new(logger)
652
+
653
+ msg = 'a' * 101
654
+ log.info msg
655
+ assert_match msg, File.read(path)
656
+ assert_true File.exist?(path)
657
+ assert_true !File.exist?(path0)
658
+ assert_true !File.exist?(path1)
659
+
660
+ # create log.0
661
+ msg2 = 'b' * 101
662
+ log.info msg2
663
+ c = File.read(path)
664
+ c0 = File.read(path0)
665
+ assert_match msg2, c
666
+ assert_match msg, c0
667
+ assert_true File.exist?(path)
668
+ assert_true File.exist?(path0)
669
+ assert_true !File.exist?(path1)
670
+
671
+ # rotate
672
+ msg3 = 'c' * 101
673
+ log.info msg3
674
+ c = File.read(path)
675
+ c0 = File.read(path0)
676
+ assert_match msg3, c
677
+ assert_match msg2, c0
678
+ assert_true File.exist?(path)
679
+ assert_true File.exist?(path0)
680
+ assert_true !File.exist?(path1)
681
+ ensure
682
+ logdev&.close
683
+ end
684
+ end
685
+ end
601
686
 
602
- logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
603
- logger = ServerEngine::DaemonLogger.new(logdev)
604
- log = Fluent::Log.new(logger)
687
+ def test_reopen
688
+ path = Pathname(@tmp_dir) + "fluent.log"
605
689
 
606
- msg = 'a' * 101
607
- log.info msg
608
- assert_match msg, File.read(path)
609
- assert_true File.exist?(path)
610
- assert_true !File.exist?(path0)
611
- assert_true !File.exist?(path1)
690
+ logdev = Fluent::LogDeviceIO.new(path.to_s)
691
+ logger = ServerEngine::DaemonLogger.new(logdev)
692
+ log = Fluent::Log.new(logger, path: path)
612
693
 
613
- # create log.0
614
- msg2 = 'b' * 101
615
- log.info msg2
616
- c = File.read(path)
617
- c0 = File.read(path0)
618
- assert_match msg2, c
619
- assert_match msg, c0
620
- assert_true File.exist?(path)
621
- assert_true File.exist?(path0)
622
- assert_true !File.exist?(path1)
623
-
624
- # rotate
625
- msg3 = 'c' * 101
626
- log.info msg3
627
- c = File.read(path)
628
- c0 = File.read(path0)
629
- assert_match msg3, c
630
- assert_match msg2, c0
631
- assert_true File.exist?(path)
632
- assert_true File.exist?(path0)
633
- assert_true !File.exist?(path1)
634
- end
694
+ message = "This is test message."
695
+
696
+ log.info message
697
+ log.reopen!
698
+ log.info message
699
+
700
+ assert { path.read.lines.count{ |line| line.include?(message) } == 2 }
701
+ # Assert reopening the same file.
702
+ # Especially, on Windows, the filepath is fixed for each process with rotate,
703
+ # so we need to care about this.
704
+ assert { path.parent.entries.size == 3 } # [".", "..", "fluent.log"]
705
+ ensure
706
+ logdev&.close
635
707
  end
636
708
  end
637
709
 
@@ -15,4 +15,36 @@ class MessagePackFactoryTest < Test::Unit::TestCase
15
15
  assert mp.msgpack_factory
16
16
  assert mp.msgpack_factory
17
17
  end
18
+
19
+ sub_test_case 'thread_local_msgpack_packer' do
20
+ test 'packer is cached' do
21
+ packer1 = Fluent::MessagePackFactory.thread_local_msgpack_packer
22
+ packer2 = Fluent::MessagePackFactory.thread_local_msgpack_packer
23
+ assert_equal packer1, packer2
24
+ end
25
+ end
26
+
27
+ sub_test_case 'thread_local_msgpack_unpacker' do
28
+ test 'unpacker is cached' do
29
+ unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
30
+ unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
31
+ assert_equal unpacker1, unpacker2
32
+ end
33
+
34
+ # We need to reset the buffer every time so that received incomplete data
35
+ # must not affect data from other senders.
36
+ test 'reset the internal buffer of unpacker every time' do
37
+ unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
38
+ unpacker1.feed_each("\xA6foo") do |result|
39
+ flunk("This callback must not be called since the data is uncomplete.")
40
+ end
41
+
42
+ records = []
43
+ unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
44
+ unpacker2.feed_each("\xA3foo") do |result|
45
+ records.append(result)
46
+ end
47
+ assert_equal ["foo"], records
48
+ end
49
+ end
18
50
  end