fluentd 1.14.4-x86-mingw32 → 1.15.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +115 -19
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +2 -23
  10. data/Rakefile +1 -1
  11. data/fluentd.gemspec +3 -1
  12. data/lib/fluent/command/ctl.rb +4 -1
  13. data/lib/fluent/command/fluentd.rb +14 -0
  14. data/lib/fluent/config/error.rb +12 -0
  15. data/lib/fluent/config/literal_parser.rb +2 -2
  16. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  17. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  18. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  19. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  20. data/lib/fluent/config/yaml_parser.rb +56 -0
  21. data/lib/fluent/config.rb +14 -1
  22. data/lib/fluent/event_router.rb +19 -1
  23. data/lib/fluent/plugin/bare_output.rb +1 -1
  24. data/lib/fluent/plugin/base.rb +1 -1
  25. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  26. data/lib/fluent/plugin/in_forward.rb +1 -1
  27. data/lib/fluent/plugin/in_http.rb +11 -1
  28. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  29. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  30. data/lib/fluent/plugin/in_tail.rb +66 -47
  31. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  32. data/lib/fluent/plugin/output.rb +43 -33
  33. data/lib/fluent/plugin/parser.rb +3 -4
  34. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  35. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  36. data/lib/fluent/plugin_helper/server.rb +23 -4
  37. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  38. data/lib/fluent/plugin_helper/socket.rb +13 -2
  39. data/lib/fluent/registry.rb +2 -1
  40. data/lib/fluent/rpc.rb +4 -3
  41. data/lib/fluent/supervisor.rb +114 -27
  42. data/lib/fluent/system_config.rb +2 -1
  43. data/lib/fluent/version.rb +1 -1
  44. data/lib/fluent/winsvc.rb +2 -0
  45. data/test/command/test_ctl.rb +0 -1
  46. data/test/command/test_fluentd.rb +33 -0
  47. data/test/compat/test_parser.rb +1 -1
  48. data/test/config/test_system_config.rb +3 -1
  49. data/test/config/test_types.rb +1 -1
  50. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  51. data/test/plugin/in_tail/test_position_file.rb +0 -63
  52. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  53. data/test/plugin/test_file_wrapper.rb +0 -68
  54. data/test/plugin/test_filter_parser.rb +1 -1
  55. data/test/plugin/test_filter_stdout.rb +2 -2
  56. data/test/plugin/test_in_forward.rb +0 -2
  57. data/test/plugin/test_in_http.rb +23 -0
  58. data/test/plugin/test_in_object_space.rb +9 -3
  59. data/test/plugin/test_in_syslog.rb +1 -1
  60. data/test/plugin/test_in_tail.rb +629 -353
  61. data/test/plugin/test_out_forward.rb +30 -20
  62. data/test/plugin/test_out_stdout.rb +2 -2
  63. data/test/plugin/test_output_as_buffered_retries.rb +53 -6
  64. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  65. data/test/plugin/test_parser_syslog.rb +1 -1
  66. data/test/plugin_helper/test_cert_option.rb +1 -1
  67. data/test/plugin_helper/test_child_process.rb +16 -4
  68. data/test/plugin_helper/test_retry_state.rb +602 -38
  69. data/test/plugin_helper/test_server.rb +18 -0
  70. data/test/test_config.rb +135 -4
  71. data/test/test_event_router.rb +17 -0
  72. data/test/test_formatter.rb +1 -1
  73. data/test/test_supervisor.rb +196 -6
  74. metadata +39 -5
@@ -40,10 +40,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
40
40
 
41
41
  test '.load' do
42
42
  write_data(@file, TEST_CONTENT)
43
- paths = {
44
- "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
45
- "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
46
- }
47
43
  Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
48
44
 
49
45
  @file.seek(0)
@@ -146,11 +142,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
146
142
  sub_test_case '#load' do
147
143
  test 'compact invalid and convert 32 bit inode value' do
148
144
  write_data(@file, TEST_CONTENT)
149
- invalid_path = "invalidpath100000000000000000000000000000000"
150
- paths = TEST_CONTENT_PATHS.merge({
151
- invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
152
- "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
153
- })
154
145
  Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
155
146
 
156
147
  @file.seek(0)
@@ -322,58 +313,4 @@ class IntailPositionFileTest < Test::Unit::TestCase
322
313
  assert_equal 2, f.read_inode
323
314
  end
324
315
  end
325
-
326
- sub_test_case "TargetInfo equality rules" do
327
- sub_test_case "== operator" do
328
- def test_equal
329
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
330
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
331
-
332
- assert_equal t1, t2
333
- end
334
-
335
- def test_not_equal
336
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
337
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
338
-
339
- assert_not_equal t1, t2
340
- end
341
- end
342
-
343
- sub_test_case "eql? method" do
344
- def test_eql?
345
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
346
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
347
-
348
- assert do
349
- t1.eql? t2
350
- end
351
- end
352
-
353
- def test_not_eql?
354
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
355
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
356
-
357
- assert do
358
- !t1.eql? t2
359
- end
360
- end
361
- end
362
-
363
- sub_test_case "hash" do
364
- def test_equal
365
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
366
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
367
-
368
- assert_equal t1.hash, t2.hash
369
- end
370
-
371
- def test_not_equal
372
- t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
373
- t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
374
-
375
- assert_not_equal t1.hash, t2.hash
376
- end
377
- end
378
- end
379
316
  end
@@ -110,6 +110,10 @@ class SocketCacheTest < Test::Unit::TestCase
110
110
  end
111
111
 
112
112
  sub_test_case 'purge_obsolete_socks' do
113
+ def teardown
114
+ Timecop.return
115
+ end
116
+
113
117
  test 'delete key in inactive_socks' do
114
118
  c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
115
119
  sock = mock!.close { 'closed' }.subject
@@ -134,7 +138,7 @@ class SocketCacheTest < Test::Unit::TestCase
134
138
  c.checkin(sock)
135
139
 
136
140
  # wait timeout
137
- Timecop.freeze(Time.parse('2016-04-13 14:20:00 +0900'))
141
+ Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
138
142
  c.checkout_or('key') { sock2 }
139
143
 
140
144
  assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
@@ -145,5 +149,26 @@ class SocketCacheTest < Test::Unit::TestCase
145
149
  assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
146
150
  assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
147
151
  end
152
+
153
+ test 'should not purge just after checkin and purge after timeout' do
154
+ Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
155
+
156
+ c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
157
+ sock = dont_allow(mock!).close
158
+ stub(sock).inspect
159
+ c.checkout_or('key') { sock }
160
+
161
+ Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
162
+ c.checkin(sock)
163
+
164
+ assert_equal(1, c.instance_variable_get(:@available_sockets).size)
165
+ c.purge_obsolete_socks
166
+ assert_equal(1, c.instance_variable_get(:@available_sockets).size)
167
+
168
+ Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900'))
169
+ assert_equal(1, c.instance_variable_get(:@available_sockets).size)
170
+ c.purge_obsolete_socks
171
+ assert_equal(0, c.instance_variable_get(:@available_sockets).size)
172
+ end
148
173
  end
149
174
  end
@@ -17,58 +17,6 @@ class FileWrapperTest < Test::Unit::TestCase
17
17
  FileUtils.rm_rf(TMP_DIR)
18
18
  end
19
19
 
20
- sub_test_case 'Win32Error' do
21
- test 'equal' do
22
- assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
23
- Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
24
- end
25
-
26
- test 'different error code' do
27
- assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
28
- Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
29
- end
30
-
31
- test 'different error message' do
32
- assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
33
- Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
34
- end
35
-
36
- test 'different class' do
37
- assert_not_equal(Errno::EPIPE,
38
- Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
39
- end
40
-
41
- test 'ERROR_SHARING_VIOLATION message' do
42
- assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
43
- "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
44
- end
45
-
46
- test 'ERROR_SHARING_VIOLATION with a message' do
47
- assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
48
- "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
49
- " - cannot open the file")
50
- end
51
-
52
- test 'to_s' do
53
- assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
54
- Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
55
- end
56
-
57
- test 'inspect' do
58
- assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
59
- Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
60
- end
61
-
62
- data('0' => [false, 0],
63
- '9999' => [false, 9999],
64
- '10000' => [true, 10000],
65
- '10001' => [true, 10001])
66
- test 'wsaerr?' do |data|
67
- expected, code = data
68
- assert_equal(expected, Fluent::Win32Error.new(code).wsaerr?)
69
- end
70
- end
71
-
72
20
  sub_test_case 'WindowsFile exceptions' do
73
21
  test 'nothing raised' do
74
22
  begin
@@ -106,21 +54,5 @@ class FileWrapperTest < Test::Unit::TestCase
106
54
  file.close if file
107
55
  end
108
56
  end
109
-
110
- test 'ERROR_SHARING_VIOLATION raised' do
111
- begin
112
- path = "#{TMP_DIR}/test_windows_file.txt"
113
- file1 = file2 = nil
114
- file1 = File.open(path, "wb")
115
- win32err = Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)
116
- assert_raise(Errno::EACCES.new(win32err.message)) do
117
- file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
118
- ensure
119
- file2.close if file2
120
- end
121
- ensure
122
- file1.close if file1
123
- end
124
- end
125
57
  end
126
58
  end if Fluent.windows?
@@ -46,7 +46,7 @@ class ParserFilterTest < Test::Unit::TestCase
46
46
  assert_raise(Fluent::ConfigError) {
47
47
  create_driver('')
48
48
  }
49
- assert_raise(Fluent::ConfigError) {
49
+ assert_raise(Fluent::NotFoundPluginError) {
50
50
  create_driver %[
51
51
  key_name foo
52
52
  <parse>
@@ -63,7 +63,7 @@ class StdoutFilterTest < Test::Unit::TestCase
63
63
  end
64
64
 
65
65
  def test_invalid_output_type
66
- assert_raise(Fluent::ConfigError) do
66
+ assert_raise(Fluent::NotFoundPluginError) do
67
67
  d = create_driver(CONFIG + config_element("", "", { "output_type" => "foo" }))
68
68
  d.run {}
69
69
  end
@@ -139,7 +139,7 @@ class StdoutFilterTest < Test::Unit::TestCase
139
139
  def test_invalid_output_type
140
140
  conf = config_element
141
141
  conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "foo" })
142
- assert_raise(Fluent::ConfigError) do
142
+ assert_raise(Fluent::NotFoundPluginError) do
143
143
  d = create_driver(conf)
144
144
  d.run {}
145
145
  end
@@ -68,7 +68,6 @@ class ForwardInputTest < Test::Unit::TestCase
68
68
  @d = d = create_driver
69
69
  assert_equal @port, d.instance.port
70
70
  assert_equal '127.0.0.1', d.instance.bind
71
- assert_equal 0, d.instance.linger_timeout
72
71
  assert_equal 0.5, d.instance.blocking_timeout
73
72
  assert !d.instance.backlog
74
73
  end
@@ -77,7 +76,6 @@ class ForwardInputTest < Test::Unit::TestCase
77
76
  @d = d = create_driver(config_auth)
78
77
  assert_equal @port, d.instance.port
79
78
  assert_equal '127.0.0.1', d.instance.bind
80
- assert_equal 0, d.instance.linger_timeout
81
79
  assert !d.instance.backlog
82
80
 
83
81
  assert d.instance.security
@@ -540,6 +540,29 @@ class HttpInputTest < Test::Unit::TestCase
540
540
  assert_equal_event_time time, d.events[1][1]
541
541
  end
542
542
 
543
+ def test_application_ndjson
544
+ d = create_driver
545
+ events = [
546
+ ["tag1", 1643935663, "{\"a\":1}\n{\"b\":2}"],
547
+ ["tag2", 1643935664, "{\"a\":3}\r\n{\"b\":4}"]
548
+ ]
549
+
550
+ expected = [
551
+ ["tag1", 1643935663, {"a"=>1}],
552
+ ["tag1", 1643935663, {"b"=>2}],
553
+ ["tag2", 1643935664, {"a"=>3}],
554
+ ["tag2", 1643935664, {"b"=>4}]
555
+ ]
556
+
557
+ d.run(expect_records: 1) do
558
+ events.each do |tag, time, record|
559
+ res = post("/#{tag}?time=#{time}", record, {"Content-Type"=>"application/x-ndjson"})
560
+ assert_equal("200", res.code)
561
+ end
562
+ end
563
+ assert_equal(expected, d.events)
564
+ end
565
+
543
566
  def test_msgpack
544
567
  d = create_driver
545
568
  time = event_time("2011-01-02 13:14:15 UTC")
@@ -17,13 +17,19 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
17
17
  end
18
18
 
19
19
  class FailObject
20
- def self.class
21
- raise "error"
22
- end
23
20
  end
24
21
 
25
22
  def setup
26
23
  Fluent::Test.setup
24
+ # Overriding this behavior in the global scope will have an unexpected influence on other tests.
25
+ # So this should be overridden here and be removed in `teardown`.
26
+ def FailObject.class
27
+ raise "FailObject error for tests in ObjectSpaceInputTest."
28
+ end
29
+ end
30
+
31
+ def teardown
32
+ FailObject.singleton_class.remove_method(:class)
27
33
  end
28
34
 
29
35
  TESTCONFIG = %[
@@ -497,7 +497,7 @@ EOS
497
497
 
498
498
  def test_send_keepalive_packet_can_not_be_enabled_for_udp
499
499
  assert_raise(Fluent::ConfigError) do
500
- d = create_driver(ipv4_config + %[
500
+ create_driver(ipv4_config + %[
501
501
  send_keepalive_packet true
502
502
  ])
503
503
  end