fluentd 1.9.3 → 1.11.1

data/test/plugin/test_parser_syslog.rb

@@ -21,7 +21,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
     }
-    assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
@@ -35,6 +35,25 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
   end
 
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_time_format2(param)
+    @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'parser_type' => param)
+    @parser.instance.parse('2020-03-03T10:14:29Z 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
+      assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)
+      assert_equal(@expected, record)
+    }
+    assert_equal('%Y-%m-%dT%H:%M:%SZ', @parser.instance.patterns['time_format'])
+  end
+
+  def test_parse_with_time_format_rfc5424
+    @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'message_format' => 'rfc5424')
+    @parser.instance.parse('2020-03-03T10:14:29Z 192.168.0.1 fluentd 11111 - - [error] Syslog test') { |time, record|
+      assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)
+      assert_equal(@expected.merge('host' => '192.168.0.1', 'msgid' => '-', 'extradata' => '-'), record)
+    }
+    assert_equal('%Y-%m-%dT%H:%M:%SZ', @parser.instance.patterns['time_format'])
+  end
+
   data('regexp' => 'regexp', 'string' => 'string')
   def test_parse_with_subsecond_time(param)
     @parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)
@@ -51,10 +70,20 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected.merge('pri' => 6), record)
     }
-    assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_rfc5452_with_priority(param)
+    @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424')
+    @parser.instance.parse('<30>1 2020-03-31T20:32:54Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00') do |time, record|
+      assert_equal(event_time('2020-03-31T20:32:54Z', format: '%Y-%m-%dT%H:%M:%S%z'), time)
+      expected = { 'extradata' => '-', 'host' => 'myhostname', 'ident' => '02abaf0687f5', 'message' => 'method=POST db=0.00', 'msgid' => '02abaf0687f5', 'pid' => '10339', 'pri' => 30 }
+      assert_equal(expected, record)
+    end
+  end
+
   data('regexp' => 'regexp', 'string' => 'string')
   def test_parse_with_empty_priority(param)
     @parser.configure('with_priority' => true, 'parser_type' => param)
@@ -71,7 +100,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
     }
-    assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+    assert_equal(Fluent::Plugin::SyslogParser::RFC3164_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
@@ -170,11 +199,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
   end
 
   class TestRFC5424Regexp < self
-    def test_parse_with_rfc5424_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -184,15 +215,16 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
+      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_message_trailing_eol
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_trailing_eol(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!\n"
       @parser.instance.parse(text) do |time, record|
@@ -202,15 +234,16 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
+      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_multiline_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_multiline_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi,\nfrom\nFluentd!"
       @parser.instance.parse(text) do |time, record|
@@ -220,14 +253,15 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi,\nfrom\nFluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
+      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_message_and_without_priority
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_and_without_priority(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
+                        'parser_type' => param
                         )
       text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -237,14 +271,15 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
-                   @parser.instance.patterns['format'])
+      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_empty_message_and_without_priority
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_empty_message_and_without_priority(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
+                        'parser_type' => param
                         )
       text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
       @parser.instance.parse(text) do |time, record|
@@ -254,14 +289,15 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_nil record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
-                   @parser.instance.patterns['format'])
+      assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_message_without_time_format
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_without_time_format(param)
       @parser.configure(
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -273,10 +309,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_message_with_priority_and_pid
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_with_priority_and_pid(param)
       @parser.configure(
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - -  peer 192.168.0.5:50123 is not from a LAN'
       @parser.instance.parse(text) do |time, record|
@@ -288,11 +326,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_structured_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_structured_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -305,11 +345,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_multiple_structured_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_multiple_structured_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -322,11 +364,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_message_includes_right_bracket
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_includes_right_bracket(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd]!'
       @parser.instance.parse(text) do |time, record|
@@ -339,11 +383,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_empty_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_empty_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
       @parser.instance.parse(text) do |time, record|
@@ -356,10 +402,35 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_message_without_subseconds
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_space_empty_message(param)
+      @parser.configure(
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        'parser_type' => param
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] '
+      @parser.instance.parse(text) do |time, record|
+        if param == 'string'
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal '', record["message"]
+        else
+          assert_nil time
+          assert_nil record
+        end
+      end
+    end
+
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_without_subseconds(param)
       @parser.configure(
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -371,10 +442,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
 
-    def test_parse_with_rfc5424_message_both_timestamp
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message_both_timestamp(param)
       @parser.configure(
                         'message_format' => 'rfc5424',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -408,7 +481,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected, record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
     end
 
     data('regexp' => 'regexp', 'string' => 'string')
@@ -424,7 +496,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
     end
 
     data('regexp' => 'regexp', 'string' => 'string')
@@ -441,10 +512,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["pid"]
         assert_equal "-", record["msgid"]
         assert_equal "-", record["extradata"]
+        assert_equal 16, record["pri"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
     end
 
     data('regexp' => 'regexp', 'string' => 'string')
@@ -464,8 +534,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
     end
 
     data('regexp' => 'regexp', 'string' => 'string')
@@ -482,7 +550,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 1), record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
 
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -493,15 +560,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
+
+      text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test 2>1'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 1, 'message'=> '[error] Syslog test 2>1'), record)
+      end
 
       text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 1), record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
 
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -511,8 +581,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
     end
 
     data('regexp' => 'regexp', 'string' => 'string')
@@ -529,7 +597,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
 
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -540,15 +607,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
 
       text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 16), record)
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
 
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -558,8 +622,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
 
       text = '<16>1 2017-02-06T13:14:15Z 192.168.0.1 fluentd - - - Hi, from Fluentd without subseconds!'
       @parser.instance.parse(text) do |time, record|
@@ -569,8 +631,6 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd without subseconds!", record["message"]
       end
-      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
     end
   end
 end

data/test/plugin/test_sd_file.rb

@@ -149,6 +149,23 @@ class FileServiceDiscoveryTest < ::Test::Unit::TestCase
       assert_empty queue
     end
 
+    test 'Skip if error is occured' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))
+      queue = []
+
+      FileUtils.rm_r(File.join(@dir, 'tmp', 'config.json'))
+      mock.proxy(@sd_file).refresh_file(queue).twice
+
+      @sd_file.start(queue)
+      assert_empty queue
+
+      @sd_file.resume
+      assert_empty queue
+    end
+
     test 'if service is updated, service_in and service_out event happen' do
       @sd_file.extend(TestStatEventHelperWrapper)
 

data/test/plugin/test_sd_srv.rb

@@ -0,0 +1,230 @@
+require_relative '../helper'
+require 'fluent/plugin/sd_srv'
+require 'fileutils'
+require 'flexmock/test_unit'
+require 'json'
+
+class SrvServiceDiscoveryTest < ::Test::Unit::TestCase
+  SRV_RECORD1 = Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')
+  SRV_RECORD2 = Resolv::DNS::Resource::IN::SRV.new(2, 20, 8082, 'service2.example.com')
+
+  sub_test_case 'configure' do
+    test 'set services ordered by priority' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'reuturn host name without revolving name when dns_lookup is flase' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service1.example.com', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service2.example.com', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'pass a value as :nameserver to Resolve::DNS when dns_server_host is given' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new(nameserver: '8.8.8.8') { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'pass a value as :nameserver_port to Resolve::DNS when dns_server_host has port' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new(nameserver_port: [['8.8.8.8', 8080]]) { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8:8080' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'target follows RFC2782' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources).with("_service1._tcp.example.com", Resolv::DNS::Resource::IN::SRV)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'can change protocol' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources).with("_service1._udp.example.com", Resolv::DNS::Resource::IN::SRV)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'proto' => 'udp' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'can set password, username, password' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'shared_key' => 'key', 'username' => 'user', 'password' => 'pass' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, 'user', 'pass', 'key'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, 'user', 'pass', 'key'), sdf.services[1]
+    end
+  end
+
+  sub_test_case '#start' do
+    module TestTimerEventHelperWrapper
+      # easy to control statsevent
+      def timer_execute(_name, _interval, &block)
+        @test_timer_event_helper_wrapper_context = Fiber.new do
+          loop do
+            block.call
+
+            if Fiber.yield == :finish
+              break
+            end
+          end
+        end
+
+        resume
+      end
+
+      def resume
+        @test_timer_event_helper_wrapper_context.resume(:resume)
+      end
+
+      def shutdown
+        super
+
+        if @test_timer_event_helper_wrapper_context
+          @test_timer_event_helper_wrapper_context.resume(:finish)
+        end
+
+      end
+    end
+
+    setup do
+      sds = Fluent::Plugin::SrvServiceDiscovery.new
+      @sd_srv = sds
+    end
+
+    teardown do
+      if @sd_srv
+        @sd_srv.stop unless @sd_srv.stopped?
+        @sd_srv.before_shutdown unless @sd_srv.before_shutdown?
+        @sd_srv.shutdown unless @sd_srv.shutdown?
+        @sd_srv.after_shutdown unless @sd_srv.after_shutdown?
+        @sd_srv.close unless @sd_srv.closed?
+        @sd_srv.terminate unless @sd_srv.terminated?
+      end
+    end
+
+    test 'Skip if srv record is not updated' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      queue = []
+
+      @sd_srv.start(queue)
+      assert_empty queue
+
+      @sd_srv.resume
+      assert_empty queue
+    end
+
+    test 'Skip if DNS resolver raise an error' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .and_return { raise 'some error' } # for start
+               .and_return { raise 'some error' } # for resume
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      queue = []
+
+      @sd_srv.start(queue)
+      assert_empty queue
+
+      @sd_srv.resume
+      assert_empty queue
+    end
+
+    test 'if service is updated, service_in and service_out event happen' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1])
+               .and_return([SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+      join = queue.shift
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 8082, join.service.port
+      assert_equal 'service2.example.com', join.service.host
+
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 8081, drain.service.port
+      assert_equal 'service1.example.com', drain.service.host
+    end
+
+    test 'if service is deleted, service_out event happens' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .and_return([SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+
+      assert_equal 1, queue.size
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 8081, drain.service.port
+      assert_equal 'service1.example.com', drain.service.host
+    end
+
+    test 'if new service is added, service_in event happens' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD2])
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+
+      assert_equal 1, queue.size
+      join = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 8081, join.service.port
+      assert_equal 'service1.example.com', join.service.host
+    end
+  end
+end