fluentd 0.12.36 → 0.12.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f13bb194b66ee28b2bd8cf431379f73d47dcda0f
-  data.tar.gz: ca8542ad12a51d6d11f70de34caca3e5a2c559b2
+  metadata.gz: f88c1cef594f4ca5773a92310fd32f0fbf728a89
+  data.tar.gz: e9fc666c618c2b9bf35d10c21466a0539bb45d10
 SHA512:
-  metadata.gz: fa2d7ec85fa220ae426509a3f97b1438521b8451b82bfed553097fd40fc7805e0f9e6e4f396c08d4b5e5ea7b4308809be43f060f1744e27811dc56f31e8b0c8f
-  data.tar.gz: 99d942d58b3e04e6f172849ae60c4c23bb2abe259bdbcdd6297803fb005ba57905caf5ceaef320c7a051686b0bec7bb47081f0fbac20c13041eff903ec18af44
+  metadata.gz: 5652613563369d7f73bd290c5e550a9fd02bff462502bf9074a399a4fa2df89e3da69a4f63e34480751d3411938f57346aa1d1e049a26fb921b40609c7d7d311
+  data.tar.gz: 641ea08ba43f5f646a6444f91939c7f99fc95d5d59a562f59123e64f2d79bf120adb17e32e3b9dea3daa51bc432619db688567c74319b131743ad6c1b475606b

data/ChangeLog

@@ -1,5 +1,19 @@
 # v0.12
 
+## Release 0.12.37 - 2017/06/21
+
+### New features / Enhancement
+
+* parser: Add rfc5424 regex without priority
+  https://github.com/fluent/fluentd/pull/1601
+
+### Bug fixes
+
+* in_tail: fix timing issue that the excluded_path doesn't apply.
+  https://github.com/fluent/fluentd/pull/1597
+* in_tail: Detach watchers before updating watcher to avoid broken pos file
+  https://github.com/fluent/fluentd/pull/1598
+
 ## Release 0.12.36 - 2017/05/24
 
 ### New features / Enhancement

data/lib/fluent/parser.rb

@@ -540,7 +540,8 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+       REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_param :time_format, :string, default: "%b %d %H:%M:%S"
@@ -568,7 +569,7 @@ module Fluent
                       alias_method :parse, :parse_plain
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
-                    REGEXP_RFC5424
+                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
@@ -590,7 +591,7 @@ module Fluent
 
       def parse_auto(text, &block)
         if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = REGEXP_RFC5424
+          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
           @time_parser = @time_parser_rfc5424
         else
           @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP

data/lib/fluent/plugin/in_tail.rb

@@ -155,7 +155,6 @@ module Fluent
       date = Time.now
       paths = []
 
-      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
@@ -182,6 +181,7 @@ module Fluent
           paths << path
         end
       }
+      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       paths - excluded
     end
 
@@ -508,11 +508,13 @@ module Fluent
               io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
               @io_handler = io_handler
             else # file is rotated and new file found
+              detach
               @update_watcher.call(@path, swap_state(@pe))
             end
           else # file is rotated and new file not found
             # Clear RotateHandler to avoid duplicated file watch in same path.
             @rotate_handler = nil
+            detach
             @update_watcher.call(@path, swap_state(@pe))
           end
         end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.36'
+  VERSION = '0.12.37'
 
 end

data/test/test_parser.rb

@@ -379,6 +379,7 @@ module ParserTest
         @parser.configure(
           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
           'message_format' => 'rfc5424',
+          'with_priority' => true,
         )
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
@@ -388,11 +389,32 @@ module ParserTest
           assert_equal "-", record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
+      end
+
+      def test_parse_with_rfc5424_message_and_without_priority
+        @parser.configure(
+           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+           'message_format' => 'rfc5424',
+        )
+        text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.instance.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424,
+                     @parser.instance.patterns['format'])
       end
 
       def test_parse_with_rfc5424_message_without_time_format
         @parser.configure(
           'message_format' => 'rfc5424',
+          'with_priority' => true,
         )
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
         @parser.instance.parse(text) do |time, record|
@@ -408,6 +430,7 @@ module ParserTest
         @parser.configure(
           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
           'message_format' => 'rfc5424',
+          'with_priority' => true,
         )
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
@@ -445,12 +468,15 @@ module ParserTest
           assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
           assert_equal(@expected.merge('pri' => 6), record)
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
+                     @parser.instance.patterns['format'])
       end
 
       def test_parse_with_rfc5424_message
         @parser.configure(
           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
           'message_format' => 'auto',
+          'with_priority' => true,
         )
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
@@ -460,12 +486,15 @@ module ParserTest
           assert_equal "-", record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
       end
 
       def test_parse_with_rfc5424_structured_message
         @parser.configure(
           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
           'message_format' => 'auto',
+          'with_priority' => true,
         )
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
@@ -476,6 +505,8 @@ module ParserTest
                        record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
       end
 
       def test_parse_with_both_message_type
@@ -483,12 +514,16 @@ module ParserTest
           'time_format' => '%b %d %M:%S:%H',
           'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
           'message_format' => 'auto',
+          'with_priority' => true,
         )
-        text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected, record)
+          assert_equal(@expected.merge('pri' => 1), record)
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -498,11 +533,17 @@ module ParserTest
                        record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
-        text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+
+        text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected, record)
+          assert_equal(@expected.merge('pri' => 1), record)
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
+                   @parser.instance.patterns['format'])
+
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -511,6 +552,9 @@ module ParserTest
           assert_equal "-", record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
       end
 
       def test_parse_with_both_message_type_and_priority
@@ -525,6 +569,9 @@ module ParserTest
           assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
           assert_equal(@expected.merge('pri' => 6), record)
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -534,11 +581,17 @@ module ParserTest
                        record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
         text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
           assert_equal(@expected.merge('pri' => 16), record)
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
         text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
         @parser.parse(text) do |time, record|
           assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -547,6 +600,9 @@ module ParserTest
           assert_equal "-", record["extradata"]
           assert_equal "Hi, from Fluentd!", record["message"]
         end
+        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                     @parser.instance.patterns['format'])
+
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.36
+  version: 0.12.37
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-24 00:00:00.000000000 Z
+date: 2017-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack