fluentd 0.14.17 → 0.14.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e746832a9687d6e680231e7f617584b50a6db91b
-  data.tar.gz: a084bf0df70ef3fe3c3bf21db2f21b6745640360
+  metadata.gz: 4c1133959101a3b085e0bcc08fbf62a87d2768a3
+  data.tar.gz: 4f8d96e61d587a5c89d2f3c73a6e1d0ba0e4fecc
 SHA512:
-  metadata.gz: 8c0bd7a4c82757b3f4507c7d79e171e414ccb4d44f99ce60a4ad1a0802bcc9cf18b9de0ef8d9d7cc755e61d5a5112b944f2102f68d3e71cc403f917034747aaa
-  data.tar.gz: 4e00a8d1147b4371ddc2a890543eef8efa2bdbec53fefc1cb39076ab8cd68e8b6839e9bfd3b1e5c7ef90856c8b77f7cd60205d83596a388b413936b5760be1e2
+  metadata.gz: 1985d37f5b9b9dffd4b40af81112768653a71afe1249be0d36c4a57f2ef768d1f9e3e5007ffe3df85246bc745bf6a8bf1e4ba2d161fa391c0fa87da8b24a46f7
+  data.tar.gz: fe02e6749d04169de7b79eb2c5c6dadbe1f223acbc72d3b2415e1ad2a48b6a9b9d6bb0ea36b06d41b623c7486149fdc4e5df6f0a125f6b0cf936c53cd69439ef

data/ChangeLog

@@ -1,5 +1,23 @@
 # v0.14
 
+## Release v0.14.18 - 2017/06/21
+
+### New features / Enhancements
+
+* parser: Add rfc5424 regex without priority
+  https://github.com/fluent/fluentd/pull/1600
+
+### Bug fixes
+
+* in_tail: Fix timing issue that the excluded_path doesn't apply.
+  https://github.com/fluent/fluentd/pull/1597
+* config: Fix broken UTF-8 encoded configuration file handling
+  https://github.com/fluent/fluentd/pull/1592
+* out_forward: Don't stop heartbeat when error happen
+  https://github.com/fluent/fluentd/pull/1602
+* Fix command name typo in plugin template
+  https://github.com/fluent/fluentd/pull/1603
+
 ## Release v0.14.17 - 2017/05/29
 
 ### New features / Enhancements

data/README.md

@@ -51,6 +51,11 @@ You can run specified test via `TEST` environment variable:
     $ bundle exec rake test TEST=test/test_specified_path.rb
     $ bundle exec rake test TEST=test/test_*.rb
 
+## Running in Production
+
+Many enterprises run Fluentd in production to handle all of their logging needs. For enterprises requiring Security tested binaries, SLA-based support, architectural guidance, and enhanced plugin connectors see [Fluentd Enterprise](https://www.fluentd.org/enterprise).
+
+
 ## Fluentd UI: Admin GUI
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
@@ -68,7 +73,6 @@ You can run specified test via `TEST` environment variable:
 - Author: Sadayuki Furuhashi
 - Copyright: (c) 2011 FURUHASHI Sadayuki
 - License: Apache License, Version 2.0
-
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/lib/fluent/plugin/in_tail.rb

@@ -187,7 +187,6 @@ module Fluent::Plugin
       date = Time.now
       paths = []
 
-      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
@@ -214,6 +213,7 @@ module Fluent::Plugin
           paths << path
         end
       }
+      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       paths - excluded
     end
 

data/lib/fluent/plugin/out_forward.rb

@@ -402,8 +402,10 @@ module Fluent::Plugin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
           n.send_heartbeat
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
-          log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: $!
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+          log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
+        rescue => e
+          log.debug "unexpected error happen during heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
         end
       }
     end

data/lib/fluent/plugin/parser_syslog.rb

@@ -27,7 +27,8 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
@@ -55,7 +56,7 @@ module Fluent
                       alias_method :parse, :parse_plain
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
-                    REGEXP_RFC5424
+                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
@@ -77,7 +78,7 @@ module Fluent
 
       def parse_auto(text, &block)
         if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = REGEXP_RFC5424
+          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
           @time_parser = @time_parser_rfc5424
         else
           @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP

data/lib/fluent/supervisor.rb

@@ -218,7 +218,8 @@ module Fluent
 
       config_fname = File.basename(path)
       config_basedir = File.dirname(path)
-      config_data = File.read(path)
+      # Assume fluent.conf encoding is UTF-8
+      config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
       inline_config = params['inline_config']
       if inline_config == '-'
         config_data << "\n" << STDIN.read
@@ -717,7 +718,7 @@ module Fluent
       $log.info :supervisor, "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
-      @config_data = File.read(@config_path)
+      @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
       if @inline_config == '-'
         @config_data << "\n" << STDIN.read
       elsif @inline_config

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.17'
+  VERSION = '0.14.18'
 
 end

data/templates/new_gem/README.md.erb

@@ -31,7 +31,7 @@ $ bundle
 You can generate configuration template:
 
 ```
-$ fluent-plugin-format-config <%= type %> <%= name %>
+$ fluent-plugin-config-format <%= type %> <%= name %>
 ```
 
 You can copy and paste generated documents here.

data/test/plugin/test_parser_syslog.rb

@@ -69,6 +69,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
+                        'with_priority' => true,
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -78,11 +79,31 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+    end
+
+    def test_parse_with_rfc5424_message_and_without_priority
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
+                   @parser.instance.patterns['format'])
     end
 
     def test_parse_with_rfc5424_message_without_time_format
       @parser.configure(
                         'message_format' => 'rfc5424',
+                        'with_priority' => true,
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -98,6 +119,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'rfc5424',
+                        'with_priority' => true,
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -122,6 +144,7 @@ assert_equal "11111", record["pid"]
         assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected, record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
     end
 
     def test_auto_with_legacy_syslog_priority_message
@@ -135,12 +158,14 @@ assert_equal "11111", record["pid"]
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
     end
 
     def test_parse_with_rfc5424_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'auto',
+                        'with_priority' => true,
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -150,12 +175,15 @@ assert_equal "11111", record["pid"]
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
     end
 
     def test_parse_with_rfc5424_structured_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'auto',
+                        'with_priority' => true,
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -166,6 +194,8 @@ assert_equal "11111", record["pid"]
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
     end
 
     def test_parse_with_both_message_type
@@ -173,12 +203,15 @@ assert_equal "11111", record["pid"]
         'time_format' => '%b %d %M:%S:%H',
         'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
         'message_format' => 'auto',
+        'with_priority' => true,
       )
-      text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
-        assert_equal(@expected, record)
+        assert_equal(@expected.merge('pri' => 1), record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -188,11 +221,16 @@ assert_equal "11111", record["pid"]
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
-      text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+
+      text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
-        assert_equal(@expected, record)
+        assert_equal(@expected.merge('pri' => 1), record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -201,6 +239,8 @@ assert_equal "11111", record["pid"]
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
     end
 
     def test_parse_with_both_message_type_and_priority
@@ -215,6 +255,8 @@ assert_equal "11111", record["pid"]
         assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -224,11 +266,16 @@ assert_equal "11111", record["pid"]
                      record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+
       text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
         assert_equal(@expected.merge('pri' => 16), record)
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -237,6 +284,8 @@ assert_equal "11111", record["pid"]
         assert_equal "-", record["extradata"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
     end
   end
 end

data/test/test_supervisor.rb

@@ -77,6 +77,46 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset
   end
 
+  def test_read_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+
+    use_v1_config = {}
+    use_v1_config['use_v1_config'] = true
+
+    sv.instance_variable_set(:@config_path, tmp_path)
+    sv.instance_variable_set(:@use_v1_config, use_v1_config)
+    sv.send(:read_config)
+
+    conf = sv.instance_variable_get(:@conf)
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_system_config
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
@@ -319,6 +359,45 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
   end
 
+  def test_load_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    params = {}
+    params['workers'] = 1
+    params['use_v1_config'] = true
+    params['log_path'] = 'test/tmp/supervisor/log'
+    params['suppress_repeated_stacktrace'] = true
+    params['log_level'] = Fluent::Log::LEVEL_INFO
+    load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_path, params) }
+
+    se_config = load_config_proc.call
+    conf = se_config[:fluentd_conf]
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_logger
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.17
+  version: 0.14.18
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack