fluentd 0.14.18 → 0.14.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c1133959101a3b085e0bcc08fbf62a87d2768a3
-  data.tar.gz: 4f8d96e61d587a5c89d2f3c73a6e1d0ba0e4fecc
+  metadata.gz: 0e8cde86e1ab8e515cd2573c413a821fd7fec5ac
+  data.tar.gz: f182030e17d2e1f72c63dffd08dd854e84544923
 SHA512:
-  metadata.gz: 1985d37f5b9b9dffd4b40af81112768653a71afe1249be0d36c4a57f2ef768d1f9e3e5007ffe3df85246bc745bf6a8bf1e4ba2d161fa391c0fa87da8b24a46f7
-  data.tar.gz: fe02e6749d04169de7b79eb2c5c6dadbe1f223acbc72d3b2415e1ad2a48b6a9b9d6bb0ea36b06d41b623c7486149fdc4e5df6f0a125f6b0cf936c53cd69439ef
+  metadata.gz: 490ef9c95652e18a65713eefdb0003876991c9a0d504b26edd73904cc24d7e9e7048205e056073d6637d10e75e2226ed5e1deb302596916a59dd4a82555f88e3
+  data.tar.gz: dfc41329fc70c653505d251fc9c66f51c4547693723c216c1d01dafa5c8ee38ecca561a3e4cf7aeb11b24879c0c35199265abcf02f659e366029cfcafdb7b96b

data/ChangeLog

@@ -1,5 +1,23 @@
 # v0.14
 
+## Release v0.14.19 - 2017/07/12
+
+### New features / Enhancements
+
+* in_syslog: More characters are available in tag part of syslog format
+  https://github.com/fluent/fluentd/pull/1610
+* in_syslog: Add resolve_hostname parameter
+  https://github.com/fluent/fluentd/pull/1616
+* filter_grep: Support new configuration format by config_section
+  https://github.com/fluent/fluentd/pull/1611
+
+### Bug fixes
+
+* output: Fix race condition of retry state in flush thread
+  https://github.com/fluent/fluentd/pull/1623
+* test: Fix typo in test_in_tail.rb
+  https://github.com/fluent/fluentd/pull/1622
+
 ## Release v0.14.18 - 2017/06/21
 
 ### New features / Enhancements

data/README.md

@@ -64,15 +64,16 @@ Many enterprises run Fluentd in production to handle all of their logging needs.
 
 ## More Information
 
-- Website: http://fluentd.org/
+- Website: https://www.fluentd.org/
 - Documentation: http://docs.fluentd.org/
-- Source repository: http://github.com/fluent
+- Project repository: http://github.com/fluent
 - Discussion: http://groups.google.com/group/fluentd
 - Slack / Community: http://slack.fluentd.org
 - Newsletters: http://get.treasuredata.com/Fluentd_education
 - Author: Sadayuki Furuhashi
 - Copyright: (c) 2011 FURUHASHI Sadayuki
 - License: Apache License, Version 2.0
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/lib/fluent/plugin/filter_grep.rb

@@ -24,32 +24,59 @@ module Fluent::Plugin
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
+
+    config_section :regexp, param_name: :regexps, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
+
+    config_section :exclude, param_name: :excludes, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
 
     # for test
-    attr_reader :regexps
-    attr_reader :excludes
+    attr_reader :_regexps
+    attr_reader :_excludes
 
     def configure(conf)
       super
 
-      @regexps = {}
+      @_regexps = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
-        @regexps[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
+        @_regexps[key] = Regexp.compile(regexp)
       end
 
-      @excludes = {}
+      @_excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
-        @excludes[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
+        @_excludes[key] = Regexp.compile(exclude)
+      end
+
+      @regexps.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
+        @_regexps[e.key] = e.pattern
+      end
+      @excludes.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
+        @_excludes[e.key] = e.pattern
       end
     end
 
@@ -57,10 +84,10 @@ module Fluent::Plugin
       result = nil
       begin
         catch(:break_loop) do
-          @regexps.each do |key, regexp|
+          @_regexps.each do |key, regexp|
             throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
           end
-          @excludes.each do |key, exclude|
+          @_excludes.each do |key, exclude|
             throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
           end
           result = record

data/lib/fluent/plugin/in_syslog.rb

@@ -83,6 +83,8 @@ module Fluent::Plugin
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
+    config_param :resolve_hostname, :bool, default: nil
+    desc 'Connections will be disconnected right after receiving first message if this value is true.'
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
     desc 'The field name of the priority.'
@@ -116,7 +118,13 @@ module Fluent::Plugin
         end
         @source_address_key = @source_host_key
       end
-      @resolve_name = !!@source_hostname_key
+      if @source_hostname_key
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in config
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
 
       @_event_loop_run_timeout = @blocking_timeout
     end
@@ -138,7 +146,7 @@ module Fluent::Plugin
     end
 
     def start_udp_server
-      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_name) do |data, sock|
+      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_hostname) do |data, sock|
         message_handler(data.chomp, sock)
       end
     end
@@ -147,7 +155,7 @@ module Fluent::Plugin
       # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
       delimiter = "\n"
       delimiter_size = delimiter.size
-      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_name) do |conn|
+      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
         buffer = ""
         conn.data do |data|
           buffer << data

data/lib/fluent/plugin/output.rb

@@ -1264,7 +1264,7 @@ module Fluent
             current_clock = Fluent::Clock.now
             interval = state.next_clock - current_clock
 
-            if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
+            if state.next_clock <= current_clock && @retry_mutex.synchronize { @retry ? @retry.next_time <= Time.now : true }
               try_flush
 
               # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)

data/lib/fluent/plugin/parser_syslog.rb

@@ -24,9 +24,9 @@ module Fluent
       Plugin.register_parser('syslog', self)
 
       # From existence TextParser pattern
-      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
-      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.18'
+  VERSION = '0.14.19'
 
 end

data/test/plugin/test_filter_grep.rb

@@ -23,12 +23,51 @@ class GrepFilterTest < Test::Unit::TestCase
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._regexps['message'])
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._excludes['message'])
+    end
+
+    sub_test_case "duplicate key" do
+      test "flat" do
+        conf = %[
+          regexp1 message test
+          regexp2 message test2
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "section" do
+        conf = %[
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+          <regexp>
+            key message
+            pattern test2
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "mix" do
+        conf = %[
+          regexp1 message test
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
     end
   end
 
@@ -77,6 +116,38 @@ class GrepFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'regexps' do
+      conf = %[
+        <regexp>
+          key message
+          pattern WARN
+        </regexp>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(3, filtered_records.size)
+      assert_block('only WARN logs') do
+        filtered_records.all? { |r|
+          !r['message'].include?('INFO')
+        }
+      end
+    end
+
+    test 'excludes' do
+      conf = %[
+        <exclude>
+          key message
+          pattern favicon
+        </exclude>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(3, filtered_records.size)
+      assert_block('remove favicon logs') do
+        filtered_records.all? { |r|
+          !r['message'].include?('favicon')
+        }
+      end
+    end
+
     sub_test_case 'with invalid sequence' do
       def messages
         [

data/test/plugin/test_in_syslog.rb

@@ -37,6 +37,24 @@ class SyslogInputTest < Test::Unit::TestCase
     assert_equal bind_addr, d.instance.bind
   end
 
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    test 'resolve_hostname must be true with source_hostname_key' do
+      assert_raise(Fluent::ConfigError) {
+        create_driver(CONFIG + <<EOS)
+resolve_hostname false
+source_hostname_key hostname
+EOS
+      }
+    end
+
+    data('resolve_hostname' => 'resolve_hostname true',
+         'source_hostname_key' => 'source_hostname_key source_host')
+    def test_configure_reslove_hostname(param)
+      d = create_driver([CONFIG, param].join("\n"))
+      assert_true d.instance.resolve_hostname
+    end
+  end
+
   data(
     ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
     ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],

data/test/plugin/test_in_tail.rb

@@ -823,7 +823,7 @@ class TailInputTest < Test::Unit::TestCase
   sub_test_case "path" do
     # * path test
     # TODO: Clean up tests
-    EX_RORATE_WAIT = 0
+    EX_ROTATE_WAIT = 0
 
     EX_CONFIG = config_element("", "", {
                                  "tag" => "tail",
@@ -832,7 +832,7 @@ class TailInputTest < Test::Unit::TestCase
                                  "pos_file" => "#{TMP_DIR}/tail.pos",
                                  "read_from_head" => true,
                                  "refresh_interval" => 30,
-                                 "rotate_wait" => "#{EX_RORATE_WAIT}s",
+                                 "rotate_wait" => "#{EX_ROTATE_WAIT}s",
                                })
     EX_PATHS = [
       'test/plugin/data/2010/01/20100102-030405.log',
@@ -926,7 +926,7 @@ class TailInputTest < Test::Unit::TestCase
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -944,7 +944,7 @@ class TailInputTest < Test::Unit::TestCase
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

data/test/plugin/test_parser_syslog.rb

@@ -64,6 +64,24 @@ class SyslogParserTest < ::Test::Unit::TestCase
     end
   end
 
+  def test_parse_various_characters_for_tag
+    ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
+    @parser.configure({})
+    @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
+      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+      assert_equal(@expected.merge('ident' => ident), record)
+    }
+  end
+
+  def test_parse_various_characters_for_tag_with_priority
+    ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
+    @parser.configure('with_priority' => true)
+    @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
+      assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+      assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
+    }
+  end
+
   class TestRFC5424Regexp < self
     def test_parse_with_rfc5424_message
       @parser.configure(

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.18
+  version: 0.14.19
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack