fluentd 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c973680de36bad1981b1cc6bd3c23c391a04fc66
-  data.tar.gz: eb6b2c87bd06d19c2d90050355059e6030f47c14
+  metadata.gz: ab82be7292539aa23259062e0a5fb6fbe9d6a5ee
+  data.tar.gz: f2114ce7960c197f57693e8c84e377cb2ee2edb4
 SHA512:
-  metadata.gz: 390c4766aca79e9e3e235fad17b66e5ab0d979df6d77e91ac7f03b4be9a831b60214bef63a5cf0811548a8020f4954df4f27beda58dedeb5948a0f804b6a1c42
-  data.tar.gz: cef9e1fcf326748d4a72420c5ae4f6b50f3ed8af8178ba0f405ee667a675435121de90099a27170d96ac9592ed421e71655a40b2060a1967f58f62e1c311c047
+  metadata.gz: f4cffdc9ff288294bf1ed8f93dac35045f488309b8d303765fdce93434077e820f10c326f9af4437ba26b63b7f1c1636677b03c1514fa61c0a20dcb3d3f69361
+  data.tar.gz: d7fa53ec4203b5b818d4bffea29102f295824c7120ce50217ec45d473f225e152647f6d0ed20049dd0e0a58f79e626012a48724620b203cb8b2aed54f27a06b5

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # v1.3
 
+## Release v1.3.1 - 2018/11/27
+
+### Enhancements
+
+* out_forward: Separate parameter names for certificate
+  https://github.com/fluent/fluentd/pull/2181
+  https://github.com/fluent/fluentd/pull/2190
+* out_forward: Add `verify_connection_at_startup` parameter to check connection setting at startup phase
+  https://github.com/fluent/fluentd/pull/2184
+* config: Check right slash position in regexp type
+  https://github.com/fluent/fluentd/pull/2176
+* parser_nginx: Support multiple IPs in `http_x_forwarded_for` field
+  https://github.com/fluent/fluentd/pull/2171
+
+### Bug fixes
+
+* fluent-cat: Fix retry limit handling
+  https://github.com/fluent/fluentd/pull/2193
+* record_accessor helper: Delete top level field with bracket style
+  https://github.com/fluent/fluentd/pull/2192
+* filter_record_transformer: Keep `class` methond to avoid undefined method error
+  https://github.com/fluent/fluentd/pull/2186
+
 ## Release v1.3.0 - 2018/11/10
 
 ### New features

data/lib/fluent/command/cat.rb

@@ -34,6 +34,7 @@ config_path = Fluent::DEFAULT_CONFIG_PATH
 format = 'json'
 message_key = 'message'
 time_as_integer = false
+retry_limit = 5
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -75,6 +76,10 @@ op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClas
   time_as_integer = true
 }
 
+op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_limit})", Integer) {|n|
+  retry_limit = n
+}
+
 singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -107,6 +112,8 @@ require 'msgpack'
 class Writer
   include MonitorMixin
 
+  RetryLimitError = Class.new(StandardError)
+
   class TimerThread
     def initialize(writer)
       @writer = writer
@@ -130,7 +137,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
     @tag = tag
     @connector = connector
     @socket = false
@@ -142,7 +149,7 @@ class Writer
     @pending = []
     @pending_limit = 1024  # TODO
     @retry_wait = 1
-    @retry_limit = 5  # TODO
+    @retry_limit = retry_limit
     @time_as_integer = time_as_integer
 
     super()
@@ -236,21 +243,24 @@ class Writer
   end
 
   def try_connect
-    now = Time.now.to_i
-
-    unless @error_history.empty?
-      # wait before re-connecting
-      wait = @retry_wait * (2 ** (@error_history.size-1))
-      if now <= @socket_time + wait
-        return false
+    begin
+      now = Time.now.to_i
+
+      unless @error_history.empty?
+        # wait before re-connecting
+        wait = 1 #@retry_wait * (2 ** (@error_history.size-1))
+        if now <= @socket_time + wait
+          sleep(wait)
+          try_connect
+        end
       end
-    end
 
-    begin
       @socket = @connector.call
       @error_history.clear
       return true
 
+    rescue RetryLimitError => ex
+      raise ex
     rescue
       $stderr.puts "connect failed: #{$!}"
       @error_history << $!
@@ -263,9 +273,10 @@ class Writer
         }
         @pending.clear
         @error_history.clear
+        raise RetryLimitError, "exceed retry limit"
+      else
+        retry
       end
-
-      return false
     end
   end
 
@@ -285,7 +296,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
 w.start
 
 case format

data/lib/fluent/config/types.rb

@@ -74,6 +74,9 @@ module Fluent
       return nil unless str
       return Regexp.compile(str) unless str.start_with?("/")
       right_slash_position = str.rindex("/")
+      if right_slash_position < str.size - 3
+        raise Fluent::ConfigError, "invalid regexp: missing right slash: #{str}"
+      end
       options = str[(right_slash_position + 1)..-1]
       option = 0
       option |= Regexp::IGNORECASE if options.include?("i")

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -316,7 +316,7 @@ module Fluent::Plugin
         end
 
         (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
         end
       end
     end

data/lib/fluent/plugin/out_forward.rb

@@ -77,6 +77,9 @@ module Fluent::Plugin
     desc 'Ignore DNS resolution and errors at startup time.'
     config_param :ignore_network_errors_at_startup, :bool, default: false
 
+    desc 'Verify that a connection can be made with one of out_forward nodes at the time of startup.'
+    config_param :verify_connection_at_startup, :bool, default: false
+
     desc 'Compress buffered data.'
     config_param :compress, :enum, list: [:text, :gzip], default: :text
 
@@ -91,6 +94,8 @@ module Fluent::Plugin
     desc 'Verify hostname of servers and certificates or not in TLS transport.'
     config_param :tls_verify_hostname, :bool, default: true
     desc 'The additional CA certificate path for TLS.'
+    config_param :tls_ca_cert_path, :array, value_type: :string, default: nil
+    desc 'The additional certificate path for TLS.'
     config_param :tls_cert_path, :array, value_type: :string, default: nil
 
     config_section :security, required: false, multi: false do
@@ -166,8 +171,12 @@ module Fluent::Plugin
       end
 
       if @transport == :tls
+        # socket helper adds CA cert or signed certificate to same cert store internally so unify it in this place.
         if @tls_cert_path && !@tls_cert_path.empty?
-          @tls_cert_path.each do |path|
+          @tls_ca_cert_path = @tls_cert_path
+        end
+        if @tls_ca_cert_path && !@tls_ca_cert_path.empty?
+          @tls_ca_cert_path.each do |path|
             raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path)
             raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path)
           end
@@ -253,6 +262,17 @@ module Fluent::Plugin
         @sock_ack_waiting = []
         thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
+
+      if @verify_connection_at_startup
+        @nodes.each do |node|
+          begin
+            node.verify_connection
+          rescue StandardError => e
+            log.fatal "forward's connection setting error: #{e.message}"
+            raise Fluent::UnrecoverableError, e.message
+          end
+        end
+      end
     end
 
     def close
@@ -324,7 +344,7 @@ module Fluent::Plugin
           verify_fqdn: @tls_verify_hostname,
           fqdn: hostname,
           allow_self_signed_cert: @tls_allow_self_signed_cert,
-          cert_paths: @tls_cert_path,
+          cert_paths: @tls_ca_cert_path,
           linger_timeout: @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
@@ -568,6 +588,19 @@ module Fluent::Plugin
         @standby
       end
 
+      def verify_connection
+        sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
+        begin
+          ri = RequestInfo.new(@sender.security ? :helo : :established)
+          if ri.state != :established
+            establish_connection(sock, ri)
+            raise if ri.state != :established
+          end
+        ensure
+          sock.close
+        end
+      end
+
       def establish_connection(sock, ri)
         while available? && ri.state != :established
           begin

data/lib/fluent/plugin/parser_nginx.rb

@@ -21,7 +21,7 @@ module Fluent
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>[^ ]+))?)?$/
+      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+\"?(?<http_x_forwarded_for>[^\"]*)\"?)?)?$/
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -40,8 +40,14 @@ module Fluent
           if @keys.is_a?(Array)
             @last_key = @keys.last
             @dig_keys = @keys[0..-2]
-            mcall = method(:call_dig)
-            mdelete = method(:delete_nest)
+            if @dig_keys.empty?
+              @keys = @keys.first
+              mcall = method(:call_index)
+              mdelete = method(:delete_top)
+            else
+              mcall = method(:call_dig)
+              mdelete = method(:delete_nest)
+            end
           else
             # Call [] for single key to reduce dig overhead
             mcall = method(:call_index)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.3.0'
+  VERSION = '1.3.1'
 
 end

data/test/config/test_types.rb

@@ -79,6 +79,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
     test 'w/o slashes' do |(expected, str)|
       assert_equal(expected, Config.regexp_value(str))
     end
+
+    data("missing right slash" => "/regexp",
+         "too many options" => "/regexp/imx",)
+    test 'invalid regexp' do |(str)|
+      assert_raise(Fluent::ConfigError.new("invalid regexp: missing right slash: #{str}")) do
+        Config.regexp_value(str)
+      end
+    end
   end
 
   sub_test_case 'type converters for config_param definitions' do

data/test/plugin/test_out_forward.rb

@@ -9,6 +9,8 @@ require 'fluent/plugin/in_forward'
 class ForwardOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
     @d = nil
   end
 
@@ -16,6 +18,8 @@ class ForwardOutputTest < Test::Unit::TestCase
     @d.instance_shutdown if @d
   end
 
+  TMP_DIR = File.join(__dir__, "../tmp/out_forward#{ENV['TEST_ENV_NUMBER']}")
+
   TARGET_HOST = '127.0.0.1'
   TARGET_PORT = unused_port
   CONFIG = %[
@@ -153,6 +157,27 @@ EOL
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
 
+  data('CA cert'     => 'tls_ca_cert_path',
+       'non CA cert' => 'tls_cert_path')
+  test 'configure tls_cert_path/tls_ca_cert_path' do |param|
+    dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
+    FileUtils.touch(dummy_cert_path)
+    conf = %[
+      send_timeout 5
+      transport tls
+      tls_insecure_mode true
+      #{param} #{dummy_cert_path}
+      <server>
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+      </server>
+    ]
+
+    @d = d = create_driver(conf)
+    # In the plugin, tls_ca_cert_path is used for both cases
+    assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress
@@ -217,6 +242,18 @@ EOL
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'verify_connection_at_startup is disabled in default' do
+    @d = d = create_driver(CONFIG)
+    assert_false d.instance.verify_connection_at_startup
+  end
+
+  test 'verify_connection_at_startup can be enabled' do
+    @d = d = create_driver(CONFIG + %[
+      verify_connection_at_startup true
+    ])
+    assert_true d.instance.verify_connection_at_startup
+  end
+
   test 'send tags in str (utf-8 strings)' do
     target_input_driver = create_target_input_driver
 
@@ -774,4 +811,113 @@ EOL
       i.configure(conf)
     end
   end
+
+  sub_test_case 'verify_connection_at_startup' do
+    test 'nodes are not available' do
+      @d = d = create_driver(CONFIG + %[
+        verify_connection_at_startup true
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+        </buffer>
+      ])
+      assert_raise Fluent::UnrecoverableError do
+        d.instance_start
+      end
+      d.instance_shutdown
+    end
+
+    test 'nodes_shared_key_miss_match' do
+      input_conf = TARGET_CONFIG + %[
+                   <security>
+                     self_hostname in.localhost
+                     shared_key fluentd-sharedkey
+                   </security>
+                 ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+      output_conf = %[
+        send_timeout 30
+        heartbeat_type transport
+        transport tls
+        tls_verify_hostname false
+        verify_connection_at_startup true
+        require_ack_response true
+        ack_response_timeout 5s
+        <security>
+          self_hostname localhost
+          shared_key key_miss_match
+        </security>
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+          flush_thread_interval 31s
+        </buffer>
+
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      target_input_driver.run(expect_records: 1, timeout: 15) do
+        assert_raise Fluent::UnrecoverableError do
+          d.instance_start
+        end
+        d.instance_shutdown
+      end
+    end
+
+    test 'nodes_shared_key_match' do
+      input_conf = TARGET_CONFIG + %[
+                       <security>
+                         self_hostname in.localhost
+                         shared_key fluentd-sharedkey
+                         <client>
+                           host 127.0.0.1
+                         </client>
+                       </security>
+                     ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+
+      output_conf = %[
+          send_timeout 51
+          verify_connection_at_startup true
+          <security>
+            self_hostname localhost
+            shared_key fluentd-sharedkey
+          </security>
+          <server>
+            name test
+            host #{TARGET_HOST}
+            port #{TARGET_PORT}
+            shared_key fluentd-sharedkey
+          </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      time = event_time("2011-01-02 13:14:15 UTC")
+      records = [
+          {"a" => 1},
+          {"a" => 2}
+      ]
+
+      target_input_driver.run(expect_records: 2, timeout: 15) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
+        end
+      end
+
+      events = target_input_driver.events
+      assert{ events != [] }
+      assert_equal(['test', time, records[0]], events[0])
+      assert_equal(['test', time, records[1]], events[1])
+    end
+  end
 end

data/test/plugin/test_parser_nginx.rb

@@ -28,6 +28,18 @@ class NginxParserTest < ::Test::Unit::TestCase
       'agent'                => 'Opera/12.0',
       'http_x_forwarded_for' => '-'
     }
+    @expected_extended_multiple_ip = {
+      'remote'               => '127.0.0.1',
+      'host'                 => '192.168.0.1',
+      'user'                 => '-',
+      'method'               => 'GET',
+      'path'                 => '/',
+      'code'                 => '200',
+      'size'                 => '777',
+      'referer'              => '-',
+      'agent'                => 'Opera/12.0',
+      'http_x_forwarded_for' => '127.0.0.1, 192.168.0.1'
+    }
   end
 
   def create_driver
@@ -65,4 +77,12 @@ class NginxParserTest < ::Test::Unit::TestCase
       assert_equal(@expected_extended, record)
     }
   end
+
+  def test_parse_with_http_x_forwarded_for_multiple_ip
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" "127.0.0.1, 192.168.0.1"') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected_extended_multiple_ip, record)
+    }
+  end
 end

data/test/plugin_helper/test_record_accessor.rb

@@ -110,6 +110,12 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
       assert_equal r[param], accessor.call(r)
     end
 
+    test "access single dot key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      assert_equal 'v3', accessor.call(r)
+    end
+
     test "nested bracket keys with dot" do
       r = {'key1' => {'this.is.key3' => 'value'}}
       accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
@@ -164,6 +170,13 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
       assert_not_include(r, param)
     end
 
+    test "delete top key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      accessor.delete(r)
+      assert_not_include(r, 'this.is.key3')
+    end
+
     data('bracket' => "$['key1'][0]['ke y2']",
          'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
     test "delete nested keys ['key1', 0, 'ke y2']" do |param|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-10 00:00:00.000000000 Z
+date: 2018-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack