fluentd 1.9.0-x86-mingw32 → 1.9.1-x86-mingw32

data/lib/fluent/tls.rb

@@ -0,0 +1,81 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'openssl'
+require 'fluent/config/error'
+
+module Fluent
+  module TLS
+    DEFAULT_VERSION = :'TLSv1_2'
+    SUPPORTED_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                           [:'TLSv1_1', :'TLSv1_2', :'TLSv1_3', :'TLS1_1', :'TLS1_2', :'TLS1_3'].freeze
+                         else
+                           [:'TLSv1_1', :'TLSv1_2', :'TLS1_1', :'TLS1_2'].freeze
+                         end
+    ### follow httpclient configuration by nahi
+    # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
+    CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2".freeze # OpenSSL >1.0.0 default
+
+    METHODS_MAP = begin
+                    map = {
+                      TLSv1: OpenSSL::SSL::TLS1_VERSION,
+                      TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+                      TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+                    }
+                    map[:'TLSv1_3'] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                    MIN_MAX_AVAILABLE = true
+                    map.freeze
+                  rescue NameError
+                    # ruby 2.4 doesn't have OpenSSL::SSL::TLSXXX constants and min_version=/max_version= methods
+                    map = {
+                      TLS1: :'TLSv1',
+                      TLS1_1: :'TLSv1_1',
+                      TLS1_2: :'TLSv1_2',
+                    }.freeze
+                    MIN_MAX_AVAILABLE = false
+                    map
+                  end
+    private_constant :METHODS_MAP
+
+    # Helper for old syntax/method support:
+    # ruby 2.4 uses ssl_version= but this method is now deprecated.
+    # min_version=/max_version= use 'TLS1_2' but ssl_version= uses 'TLSv1_2'
+    def set_version_to_context(ctx, version, min_version, max_version)
+      if MIN_MAX_AVAILABLE
+        case
+        when min_version.nil? && max_version.nil?
+          min_version = METHODS_MAP[version] || version
+          max_version = METHODS_MAP[version] || version
+        when min_version.nil? && max_version
+          raise Fluent::ConfigError, "When you set max_version, must set min_version together"
+        when min_version && max_version.nil?
+          raise Fluent::ConfigError, "When you set min_version, must set max_version together"
+        else
+          min_version = METHODS_MAP[min_version] || min_version
+          max_version = METHODS_MAP[max_version] || max_version
+        end
+        ctx.min_version = min_version
+        ctx.max_version = max_version
+      else
+        ctx.ssl_version = METHODS_MAP[version] || version
+      end
+
+      ctx
+    end
+    module_function :set_version_to_context
+  end
+end
+

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.9.0'
+  VERSION = '1.9.1'
 
 end

data/test/config/test_section.rb

@@ -40,8 +40,6 @@ module Fluent::Config
             assert_equal("email", s1.send)
             assert_equal("normal", s1.klass)
             assert_equal(5, s1.keys)
-
-            assert_raise(NoMethodError) { s1.dup }
           end
 
           test 'creates object which contains specified hash object itself, including fields with at prefix' do

data/test/plugin/in_tail/test_position_file.rb

@@ -0,0 +1,192 @@
+require_relative '../../helper'
+require 'fluent/plugin/in_tail/position_file'
+
+class IntailPositionFileTest < Test::Unit::TestCase
+  setup do
+    @file = StringIO.new(+'')
+  end
+
+  UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
+  TEST_CONTENT = <<~EOF
+    valid_path\t0000000000000002\t0000000000000001
+    inode23bit\t0000000000000000\t00000000
+    invalidpath100000000000000000000000000000000
+    unwatched\t#{UNWATCHED_STR}\t0000000000000000
+  EOF
+
+  def write_data(f, content)
+    f.write(content)
+    f.seek(0)
+  end
+
+  sub_test_case '.compact' do
+    test 'compact invalid and convert 32 bit inode value' do
+      write_data(@file, TEST_CONTENT)
+      Fluent::Plugin::TailInput::PositionFile.compact(@file)
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal 2, lines.size
+      assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
+      assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
+    end
+
+    test 'compact data if duplicated line' do
+      write_data(@file, <<~EOF)
+        valid_path\t0000000000000002\t0000000000000001
+        valid_path\t0000000000000003\t0000000000000004
+      EOF
+      Fluent::Plugin::TailInput::PositionFile.compact(@file)
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
+    end
+  end
+
+  test '.parse' do
+    write_data(@file, TEST_CONTENT)
+    Fluent::Plugin::TailInput::PositionFile.parse(@file)
+
+    @file.seek(0)
+    lines = @file.readlines
+    assert_equal 2, lines.size
+    assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
+    assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
+  end
+
+  sub_test_case '#[]' do
+    test 'return entry' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
+
+      f = pf['valid_path']
+      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
+      assert_equal 2, f.read_pos
+      assert_equal 1, f.read_inode
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal 2, lines.size
+
+      f = pf['nonexist_path']
+      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
+      assert_equal 0, f.read_pos
+      assert_equal 0, f.read_inode
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal 3, lines.size
+      assert_equal "nonexist_path\t0000000000000000\t0000000000000000\n", lines[2]
+    end
+
+    test 'does not change other value position if other entry try to write' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
+
+      f = pf['nonexist_path']
+      assert_equal 0, f.read_inode
+      assert_equal 0, f.read_pos
+
+      pf['valid_path'].update(1, 2)
+
+      f = pf['nonexist_path']
+      assert_equal 0, f.read_inode
+      assert_equal 0, f.read_pos
+
+      pf['nonexist_path'].update(1, 2)
+      assert_equal 1, f.read_inode
+      assert_equal 2, f.read_pos
+    end
+  end
+
+  sub_test_case '#unwatch' do
+    test 'deletes entry by path' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
+      p1 = pf['valid_path']
+      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
+
+      pf.unwatch('valid_path')
+      assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
+
+      p2 = pf['valid_path']
+      assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
+
+      assert_not_equal p1, p2
+    end
+  end
+
+  sub_test_case 'FilePositionEntry' do
+    FILE_POS_CONTENT = <<~EOF
+      valid_path\t0000000000000002\t0000000000000001
+      valid_path2\t0000000000000003\t0000000000000002
+    EOF
+
+    def build_files(file)
+      r = {}
+
+      file.each_line do |line|
+        m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
+        path = m[1]
+        pos = m[2].to_i(16)
+        ino = m[3].to_i(16)
+        seek = file.pos - line.bytesize + path.bytesize + 1
+        r[path] = Fluent::Plugin::TailInput::FilePositionEntry.new(@file, Mutex.new, seek, pos, ino)
+      end
+
+      r
+    end
+
+    test '#update' do
+      write_data(@file, FILE_POS_CONTENT)
+      fs = build_files(@file)
+      f = fs['valid_path']
+      f.update(11, 10)
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal 2, lines.size
+      assert_equal "valid_path\t000000000000000a\t000000000000000b\n", lines[0]
+      assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1]
+    end
+
+    test '#update_pos' do
+      write_data(@file, FILE_POS_CONTENT)
+      fs = build_files(@file)
+      f = fs['valid_path']
+      f.update_pos(10)
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal 2, lines.size
+      assert_equal "valid_path\t000000000000000a\t0000000000000001\n", lines[0]
+      assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1]
+    end
+
+    test '#read_pos' do
+      write_data(@file, FILE_POS_CONTENT)
+      fs = build_files(@file)
+      f = fs['valid_path']
+      assert_equal 2, f.read_pos
+
+      f.update_pos(10)
+      assert_equal 10, f.read_pos
+
+      f.update(2, 11)
+      assert_equal 11, f.read_pos
+    end
+
+    test '#read_inode' do
+      write_data(@file, FILE_POS_CONTENT)
+      fs = build_files(@file)
+      f = fs['valid_path']
+      assert_equal 1, f.read_inode
+      f.update_pos(10)
+      assert_equal 1, f.read_inode
+
+      f.update(2, 11)
+      assert_equal 2, f.read_inode
+    end
+  end
+end

data/test/plugin/test_in_tail.rb

@@ -84,6 +84,19 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    test "multi paths with path_delimiter" do
+      c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "|" })
+      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
+      assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
+    end
+
+    test "multi paths with invaid path_delimiter" do
+      c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
+      assert_raise(Fluent::ConfigError) do
+        create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)

data/test/plugin/test_out_http.rb

@@ -168,6 +168,19 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_nil d.instance.headers
   end
 
+  def test_configure_with_warn
+    d = create_driver(config)
+    assert_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
+  end
+
+  def test_configure_without_warn
+    d = create_driver(<<~CONFIG)
+      endpoint #{base_endpoint}/test
+      retryable_response_codes [503]
+    CONFIG
+    assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
+  end
+
   data('json' => ['json', 'application/x-ndjson'],
        'ltsv' => ['ltsv', 'text/tab-separated-values'],
        'msgpack' => ['msgpack', 'application/x-msgpack'],
@@ -257,7 +270,7 @@ class HTTPOutputTest < Test::Unit::TestCase
         d.feed(event)
       }
     end
-    assert_match(/got error response from.*404 Not Found Not Found/, d.instance.log.out.logs.first)
+    assert_match(/got error response from.*404 Not Found Not Found/, d.instance.log.out.logs.join)
     d.instance_shutdown
   end
 
@@ -313,7 +326,7 @@ class HTTPOutputTest < Test::Unit::TestCase
           d.feed(event)
         }
       end
-      assert_match(/got unrecoverable error/, d.instance.log.out.logs.first)
+      assert_match(/got unrecoverable error/, d.instance.log.out.logs.join)
 
       d.instance_shutdown
     end

data/test/plugin/test_output_as_buffered_backup.rb

@@ -181,7 +181,8 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
          'type error' => TypeError,
          'argument error' => ArgumentError,
          'no method error' => NoMethodError,
-         'msgpack unpack error' => MessagePack::UnpackError)
+         'msgpack unpack error' => MessagePack::UnpackError,
+         'encoding error' => EncodingError)
     test 'backup chunk without secondary' do |error_class|
       Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
         id = 'backup_test'

data/test/plugin/test_parser_syslog.rb

@@ -188,6 +188,42 @@ class SyslogParserTest < ::Test::Unit::TestCase
                    @parser.instance.patterns['format'])
     end
 
+    def test_parse_with_rfc5424_message_trailing_eol
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!\n"
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+    end
+
+    def test_parse_with_rfc5424_multiline_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi,\nfrom\nFluentd!"
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi,\nfrom\nFluentd!", record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+    end
+
     def test_parse_with_rfc5424_message_and_without_priority
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',

data/test/plugin_helper/data/cert/generate_cert.rb

@@ -0,0 +1,87 @@
+require 'fluent/plugin_helper/cert_option'
+require 'fileutils'
+
+module CertUtil
+  extend Fluent::PluginHelper::CertOption
+end
+
+WITHOUT_CA_DIR = './without_ca'.freeze
+WITH_CA_DIR = './with_ca'.freeze
+
+CA_OPTION = {
+  private_key_length: 2048,
+  country: 'US',
+  state: 'CA',
+  locality: 'Mountain View',
+  common_name: 'ca.testing.fluentd.org',
+  expiration: 30 * 86400,
+  digest: :sha256,
+}
+
+SERVER_OPTION = {
+  private_key_length: 2048,
+  country: 'US',
+  state: 'CA',
+  locality: 'Mountain View',
+  common_name: 'server.testing.fluentd.org',
+  expiration: 30 * 86400,
+  digest: :sha256,
+}
+
+def write_cert_and_key(cert_path, cert, key_path, key, passphrase)
+  File.open(cert_path, 'w') { |f| f.write(cert.to_pem) }
+
+  # Write the secret key (raw or encrypted by AES256) in PEM format
+  key_str = passphrase ? key.export(OpenSSL::Cipher.new('AES-256-CBC'), passphrase) : key.export
+  File.open(key_path, 'w') { |f| f.write(key_str) }
+  File.chmod(0o600, cert_path, key_path)
+end
+
+def create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)
+  cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(SERVER_OPTION)
+  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+  cert
+end
+
+def create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)
+  cert, key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(CA_OPTION)
+  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+  cert
+end
+
+def create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
+  cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, SERVER_OPTION)
+  write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+  cert
+end
+
+def create_without_ca
+  FileUtils.mkdir_p(WITHOUT_CA_DIR)
+  cert_path = File.join(WITHOUT_CA_DIR, 'cert.pem')
+  cert_key_path = File.join(WITHOUT_CA_DIR, 'cert-key.pem')
+  cert_pass_path = File.join(WITHOUT_CA_DIR, 'cert-pass.pem')
+  cert_key_pass_path = File.join(WITHOUT_CA_DIR, 'cert-key-pass.pem')
+
+  create_server_pair_signed_by_self(cert_path, cert_key_path, nil)
+  create_server_pair_signed_by_self(cert_pass_path, cert_key_pass_path, 'apple') # with passphrase
+end
+
+def create_with_ca
+  FileUtils.mkdir_p(WITH_CA_DIR)
+  cert_path = File.join(WITH_CA_DIR, 'cert.pem')
+  cert_key_path = File.join(WITH_CA_DIR, 'cert-key.pem')
+  ca_cert_path = File.join(WITH_CA_DIR, 'ca-cert.pem')
+  ca_key_path = File.join(WITH_CA_DIR, 'ca-cert-key.pem')
+  create_ca_pair_signed_by_self(ca_cert_path, ca_key_path, nil)
+  create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, nil, cert_path, cert_key_path, nil)
+
+  cert_pass_path = File.join(WITH_CA_DIR, 'cert-pass.pem')
+  cert_key_pass_path = File.join(WITH_CA_DIR, 'cert-key-pass.pem')
+  ca_cert_pass_path = File.join(WITH_CA_DIR, 'ca-cert-pass.pem')
+  ca_key_pass_path = File.join(WITH_CA_DIR, 'ca-cert-key-pass.pem')
+  create_ca_pair_signed_by_self(ca_cert_pass_path, ca_key_pass_path, 'orange') # with passphrase
+  create_server_pair_signed_by_ca(ca_cert_pass_path, ca_key_pass_path, 'orange', cert_pass_path, cert_key_pass_path, 'apple')
+end
+
+create_without_ca
+create_with_ca