fluentd 1.7.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa26c04f10203245130ee2457e56f1ea064df188d91c3610aa338f5f66273663
-  data.tar.gz: 48031fcf5dc26c8d1e3a6f6fc4a5f629451c6dc9a06c4a95043c3ba56da031b1
+  metadata.gz: 5bb8d41e15ede72bfbb90c06171c0eb4aa77d479cda1a8320f60e6166fbc27fc
+  data.tar.gz: d781e152f408a74e1cfed6b2c37f593db814d551d9f9179d1c5d15581218bf43
 SHA512:
-  metadata.gz: ac003c0a88bfd623a9e1caf2989db9e45b38841827dc312e4e266ff38cc1ac25001a58fda0fc5b7eebb9b4efa4bfc9cbf0266ad36314aa5ed518303040e09727
-  data.tar.gz: b84fb3fe1b06ee99fcdad4cbc37d897b507b5b916d5ad8b539863827e6d1875ca6c026ae54b6b05898750e84357db51583e59872ee76bc814e3bc7e29e2e3fc2
+  metadata.gz: 1f04e656acafae78bf4e2a770a5df16d549aa59ecd0a5883e97884b1de2dccc612108b11a1302b0655a2d56e3fb0b5f077da345441f34ce94a6bc5d31b3fac61
+  data.tar.gz: d16a3635f5c9607de95f66f3b836c44786abd7db9a4bb369b51f41862df0279bc3812962c633fe8ac8ad9f4439fe22b8aaf18ff7aa099dafe531956c0a33985e

data/CHANGELOG.md

@@ -1,11 +1,36 @@
 # v1.7
 
+## Release v1.7.1 - 2019/09/08
+
+### Enhancement
+
+* socket helper/out_forward: Support Windows certstore to load certificates
+  https://github.com/fluent/fluentd/pull/2601
+* parser_syslog: Add faster parser for rfc3164 message
+  https://github.com/fluent/fluentd/pull/2599
+
+### Bug fixes
+
+* buf_file/buf_file_single: fix to ignore placeholder based path.
+  https://github.com/fluent/fluentd/pull/2594
+* server helper: Ignore ETIMEDOUT error in SSL_accept
+  https://github.com/fluent/fluentd/pull/2595
+* buf_file: ensure to remove metadata after buffer creation failure
+  https://github.com/fluent/fluentd/pull/2598
+* buf_file_single: fix duplicated path setting check
+  https://github.com/fluent/fluentd/pull/2600
+* fix msgpack-ruby depedency to use recent feature
+  https://github.com/fluent/fluentd/pull/2606
+
+
 ## Release v1.7.0 - 2019/08/20
 
 ### New feature
 
-* buffer: Add file_single buffer
-  https://github.com/fluent/fluentd/pull/2479
+* buffer: Add file_single buffer plugin
+  https://github.com/fluent/fluentd/pull/2579
+* output: Add http output plugin
+  https://github.com/fluent/fluentd/pull/2488
 
 ### Enhancement
 
@@ -13,9 +38,9 @@
   https://github.com/fluent/fluentd/pull/2560
   https://github.com/fluent/fluentd/pull/2563
   https://github.com/fluent/fluentd/pull/2564
-* output: Use Mutext instead of Monitor
+* output: Use Mutex instead of Monitor
   https://github.com/fluent/fluentd/pull/2561
-* event: Add `OneEvent#empty?` method
+* event: Add `OneEventStrea#empty?` method
   https://github.com/fluent/fluentd/pull/2565
 * thread: Set thread name for ruby 2.3 or later
   https://github.com/fluent/fluentd/pull/2574

data/fluentd.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
+  gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
+    gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
   end
 
   gem.add_development_dependency("rake", ["~> 11.0"])

data/lib/fluent/plugin/buf_file.rb

@@ -132,7 +132,7 @@ module Fluent
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
-        Dir.glob(patterns) do |path|
+        Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
           log.debug { "restoring buffer file: path = #{path}" }
@@ -183,6 +183,15 @@ module Fluent
         # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
         File.unlink(path, path + '.meta') rescue nil
       end
+
+      private
+
+      def escaped_patterns(patterns)
+        patterns.map { |pattern|
+          # '{' '}' are special character in Dir.glob
+          pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
+        }
+      end
     end
   end
 end

data/lib/fluent/plugin/buf_file_single.rb

@@ -88,14 +88,6 @@ module Fluent
           end
         end
 
-        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !called_in_test?
-          type_using_this_path = @@buffer_paths[@path]
-          raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
-        end
-
-        @@buffer_paths[@path] = type_of_owner
-
         specified_directory_exists = File.exist?(@path) && File.directory?(@path)
         unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
 
@@ -124,6 +116,13 @@ module Fluent
           @multi_workers_available = false
         end
 
+        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
+          type_using_this_path = @@buffer_paths[@path]
+          raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
+        end
+
+        @@buffer_paths[@path] = type_of_owner
         @dir_permission = if @dir_permission
                             @dir_permission.to_i(8)
                           else
@@ -155,7 +154,7 @@ module Fluent
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
-        Dir.glob(patterns) do |path|
+        Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
           log.debug { "restoring buffer file: path = #{path}" }
@@ -206,6 +205,15 @@ module Fluent
         # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
         File.unlink(path) rescue nil
       end
+
+      private
+
+      def escaped_patterns(patterns)
+        patterns.map { |pattern|
+          # '{' '}' are special character in Dir.glob
+          pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
+        }
+      end
     end
   end
 end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -300,6 +300,13 @@ module Fluent
             # This case is easier than enqueued!. Just removing pre-create buffer file
             @chunk.close rescue nil
             File.unlink(@path) rescue nil
+
+            if @meta
+              # ensure to unlink when #write_metadata fails
+              @meta.close rescue nil
+              File.unlink(@meta_path) rescue nil
+            end
+
             # Same as @chunk case. See above
             raise BufferOverflowError, "can't create buffer metadata for #{path}. Stop creating buffer files: error = #{e}"
           end

data/lib/fluent/plugin/out_forward.rb

@@ -107,6 +107,12 @@ module Fluent::Plugin
     config_param :tls_client_private_key_path, :string, default: nil
     desc 'The client private key passphrase for TLS.'
     config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true
+    desc 'The certificate thumbprint for searching from Windows system certstore.'
+    config_param :tls_cert_thumbprint, :string, default: nil, secret: true
+    desc 'The certificate logical store name on Windows system certstore.'
+    config_param :tls_cert_logical_store_name, :string, default: nil
+    desc 'Enable to use certificate enterprise store on Windows system certstore.'
+    config_param :tls_cert_use_enterprise_store, :bool, default: true
     desc "Enable keepalive connection."
     config_param :keepalive, :bool, default: false
     desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
@@ -198,6 +204,15 @@ module Fluent::Plugin
           @tls_verify_hostname = false
           @tls_allow_self_signed_cert = true
         end
+
+        if Fluent.windows?
+          if (@tls_cert_path || @tls_ca_cert_path) && @tls_cert_logical_store_name
+            raise Fluent::ConfigError, "specified both cert path and tls_cert_logical_store_name is not permitted"
+          end
+        else
+          raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_logical_store_name
+          raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_thumbprint
+        end
       end
 
       @ack_handler = @require_ack_response ? AckHandler.new(timeout: @ack_response_timeout, log: @log, read_length: @read_length) : nil
@@ -346,6 +361,9 @@ module Fluent::Plugin
           cert_path: @tls_client_cert_path,
           private_key_path: @tls_client_private_key_path,
           private_key_passphrase: @tls_client_private_key_passphrase,
+          cert_thumbprint: @tls_cert_thumbprint,
+          cert_logical_store_name: @tls_cert_logical_store_name,
+          cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
           # Enabling SO_LINGER causes data loss on Windows
           # https://github.com/fluent/fluentd/issues/1968

data/lib/fluent/plugin/parser_syslog.rb

@@ -38,6 +38,10 @@ module Fluent
       config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
       desc 'Specify time format for event time for rfc5424 protocol'
       config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
+      desc 'The parser type used to parse syslog message'
+      config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
+      desc 'support colonless ident in string parser'
+      config_param :support_colonless_ident, :bool, default: true
 
       def initialize
         super
@@ -50,10 +54,17 @@ module Fluent
         @time_parser_rfc3164 = @time_parser_rfc5424 = nil
         @time_parser_rfc5424_without_subseconds = nil
         @support_rfc5424_without_subseconds = false
+        @regexp_parser = @parser_type == :regexp
         @regexp = case @message_format
                   when :rfc3164
-                    class << self
-                      alias_method :parse, :parse_plain
+                    if @regexp_parser
+                      class << self
+                        alias_method :parse, :parse_plain
+                      end
+                    else
+                      class << self
+                        alias_method :parse, :parse_rfc3164
+                      end
                     end
                     @with_priority ? REGEXP_WITH_PRI : REGEXP
                   when :rfc5424
@@ -88,11 +99,16 @@ module Fluent
           @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
           @time_parser = @time_parser_rfc5424
           @support_rfc5424_without_subseconds = true
+          parse_plain(text, &block)
         else
           @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
           @time_parser = @time_parser_rfc3164
+          if @regexp_parser
+            parse_plain(text, &block)
+          else
+            parse_rfc3164(text, &block)
+          end
         end
-        parse_plain(text, &block)
       end
 
       def parse_plain(text, &block)
@@ -137,6 +153,93 @@ module Fluent
 
         yield time, record
       end
+
+      SPLIT_CHAR = ' '.freeze
+
+      def parse_rfc3164(text, &block)
+        pri = nil
+        cursor = 0
+        if @with_priority
+          if text.start_with?('<'.freeze)
+            i = text.index('>'.freeze, 1)
+            if i < 2
+              yield nil, nil
+              return
+            end
+            pri = text.slice(1, i - 1).to_i
+            cursor = i + 1
+          else
+            yield nil, nil
+            return
+          end
+        end
+
+        # header part
+        time_size = 15 # skip Mmm dd hh:mm:ss
+        time_end = text[cursor + time_size]
+        if time_end == SPLIT_CHAR
+          time_str = text.slice(cursor, time_size)
+          cursor += 16 # time + ' '
+        elsif time_end == '.'.freeze
+          # support subsecond time
+          i = text.index(SPLIT_CHAR, time_size)
+          time_str = text.slice(cursor, i - cursor)
+          cursor = i + 1
+        else
+          yield nil, nil
+          return
+        end
+
+        i = text.index(SPLIT_CHAR, cursor)
+        if i.nil?
+          yield nil, nil
+          return
+        end
+        host_size = i - cursor
+        host = text.slice(cursor, host_size)
+        cursor += host_size + 1
+
+        record = {'host' => host}
+        record['pri'] = pri if pri
+
+        i = text.index(SPLIT_CHAR, cursor)
+
+        # message part
+        msg = if i.nil?  # for 'only non-space content case'
+                text.slice(cursor, text.bytesize)
+              else
+                if text[i - 1] == ':'.freeze
+                  if text[i - 2] == ']'.freeze
+                    left_braket_pos = text.index('['.freeze, cursor)
+                    record['ident'] = text.slice(cursor, left_braket_pos - cursor)
+                    record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 3) # remove '[' / ']:'
+                  else
+                    record['ident'] = text.slice(cursor, i - cursor - 1)
+                  end
+                  text.slice(i + 1, text.bytesize)
+                else
+                  if @support_colonless_ident
+                    if text[i - 1] == ']'.freeze
+                      left_braket_pos = text.index('['.freeze, cursor)
+                      record['ident'] = text.slice(cursor, left_braket_pos - cursor)
+                      record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 2) # remove '[' / ']'
+                    else
+                      record['ident'] = text.slice(cursor, i - cursor)
+                    end
+                    text.slice(i + 1, text.bytesize)
+                  else
+                    text.slice(cursor, text.bytesize)
+                  end
+                end
+              end
+        msg.chomp!
+        record['message'] = msg
+
+        time = @time_parser.parse(time_str)
+        record['time'] = time_str if @keep_time_key
+
+        yield time, record
+      end
     end
   end
 end

data/lib/fluent/plugin_helper/server.rb

@@ -722,7 +722,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/plugin_helper/socket.rb

@@ -17,6 +17,9 @@
 require 'socket'
 require 'ipaddr'
 require 'openssl'
+if Fluent.windows?
+  require 'certstore'
+end
 
 require_relative 'socket_option'
 
@@ -96,7 +99,9 @@ module Fluent
           host, port,
           version: TLS_DEFAULT_VERSION, ciphers: CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
           enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
-          cert_path: nil, private_key_path: nil, private_key_passphrase: nil, **kwargs, &block)
+          cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
+          cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
+          **kwargs, &block)
 
         host_is_ipaddress = IPAddr.new(host) rescue false
         fqdn ||= host unless host_is_ipaddress
@@ -113,6 +118,14 @@ module Fluent
           end
           begin
             if enable_system_cert_store
+              if Fluent.windows? && cert_logical_store_name
+                log.trace "loading Windows system certificate store"
+                loader = Certstore::OpenSSL::Loader.new(log, cert_store, cert_logical_store_name,
+                                                        enterprise: cert_use_enterprise_store)
+                loader.load_cert_store
+                cert_store = loader.cert_store
+                context.cert = loader.get_certificate(cert_thumbprint) if cert_thumbprint
+              end
               log.trace "loading system default certificate store"
               cert_store.set_default_paths
             end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.0'
+  VERSION = '1.7.1'
 
 end

data/test/plugin/test_buf_file.rb

@@ -537,6 +537,60 @@ class FileBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'there are some existing file chunks with placeholders path' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_${test}_file', __FILE__)
+      FileUtils.rm_rf(@bufdir)
+      FileUtils.mkdir_p(@bufdir)
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        1, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @c2id = Fluent::UniqueId.generate
+      p2 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c2id)}.log")
+      File.open(p2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
+        1, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
+      )
+
+      @bufpath = File.join(@bufdir, 'etest.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      FileUtils.rm_rf(@bufdir)
+    end
+
+    test '#resume returns staged/queued chunks with metadata' do
+      assert_equal 1, @p.stage.size
+      assert_equal 1, @p.queue.size
+    end
+  end
+
   sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
     setup do
       @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)

data/test/plugin/test_buf_file_single.rb

@@ -138,6 +138,25 @@ class FileSingleBufferTest < Test::Unit::TestCase
     end
   end
 
+  test 'raise config error when using same file path' do
+    d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
+    d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
+    Fluent::SystemConfig.overwrite_system_config({}) do
+      d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))
+    end
+
+    any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|
+      stub(klass).called_in_test? { false }
+    end
+
+    err = assert_raise(Fluent::ConfigError) do
+      Fluent::SystemConfig.overwrite_system_config({}) do
+        d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))
+      end
+    end
+    assert_match(/plugin already uses same buffer path/, err.message)
+  end
+
   sub_test_case 'buffer plugin configured only with path' do
     setup do
       @bufpath = File.join(@bufdir, 'testbuf.*.buf')
@@ -502,6 +521,54 @@ class FileSingleBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'there are some existing file chunks with placeholders path' do
+    setup do
+      @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)
+      FileUtils.rm_rf(@buf_ph_dir)
+      FileUtils.mkdir_p(@buf_ph_dir)
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      t = Time.now - 50000
+      File.utime(t, t, p1)
+
+      @c2id = Fluent::UniqueId.generate
+      p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf")
+      File.open(p2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      FileUtils.rm_rf(@buf_ph_dir)
+    end
+
+    test '#resume returns staged/queued chunks with metadata' do
+      @d = create_driver(%[
+        <buffer tag>
+          @type file_single
+          path #{@buf_ph_dir}
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+      @p.start
+
+      assert_equal 1, @p.stage.size
+      assert_equal 1, @p.queue.size
+    end
+  end
+
   sub_test_case 'there are some existing msgpack file chunks' do
     setup do
       packer = Fluent::MessagePackFactory.packer

data/test/plugin/test_buffer_file_chunk.rb

@@ -495,6 +495,24 @@ class BufferFileChunkTest < Test::Unit::TestCase
     end
   end
 
+  test 'ensure to remove metadata file if #write_metadata raise an error becuase of disk full' do
+    chunk_path = File.join(@chunkdir, 'test.*.log')
+    stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
+
+    any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass|
+      stub(klass).write_metadata(anything) do |v|
+        raise 'disk full'
+      end
+    end
+
+    err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do
+      Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
+    end
+
+    assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta'))
+    assert_match(/create buffer metadata/, err.message)
+  end
+
   sub_test_case 'chunk with file for staged chunk' do
     setup do
       @chunk_id = gen_test_chunk_id
@@ -855,6 +873,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert_equal @gzipped_src, c.read(compressed: :gzip)
 
       io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end

data/test/plugin/test_buffer_file_single_chunk.rb

@@ -613,6 +613,7 @@ class BufferFileSingleChunkTest < Test::Unit::TestCase
       assert_equal @gzipped_src, c.read(compressed: :gzip)
 
       io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end

data/test/plugin/test_buffer_memory_chunk.rb

@@ -331,6 +331,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
       assert_equal @gzipped_src, c.read(compressed: :gzip)
 
       io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end

data/test/plugin/test_out_forward.rb

@@ -185,6 +185,81 @@ EOL
     assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
   end
 
+  sub_test_case "certstore loading parameters for Windows" do
+    test 'certstore related config parameters' do
+      omit "certstore related values raise error on not Windows" if Fluent.windows?
+      conf = %[
+        send_timeout 5
+        transport tls
+        tls_cert_logical_store_name Root
+        tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
+
+    test 'cert_logical_store_name and tls_cert_thumbprint default values' do
+      conf = %[
+        send_timeout 5
+        transport tls
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+
+      @d = d = create_driver(conf)
+      assert_nil d.instance.tls_cert_logical_store_name
+      assert_nil d.instance.tls_cert_thumbprint
+    end
+
+    data('CA cert'     => 'tls_ca_cert_path',
+         'non CA cert' => 'tls_cert_path')
+    test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|
+      omit "Loading CertStore feature works only Windows" unless Fluent.windows?
+      dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
+      FileUtils.touch(dummy_cert_path)
+      conf = %[
+        send_timeout 5
+        transport tls
+        #{param} #{dummy_cert_path}
+        tls_cert_logical_store_name Root
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
+
+    test 'configure cert_logical_store_name and tls_cert_thumbprint' do
+      omit "Loading CertStore feature works only Windows" unless Fluent.windows?
+      conf = %[
+        send_timeout 5
+        transport tls
+        tls_cert_logical_store_name Root
+        tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+
+      @d = d = create_driver(conf)
+      assert_equal "Root", d.instance.tls_cert_logical_store_name
+      assert_equal "a909502dd82ae41433e6f83886b00d4277a32a7b", d.instance.tls_cert_thumbprint
+    end
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress

data/test/plugin/test_parser_syslog.rb

@@ -14,8 +14,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
     }
   end
 
-  def test_parse
-    @parser.configure({})
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse(param)
+    @parser.configure({'parser_type' => param})
     @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
@@ -24,8 +25,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
-  def test_parse_with_time_format
-    @parser.configure('time_format' => '%b %d %M:%S:%H')
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_time_format(param)
+    @parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)
     @parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
@@ -33,8 +35,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
   end
 
-  def test_parse_with_priority
-    @parser.configure('with_priority' => true)
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_subsecond_time(param)
+    @parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)
+    @parser.instance.parse('Feb 28 12:00:00.456 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
+      assert_equal(event_time('Feb 28 12:00:00.456', format: '%b %d %H:%M:%S.%N'), time)
+      assert_equal(@expected, record)
+    }
+  end
+
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_priority(param)
+    @parser.configure('with_priority' => true, 'parser_type' => param)
     @parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected.merge('pri' => 6), record)
@@ -43,8 +55,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
-  def test_parse_without_colon
-    @parser.configure({})
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_empty_priority(param)
+    @parser.configure('with_priority' => true, 'parser_type' => param)
+    @parser.instance.parse('<>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
+      assert_nil time
+      assert_nil record
+    }
+  end
+
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_without_colon(param)
+    @parser.configure({'parser_type' => param})
     @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
@@ -53,10 +75,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
-  def test_parse_with_keep_time_key
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_with_keep_time_key(param)
     @parser.configure(
                       'time_format' => '%b %d %M:%S:%H',
                       'keep_time_key'=>'true',
+                      'parser_type' => param
                       )
     text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
     @parser.instance.parse(text) do |time, record|
@@ -64,24 +88,87 @@ class SyslogParserTest < ::Test::Unit::TestCase
     end
   end
 
-  def test_parse_various_characters_for_tag
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_various_characters_for_tag(param)
     ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
-    @parser.configure({})
+    @parser.configure({'parser_type' => param})
     @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected.merge('ident' => ident), record)
     }
   end
 
-  def test_parse_various_characters_for_tag_with_priority
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_various_characters_for_tag_with_priority(param)
     ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
-    @parser.configure('with_priority' => true)
+    @parser.configure('with_priority' => true, 'parser_type' => param)
     @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
       assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
     }
   end
 
+  sub_test_case 'Check the difference of regexp and string parser' do
+    # examples from rfc3164
+    data('regexp' => 'regexp', 'string' => 'string')
+    test 'wrong result with no ident message by default' do |param|
+      @parser.configure('parser_type' => param)
+      @parser.instance.parse('Feb  5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
+        assert_equal({'host' => '10.0.0.99', 'ident' => 'Use', 'message' => 'the BFG!'}, record)
+      }
+    end
+
+    test "proper result with no ident message by 'support_colonless_ident false'" do
+      @parser.configure('parser_type' => 'string', 'support_colonless_ident' => false)
+      @parser.instance.parse('Feb  5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
+        assert_equal({'host' => '10.0.0.99', 'message' => 'Use the BFG!'}, record)
+      }
+    end
+
+    test "string parsers can't parse broken syslog message and generate wrong record" do
+      @parser.configure('parser_type' => 'string')
+      @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record|
+        expected = {'host' => 'scapegoat.dmz.example.org', 'ident' => 'sched', 'pid' => '0', 'message' => "That's All Folks!"}
+        assert_not_equal(expected, record)
+      }
+    end
+
+    test "regexp parsers can't parse broken syslog message and raises an error" do
+      @parser.configure('parser_type' => 'regexp')
+      assert_raise(Fluent::TimeParser::TimeParseError) {
+        @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record| }
+      }
+    end
+
+    data('regexp' => 'regexp', 'string' => 'string')
+    test "':' included message breaks regexp parser" do |param|
+      @parser.configure('parser_type' => param)
+      @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 test foo:bar') { |time, record|
+        expected = {'host' => '127.0.0.1', 'ident' => 'test', 'message' => 'foo:bar'}
+        if param == 'string'
+          assert_equal(expected, record)
+        else
+          assert_not_equal(expected, record)
+        end
+      }
+    end
+
+    data('regexp' => 'regexp', 'string' => 'string')
+    test "Only no whitespace content in MSG causes different result" do |param|
+      @parser.configure('parser_type' => param)
+      @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 value1,value2,value3,value4') { |time, record|
+        # 'message' is correct but regexp set it as 'ident'
+        if param == 'string'
+          expected = {'host' => '127.0.0.1', 'message' => 'value1,value2,value3,value4'}
+          assert_equal(expected, record)
+        else
+          expected = {'host' => '127.0.0.1', 'ident' => 'value1,value2,value3,value4', 'message' => ''}
+          assert_equal(expected, record)
+        end
+      }
+    end
+  end
+
   class TestRFC5424Regexp < self
     def test_parse_with_rfc5424_message
       @parser.configure(
@@ -273,10 +360,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
   end
 
   class TestAutoRegexp < self
-    def test_auto_with_legacy_syslog_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_auto_with_legacy_syslog_message(param)
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
                         'message_format' => 'auto',
+                        'parser_type' => param
                         )
       text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
@@ -286,11 +375,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
     end
 
-    def test_auto_with_legacy_syslog_priority_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_auto_with_legacy_syslog_priority_message(param)
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
                         'with_priority' => true,
                         'message_format' => 'auto',
+                        'parser_type' => param
                         )
       text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
@@ -300,11 +391,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'auto',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -318,11 +411,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
                    @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_rfc5424_structured_message
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_rfc5424_structured_message(param)
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'message_format' => 'auto',
                         'with_priority' => true,
+                        'parser_type' => param
                         )
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
@@ -337,12 +432,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
                    @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_both_message_type
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_both_message_type(param)
       @parser.configure(
         'time_format' => '%b %d %M:%S:%H',
         'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
         'message_format' => 'auto',
         'with_priority' => true,
+        'parser_type' => param
       )
       text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
@@ -382,12 +479,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
                    @parser.instance.patterns['format'])
     end
 
-    def test_parse_with_both_message_type_and_priority
+    data('regexp' => 'regexp', 'string' => 'string')
+    def test_parse_with_both_message_type_and_priority(param)
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
                         'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
                         'with_priority' => true,
                         'message_format' => 'auto',
+                        'parser_type' => param
                         )
       text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-21 00:00:00.000000000 Z
+date: 2019-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0