fluentd 1.5.0.rc1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7ba10a2d9e94456e95d304989cd7487771c3dc71
-  data.tar.gz: ce09baf07ad2c3af4fdea5665c6c57a4387e433a
+SHA256:
+  metadata.gz: 3e334a69c853a549b872123dd1b6232f04dfba60aea75c197b51940ad051e422
+  data.tar.gz: af6be10d887398a5a8efb08915cd5fec348da6fffde98199df39f7e9a9f6e803
 SHA512:
-  metadata.gz: '084c5d106ce8f806f3d52798a7a45498155d5a78c87eefcb6ee0163a79d4cccab42dac9c4e86c90ba644975f06ca0359fc5afa8f65c273f00841312491f7efee'
-  data.tar.gz: 166d9c9a41e3e6b9ae6b3881655be16348ab8d562e4890fe0d27202431a94f2fd37f203b928915b4272dc9da975c2932bb2c449fca69553227eec6f8d18df5db
+  metadata.gz: 6aec4c3726486c0fe5fd9734a33e3813f222578211bf1e71ffa102aebcc14af4ed3a99b789a12f4f38e9a3d95379c90363e8921c9557be03bda756cbfafb76c3
+  data.tar.gz: b161392482f816ad8cc4eaa45a55529ec13107ed8a1020a4e7ef45a18f2949580eab5cf833c1e4462e98118d7fd1f0a41e20537c9a2c57b55f423bb4857fd147

data/.travis.yml

@@ -23,6 +23,8 @@ matrix:
       os: linux
     - rvm: ruby-head
       os: linux
+    - rvm: ruby-head
+      os: linux-ppc64le
     - rvm: 2.1.10
       os: osx
       osx_image: xcode8.3 # OSX 10.12

data/CHANGELOG.md

@@ -1,11 +1,15 @@
 # v1.5
 
-## Pre release
+## Release v1.5.0 - 2019/05/18
 
 ### New feature
 
 * out_forward: Support keepalive feature
   https://github.com/fluent/fluentd/pull/2393
+* in_http: Support TLS via server helper
+  https://github.com/fluent/fluentd/pull/2395
+* in_syslog: Support TLS via server helper
+  https://github.com/fluent/fluentd/pull/2399
 
 ### Enhancement
 
@@ -17,11 +21,17 @@
   https://github.com/fluent/fluentd/pull/2381
 * command: Add deprecated message to show-plugin-config option
   https://github.com/fluent/fluentd/pull/2401
+* storage_local: Ignore empty file. Call sync after write for XFS.
+  https://github.com/fluent/fluentd/pull/2409
 
 ### Bug fixes
 
 * out_forward: Don't use SO_LINGER on SSL/TLS WinSock
   https://github.com/fluent/fluentd/pull/2398
+* server helper: Fix recursive lock issue in TLSServer
+  https://github.com/fluent/fluentd/pull/2341
+* Fix typo
+  https://github.com/fluent/fluentd/pull/2369
 
 # v1.4
 

data/lib/fluent/compat/socket_util.rb

@@ -148,7 +148,7 @@ module Fluent
         def on_message(msg, addr)
           @parser.parse(msg) { |time, record|
             unless time && record
-              log.warn "pattern not match: #{msg.inspect}"
+              log.warn "pattern not matched: #{msg.inspect}"
               return
             end
 

data/lib/fluent/plugin/filter_parser.rb

@@ -82,7 +82,7 @@ module Fluent::Plugin
             return t, r
           else
             if @emit_invalid_record_to_error
-              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
             if @reserve_data
               t = time

data/lib/fluent/plugin/in_http.rb

@@ -36,9 +36,7 @@ module Fluent::Plugin
   class HttpInput < Input
     Fluent::Plugin.register_input('http', self)
 
-    # TODO: update this plugin implementation to use server plugin helper, after adding keepalive feature on it
-
-    helpers :parser, :compat_parameters, :event_loop
+    helpers :parser, :compat_parameters, :event_loop, :server
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
@@ -130,28 +128,15 @@ module Fluent::Plugin
 
       log.debug "listening http", bind: @bind, port: @port
 
-      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
-      if Fluent.windows?
-        socket_manager_path = socket_manager_path.to_i
-      end
-      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
-      lsock = client.listen_tcp(@bind, @port)
-
       @km = KeepaliveManager.new(@keepalive_timeout)
-      @lsock = Coolio::TCPServer.new(
-        lsock, nil, Handler, @km, method(:on_request),
-        @body_size_limit, @format_name, log,
-        @cors_allow_origins
-      )
-      @lsock.listen(@backlog) unless @backlog.nil?
       event_loop_attach(@km)
-      event_loop_attach(@lsock)
 
+      server_create_connection(:in_http, @port, bind: @bind, backlog: @backlog, &method(:on_server_connect))
       @float_time_parser = Fluent::NumericTimeParser.new(:float)
     end
 
     def close
-      @lsock.close
+      server_wait_until_stop
       super
     end
 
@@ -240,6 +225,22 @@ module Fluent::Plugin
 
     private
 
+    def on_server_connect(conn)
+      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
+
+      conn.on(:data) do |data|
+        handler.on_read(data)
+      end
+
+      conn.on(:write_complete) do |_|
+        handler.on_write_complete
+      end
+
+      conn.on(:close) do |_|
+        handler.on_close
+      end
+    end
+
     def parse_params_default(params)
       if msgpack = params['msgpack']
         @parser_msgpack.parse(msgpack) do |_time, record|
@@ -265,11 +266,11 @@ module Fluent::Plugin
       end
     end
 
-    class Handler < Coolio::Socket
+    class Handler
       attr_reader :content_type
 
       def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
-        super(io)
+        @io = io
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
@@ -280,7 +281,8 @@ module Fluent::Plugin
         @idle = 0
         @km.add(self)
 
-        @remote_port, @remote_addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
+        @remote_port, @remote_addr = io.remote_port, io.remote_addr
+        @parser = Http::Parser.new(self)
       end
 
       def step_idle
@@ -291,17 +293,13 @@ module Fluent::Plugin
         @km.delete(self)
       end
 
-      def on_connect
-        @parser = Http::Parser.new(self)
-      end
-
       def on_read(data)
         @idle = 0
         @parser << data
       rescue
         @log.warn "unexpected error", error: $!.to_s
         @log.warn_backtrace
-        close
+        @io.close
       end
 
       def on_message_begin
@@ -477,8 +475,12 @@ module Fluent::Plugin
         end
       end
 
+      def close
+        @io.close
+      end
+
       def on_write_complete
-        close if @next_close
+        @io.close if @next_close
       end
 
       def send_response_and_close(code, header, body)
@@ -499,9 +501,9 @@ module Fluent::Plugin
           data << "#{k}: #{v}\r\n"
         }
         data << "\r\n"
-        write data
+        @io.write(data)
 
-        write body
+        @io.write(body)
       end
 
       def send_response_nobody(code, header)
@@ -510,7 +512,7 @@ module Fluent::Plugin
           data << "#{k}: #{v}\r\n"
         }
         data << "\r\n"
-        write data
+        @io.write(data)
       end
 
       def include_cors_allow_origin

data/lib/fluent/plugin/in_syslog.rb

@@ -74,7 +74,7 @@ module Fluent::Plugin
     desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
-    config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
+    config_param :protocol_type, :enum, list: [:tcp, :udp], default: nil, deprecated: "use transport directive"
     desc 'The message frame type.(traditional, octet_count)'
     config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
 
@@ -107,6 +107,11 @@ module Fluent::Plugin
       config_param :with_priority, :bool, default: true
     end
 
+    # overwrite server plugin to change default to :udp
+    config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
+      config_argument :protocol, :enum, list: [:tcp, :udp, :tls], default: :udp
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
 
@@ -141,12 +146,13 @@ module Fluent::Plugin
     def start
       super
 
-      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
-      case @protocol_type
+      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type || @transport_config.protocol}"
+      case @protocol_type || @transport_config.protocol
       when :udp then start_udp_server
       when :tcp then start_tcp_server
+      when :tls then start_tcp_server(tls: true)
       else
-        raise "BUG: invalid protocol_type value:#{@protocol_type}"
+        raise "BUG: invalid transport value: #{@protocol_type || @transport_config.protocol}"
       end
     end
 
@@ -156,12 +162,12 @@ module Fluent::Plugin
       end
     end
 
-    def start_tcp_server
+    def start_tcp_server(tls: false)
       octet_count_frame = @frame_type == :octet_count
 
       delimiter = octet_count_frame ? " " : @delimiter
       delimiter_size = delimiter.size
-      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
+      server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
         conn.data do |data|
           buffer = conn.buffer
           buffer << data

data/lib/fluent/plugin/in_tail.rb

@@ -427,7 +427,7 @@ module Fluent::Plugin
               record[@path_key] ||= tail_watcher.path unless @path_key.nil?
               es.add(Fluent::EventTime.now, record)
             end
-            log.warn "pattern not match: #{line.inspect}"
+            log.warn "pattern not matched: #{line.inspect}"
           end
         }
       rescue => e

data/lib/fluent/plugin/in_tcp.rb

@@ -71,7 +71,7 @@ module Fluent::Plugin
 
             @parser.parse(msg) do |time, record|
               unless time && record
-                log.warn "pattern not match", message: msg
+                log.warn "pattern not matched", message: msg
                 next
               end
 

data/lib/fluent/plugin/in_udp.rb

@@ -74,7 +74,7 @@ module Fluent::Plugin
         begin
           @parser.parse(data) do |time, record|
             unless time && record
-              log.warn "pattern not match", data: data
+              log.warn "pattern not matched", data: data
               next
             end
 

data/lib/fluent/plugin/storage_local.rb

@@ -87,7 +87,12 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = Yajl::Parser.parse(open(@path, 'r:utf-8'){ |io| io.read })
+              data = open(@path, 'r:utf-8') { |io| io.read }
+              if data.empty?
+                log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
+                return
+              end
+              data = Yajl::Parser.parse(data)
               raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
             rescue => e
               log.error "failed to read data from plugin storage file", path: @path, error: e
@@ -128,7 +133,7 @@ module Fluent
         tmp_path = @path + '.tmp'
         begin
           json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
-          open(tmp_path, 'w:utf-8', @mode){ |io| io.write json_string }
+          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
           File.rename(tmp_path, @path)
         rescue => e
           log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e

data/lib/fluent/plugin_helper/server.rb

@@ -751,8 +751,15 @@ module Fluent
                 # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
                 written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
                 @_handler_write_buffer.slice!(0, written_bytes)
-                super
               end
+
+              # No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.
+              # Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.
+              # Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.
+              # If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete
+              # eventually calls TLSServer#close which try to get a lock.
+              super
+
               close if @close_after_write_complete
             rescue IO::WaitWritable, IO::WaitReadable
               return

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.5.0.rc1'
+  VERSION = '1.5.0'
 
 end

data/test/plugin/test_filter_parser.rb

@@ -680,7 +680,7 @@ class ParserFilterTest < Test::Unit::TestCase
 
     def test_call_emit_error_event_when_pattern_is_mismached
       flexmock(@d.instance.router).should_receive(:emit_error_event).
-        with(String, Integer, Hash, ParserError.new("pattern not match with data '#{INVALID_MESSAGE}'")).once
+        with(String, Integer, Hash, ParserError.new("pattern not matched with data '#{INVALID_MESSAGE}'")).once
       @d.run do
         @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
       end

data/test/plugin/test_in_syslog.rb

@@ -55,6 +55,35 @@ EOS
     end
   end
 
+  data('Use protocol_type' => ['protocol_type tcp', :tcp, :udp],
+       'Use transport' => ["<transport tcp>\n </transport>", nil, :tcp],
+       'Use transport and protocol' => ["protocol_type udp\n<transport tcp>\n </transport>", :udp, :tcp])
+  def test_configure_protocol(param)
+    conf, proto_type, transport_proto_type = *param
+    d = create_driver([CONFIG, conf].join("\n"))
+
+    assert_equal(d.instance.protocol_type, proto_type)
+    assert_equal(d.instance.transport_config.protocol, transport_proto_type)
+  end
+
+  # For backward compat
+  def test_respect_protocol_type_than_transport
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>", "protocol_type udp"].join("\n"))
+    tests = create_test_case
+
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
+  end
+
+
   data(
     ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
     ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
@@ -119,7 +148,7 @@ EOS
   end
 
   def test_msg_size_with_tcp
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
     d.run(expect_emits: 2) do
@@ -135,7 +164,7 @@ EOS
   end
 
   def test_msg_size_with_same_tcp_connection
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
     d.run(expect_emits: 2) do
@@ -289,7 +318,7 @@ EOS
 
   sub_test_case 'octet counting frame' do
     def test_msg_size_with_tcp
-      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
       tests = create_test_case
 
       d.run(expect_emits: 2) do
@@ -305,7 +334,7 @@ EOS
     end
 
     def test_msg_size_with_same_tcp_connection
-      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
       tests = create_test_case
 
       d.run(expect_emits: 2) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.5.0.rc1
+  version: 1.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -772,12 +772,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector