fluentd 1.4.2 → 1.5.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8855d66a886343a3865b3f224936aa433822027
-  data.tar.gz: 96d848425ac12776baf47bb2d999127697a6e7b1
+  metadata.gz: 7ba10a2d9e94456e95d304989cd7487771c3dc71
+  data.tar.gz: ce09baf07ad2c3af4fdea5665c6c57a4387e433a
 SHA512:
-  metadata.gz: f9e0ab8e68c24424f1eb75e459211b8195fb4a6ee1b2be99ef1a88da3d5e3e643b5243cadb87f9e68aa0f23eedad0d1c73aa9a53e72f75adfd52d7709dfe47af
-  data.tar.gz: 11e5d4293430e2675c70e323111044599be4de9bbf1bcb22045f7514f90788011949ddd27a07ba66b2c9b1ac2ee319d948b0f29113ccbd098ef4b5eb1f7cdebe
+  metadata.gz: '084c5d106ce8f806f3d52798a7a45498155d5a78c87eefcb6ee0163a79d4cccab42dac9c4e86c90ba644975f06ca0359fc5afa8f65c273f00841312491f7efee'
+  data.tar.gz: 166d9c9a41e3e6b9ae6b3881655be16348ab8d562e4890fe0d27202431a94f2fd37f203b928915b4272dc9da975c2932bb2c449fca69553227eec6f8d18df5db

data/.travis.yml

@@ -11,11 +11,15 @@ matrix:
       os: linux
     - rvm: 2.2.10
       os: linux
-    - rvm: 2.4.5
+    - rvm: 2.2.10
+      os: linux-ppc64le
+    - rvm: 2.4.6
       os: linux
-    - rvm: 2.5.3
+    - rvm: 2.4.6
+      os: linux-ppc64le
+    - rvm: 2.5.5
       os: linux
-    - rvm: 2.6.0
+    - rvm: 2.6.3
       os: linux
     - rvm: ruby-head
       os: linux
@@ -28,7 +32,7 @@ matrix:
     # - rvm: 2.3.3
     #   os: osx
     #   osx_image: xcode8.2 # OSX 10.12
-    - rvm: 2.4.1
+    - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3 # OSX 10.12
     - rvm: ruby-head
@@ -40,7 +44,7 @@ matrix:
     - rvm: 2.1.10
       os: osx
       osx_image: xcode8.3
-    - rvm: 2.4.1
+    - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
     - rvm: ruby-head

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+# v1.5
+
+## Pre release
+
+### New feature
+
+* out_forward: Support keepalive feature
+  https://github.com/fluent/fluentd/pull/2393
+
+### Enhancement
+
+* in_syslog: Add delimiter parameter
+  https://github.com/fluent/fluentd/pull/2378
+* in_forward: Add tag/add_tag_prefix parameters
+  https://github.com/fluent/fluentd/pull/2396
+* parser_json: Add stream_buffer_size parameter for yajl
+  https://github.com/fluent/fluentd/pull/2381
+* command: Add deprecated message to show-plugin-config option
+  https://github.com/fluent/fluentd/pull/2401
+
+### Bug fixes
+
+* out_forward: Don't use SO_LINGER on SSL/TLS WinSock
+  https://github.com/fluent/fluentd/pull/2398
+
 # v1.4
 
 ## Release v1.4.2 - 2019/04/02

data/README.md

@@ -30,6 +30,11 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 
 ## Development
 
+### Branch
+
+- master: For v1 development.
+- v0.12: For v0.12. This is security maintenance mode. Only security fix is accepted.
+
 ### Prerequisites
 
 - Ruby 2.1 or later

data/example/in_http.conf

@@ -6,7 +6,9 @@
   keepalive_timeout 10
   # backlog 0
   add_http_headers false
-  format default
+  <parse>
+    @type json
+  </parse>
 </source>
 
 <match test>

data/lib/fluent/command/fluentd.rb

@@ -40,7 +40,7 @@ op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
   opts[:dry_run] = b
 }
 
-op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
   opts[:show_plugin_config] = plugin
 }
 

data/lib/fluent/log.rb

@@ -148,10 +148,10 @@ module Fluent
     end
 
     attr_reader :format
+    attr_reader :time_format
     attr_accessor :log_event_enabled
     attr_accessor :out
     attr_accessor :level
-    attr_accessor :time_format
     attr_accessor :optional_header, :optional_attrs
 
     def logdev=(logdev)

data/lib/fluent/plugin/in_forward.rb

@@ -62,6 +62,11 @@ module Fluent::Plugin
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
 
+    desc "New tag instead of incoming tag"
+    config_param :tag, :string, default: nil
+    desc "Add prefix to incoming tag"
+    config_param :add_tag_prefix, :string, default: nil
+
     config_section :security, required: false, multi: false do
       desc 'The hostname'
       config_param :self_hostname, :string
@@ -106,6 +111,9 @@ module Fluent::Plugin
       end
       @enable_field_injection = @source_address_key || @source_hostname_key
 
+      raise Fluent::ConfigError, "'tag' parameter must not be empty" if @tag && @tag.empty?
+      raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
+
       if @security
         if @security.user_auth && @security.users.empty?
           raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
@@ -293,6 +301,9 @@ module Fluent::Plugin
         log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
       end
 
+      tag = @tag.dup if @tag
+      tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
+
       case entries
       when String
         # PackedForward

data/lib/fluent/plugin/in_syslog.rb

@@ -99,6 +99,9 @@ module Fluent::Plugin
 
     config_param :blocking_timeout, :time, default: 0.5
 
+    desc 'The delimiter value "\n"'
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message
+
     config_section :parse do
       config_set_default :@type, DEFAULT_PARSER
       config_param :with_priority, :bool, default: true
@@ -156,8 +159,7 @@ module Fluent::Plugin
     def start_tcp_server
       octet_count_frame = @frame_type == :octet_count
 
-      # syslog family adds "\n" to each message when transport is TCP and traditional frame
-      delimiter = octet_count_frame ? " " : "\n"
+      delimiter = octet_count_frame ? " " : @delimiter
       delimiter_size = delimiter.size
       server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
         conn.data do |data|

data/lib/fluent/plugin/out_exec_filter.rb

@@ -95,6 +95,7 @@ module Fluent::Plugin
     COMPAT_PARSE_PARAMS = {
       'out_format' => '@type',
       'out_keys' => 'keys',
+      'out_stream_buffer_size' => 'stream_buffer_size',
     }
     COMPAT_EXTRACT_PARAMS = {
       'out_tag_key' => 'tag_key',

data/lib/fluent/plugin/out_forward.rb

@@ -36,7 +36,6 @@ module Fluent::Plugin
     desc 'The transport protocol.'
     config_param :transport, :enum, list: [:tcp, :tls], default: :tcp
     # TODO: TLS session cache/tickets
-    # TODO: Connection keepalive
 
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
@@ -103,6 +102,10 @@ module Fluent::Plugin
     config_param :tls_client_private_key_path, :string, default: nil
     desc 'The client private key passphrase for TLS.'
     config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true
+    desc "Enable keepalive connection."
+    config_param :keepalive, :bool, default: false
+    desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
+    config_param :keepalive_timeout, :time, default: nil
 
     config_section :security, required: false, multi: false do
       desc 'The hostname'
@@ -151,6 +154,7 @@ module Fluent::Plugin
       @usock = nil
       @sock_ack_waiting = nil
       @sock_ack_waiting_mutex = nil
+      @keep_alive_watcher_interval = 5 # TODO
     end
 
     def configure(conf)
@@ -201,9 +205,9 @@ module Fluent::Plugin
 
         log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id
         if @heartbeat_type == :none
-          @nodes << NoneHeartbeatNode.new(self, server, failure: failure)
+          @nodes << NoneHeartbeatNode.new(self, server, failure: failure, keepalive: @keepalive, keepalive_timeout: @keepalive_timeout)
         else
-          node = Node.new(self, server, failure: failure)
+          node = Node.new(self, server, failure: failure, keepalive: @keepalive, keepalive_timeout: @keepalive_timeout)
           begin
             node.validate_host_resolution!
           rescue => e
@@ -227,6 +231,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
       end
 
+      if !@keepalive && @keepalive_timeout
+        log.warn('The value of keepalive_timeout is ignored. if you want to use keepalive, please add `keepalive true` to your conf.')
+      end
+
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
     end
 
@@ -279,6 +287,10 @@ module Fluent::Plugin
           end
         end
       end
+
+      if @keepalive && @keepalive_timeout
+        timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))
+      end
     end
 
     def close
@@ -286,6 +298,10 @@ module Fluent::Plugin
         # close socket and ignore errors: this socket will not be used anyway.
         @usock.close rescue nil
       end
+
+      if @keepalive && @keepalive_timeout
+        @nodes.each(&:clear)
+      end
       super
     end
 
@@ -354,7 +370,10 @@ module Fluent::Plugin
           cert_path: @tls_client_cert_path,
           private_key_path: @tls_client_private_key_path,
           private_key_passphrase: @tls_client_private_key_passphrase,
-          linger_timeout: @send_timeout,
+
+          # Enabling SO_LINGER causes data loss on Windows
+          # https://github.com/fluent/fluentd/issues/1968
+          linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
           &block
@@ -450,6 +469,10 @@ module Fluent::Plugin
       end
     end
 
+    def on_purge_obsolete_socks
+      @nodes.each(&:purge_obsolete_socks)
+    end
+
     # return chunk id to be committed
     def read_ack_from_sock(sock, unpacker)
       begin
@@ -484,8 +507,13 @@ module Fluent::Plugin
       log.error "unexpected error while receiving ack message", error: e
       log.error_backtrace
     ensure
-      info.sock.close_write rescue nil
-      info.sock.close rescue nil
+      if @keepalive
+        info.node.socket_cache.dec_ref_by_value(info.sock)
+      else
+        info.sock.close_write rescue nil
+        info.sock.close rescue nil
+      end
+
       @sock_ack_waiting_mutex.synchronize do
         @sock_ack_waiting.delete(info)
       end
@@ -513,6 +541,9 @@ module Fluent::Plugin
                 # (2) the node does support sending response but responses have not arrived for some reasons.
                 log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
                 info.node.disable!
+                if @keepalive
+                  info.node.socket_cache.revoke_by_value(info.sock)
+                end
                 info.sock.close rescue nil
                 rollback_write(info.chunk_id, update_retry: false)
               else
@@ -538,7 +569,142 @@ module Fluent::Plugin
     end
 
     class Node
-      def initialize(sender, server, failure:)
+      class SocketCache
+        TimedSocket = Struct.new(:timeout, :sock, :ref)
+
+        def initialize(timeout, log)
+          @log = log
+          @timeout = timeout
+          @active_socks = {}
+          @inactive_socks = {}
+          @mutex = Mutex.new
+        end
+
+        def revoke(key = Thread.current.object_id)
+          @mutex.synchronize do
+            if @active_socks[key]
+              @inactive_socks[key] = @active_socks.delete(key)
+              @inactive_socks[key].ref = 0
+            end
+          end
+        end
+
+        def clear
+          @mutex.synchronize do
+            @inactive_socks.values.each do |s|
+              s.sock.close rescue nil
+            end
+            @inactive_socks.clear
+
+            @active_socks.values.each do |s|
+              s.sock.close rescue nil
+            end
+            @active_socks.clear
+          end
+        end
+
+        def purge_obsolete_socks
+          @mutex.synchronize do
+            @inactive_socks.keys.each do |k|
+              # 0 means sockets stored in this class received all acks
+              if @inactive_socks[k].ref <= 0
+                s = @inactive_socks.delete(k)
+                s.sock.close  rescue nil
+                @log.debug("purged obsolete socket #{s.sock}")
+              end
+            end
+
+            @active_socks.keys.each do |k|
+              if expired?(k) && @active_socks[k].ref <= 0
+                @inactive_socks[k] = @active_socks.delete(k)
+              end
+            end
+          end
+        end
+
+        # We expect that `yield` returns a unique object in this class
+        def fetch_or(key = Thread.current.object_id)
+          @mutex.synchronize do
+            unless @active_socks[key]
+              @active_socks[key] = TimedSocket.new(timeout, yield, 1)
+              @log.debug("connect new socket #{@active_socks[key]}")
+              return @active_socks[key].sock
+            end
+
+            if expired?(key)
+              # Do not close this socket here in case of it will be used by other place (e.g. wait for receiving ack)
+              @inactive_socks[key] = @active_socks.delete(key)
+              @log.debug("connection #{@inactive_socks[key]} is expired. reconnecting...")
+              @active_socks[key] = TimedSocket.new(timeout, yield, 0)
+            end
+
+            @active_socks[key].ref += 1;
+            @active_socks[key].sock
+          end
+        end
+
+        def dec_ref(key = Thread.current.object_id)
+          @mutex.synchronize do
+            if @active_socks[key]
+              @active_socks[key].ref -= 1
+            elsif @inactive_socks[key]
+              @inactive_socks[key].ref -= 1
+            else
+              @log.warn("Not found key for dec_ref: #{key}")
+            end
+          end
+        end
+
+        # This method is expected to be called in class which doesn't call #inc_ref
+        def dec_ref_by_value(val)
+          @mutex.synchronize do
+            sock = @active_socks.detect { |_, v| v.sock == val }
+            if sock
+              key = sock.first
+              @active_socks[key].ref -= 1
+              return
+            end
+
+            sock = @inactive_socks.detect { |_, v| v.sock == val }
+            if sock
+              key = sock.first
+              @inactive_socks[key].ref -= 1
+              return
+            else
+              @log.warn("Not found key for dec_ref_by_value: #{key}")
+            end
+          end
+        end
+
+        # This method is expected to be called in class which doesn't call #fetch_or
+        def revoke_by_value(val)
+          @mutex.synchronize do
+            sock = @active_socks.detect { |_, v| v.sock == val }
+            if sock
+              key = sock.first
+              @inactive_socks[key] = @active_socks.delete(key)
+              @inactive_socks[key].ref = 0
+            else
+              @log.debug("Not found for revoke_by_value :#{val}")
+            end
+          end
+        end
+
+        private
+
+        def timeout
+          @timeout && Time.now + @timeout
+        end
+
+        # This method is thread unsafe
+        def expired?(key = Thread.current.object_id)
+           @active_socks[key].timeout ? @active_socks[key].timeout < Time.now : false
+        end
+      end
+
+      # @param keepalive [Bool]
+      # @param keepalive_timeout [Integer | nil]
+      def initialize(sender, server, failure:, keepalive: false, keepalive_timeout: nil)
         @sender = sender
         @log = sender.log
         @compress = sender.compress
@@ -571,6 +737,11 @@ module Fluent::Plugin
         @resolved_host = nil
         @resolved_time = 0
         @resolved_once = false
+
+        @keepalive = keepalive
+        if @keepalive
+          @socket_cache = SocketCache.new(keepalive_timeout, @log)
+        end
       end
 
       attr_accessor :usock
@@ -578,6 +749,7 @@ module Fluent::Plugin
       attr_reader :name, :host, :port, :weight, :standby, :state
       attr_reader :sockaddr  # used by on_heartbeat
       attr_reader :failure, :available # for test
+      attr_reader :socket_cache        # for ack
 
       RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
 
@@ -598,15 +770,12 @@ module Fluent::Plugin
       end
 
       def verify_connection
-        sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
-        begin
+        connect do |sock|
           ri = RequestInfo.new(@sender.security ? :helo : :established)
           if ri.state != :established
             establish_connection(sock, ri)
             raise if ri.state != :established
           end
-        ensure
-          sock.close
         end
       end
 
@@ -672,11 +841,16 @@ module Fluent::Plugin
       end
 
       def send_data(tag, chunk)
-        sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
+        sock = connect
+
         begin
           send_data_actual(sock, tag, chunk)
         rescue
-          sock.close rescue nil
+          if @keepalive
+            @socket_cache.revoke
+          else
+            sock.close rescue nil
+          end
           raise
         end
 
@@ -684,12 +858,27 @@ module Fluent::Plugin
           return sock # to read ACK from socket
         end
 
-        sock.close_write rescue nil
-        sock.close rescue nil
+        if @keepalive
+          @socket_cache.dec_ref
+        else
+          sock.close_write rescue nil
+          sock.close rescue nil
+        end
         heartbeat(false)
         nil
       end
 
+      def clear
+        @keepalive && @socket_cache.clear
+      end
+
+      def purge_obsolete_socks
+        unless @keepalive
+          raise "Don not call this method without keepalive option"
+        end
+        @socket_cache.purge_obsolete_socks
+      end
+
       # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
       def send_heartbeat
         begin
@@ -705,7 +894,7 @@ module Fluent::Plugin
 
         case @sender.heartbeat_type
         when :transport
-          @sender.create_transfer_socket(dest_addr, port, @hostname) do |sock|
+          connect(dest_addr) do |sock|
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
@@ -880,6 +1069,32 @@ module Fluent::Plugin
           raise "BUG: unknown session state: #{ri.state}"
         end
       end
+
+      private
+
+      def connect(host = nil)
+        sock = if @keepalive
+                 @socket_cache.fetch_or { @sender.create_transfer_socket(host || resolved_host, port, @hostname) }
+               else
+                 @log.debug('connect new socket')
+                 @sender.create_transfer_socket(host || resolved_host, port, @hostname)
+               end
+
+        if block_given?
+          begin
+            yield(sock)
+          rescue
+            @socket_cache.revoke(sock) if @keepalive
+            raise
+          else
+            @socket_cache.dec_ref(sock) if @keepalive
+          ensure
+            sock.close unless @keepalive
+          end
+        else
+          sock
+        end
+      end
     end
 
     # Override Node to disable heartbeat

data/lib/fluent/plugin/parser_json.rb

@@ -30,6 +30,12 @@ module Fluent
       desc 'Set JSON parser'
       config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
+      # The Yajl library defines a default buffer size of 8KiB when parsing
+      # from IO streams, so maintain this for backwards-compatibility.
+      # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
+      desc 'Set the buffer size that Yajl will use when parsing streaming input'
+      config_param :stream_buffer_size, :integer, default: 8192
+
       config_set_default :time_type, :float
 
       def configure(conf)
@@ -81,7 +87,7 @@ module Fluent
         y.on_parse_complete = ->(record){
           block.call(parse_time(record), record)
         }
-        y.parse(io)
+        y.parse(io, @stream_buffer_size)
       end
     end
   end

data/lib/fluent/supervisor.rb

@@ -588,7 +588,7 @@ module Fluent
 
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
-      $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
+      $log.info "show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}"
       exit 0
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.4.2'
+  VERSION = '1.5.0.rc1'
 
 end

data/test/plugin/test_in_forward.rb

@@ -82,6 +82,14 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal 1, d.instance.security.clients.size
     end
 
+    data(tag: "tag",
+         add_tag_prefix: "add_tag_prefix")
+    test 'tag parameters' do |data|
+      assert_raise(Fluent::ConfigError.new("'#{data}' parameter must not be empty")) {
+        create_driver(CONFIG + "#{data} ''")
+      }
+    end
+
     test 'send_keepalive_packet is disabled by default' do
       @d = d = create_driver(CONFIG_AUTH)
       assert_false d.instance.send_keepalive_packet
@@ -269,6 +277,34 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal(records, d.events)
     end
 
+    data(tag: {
+           param: "tag new_tag",
+           result: "new_tag"
+         },
+         add_tag_prefix: {
+           param: "add_tag_prefix new_prefix",
+           result: "new_prefix.tag1"
+         })
+    test 'tag parameters' do |data|
+      @d = create_driver(CONFIG + data[:param])
+      time = event_time("2011-01-02 13:14:15 UTC")
+      options = {auth: false}
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      @d.run(expect_records: records.length, timeout: 20) do
+        entries = []
+        records.each {|tag, _time, record|
+          entries << [_time, record]
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(data[:result], @d.events[0][0])
+    end
+
     data(tcp: {
            config: CONFIG,
            options: {
@@ -378,6 +414,34 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal(records, d.events)
     end
 
+    data(tag: {
+           param: "tag new_tag",
+           result: "new_tag"
+         },
+         add_tag_prefix: {
+           param: "add_tag_prefix new_prefix",
+           result: "new_prefix.tag1"
+         })
+    test 'tag parameters' do |data|
+      @d = create_driver(CONFIG + data[:param])
+      time = event_time("2011-01-02 13:14:15 UTC")
+      options = {auth: false}
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      @d.run(expect_records: records.length, timeout: 20) do
+        entries = ''
+        records.each {|_tag, _time, record|
+          packer(entries).write([_time, record]).flush
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(data[:result], @d.events[0][0])
+    end
+
     data(tcp: {
            config: CONFIG,
            options: {

data/test/plugin/test_out_exec_filter.rb

@@ -328,6 +328,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -338,6 +339,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     tag_key tag
   ]
@@ -372,6 +374,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -382,6 +385,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     tag_key tag
   ]
@@ -414,6 +418,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -426,6 +431,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     time_format %d/%b/%Y %H:%M:%S.%N %z
     tag_key tag

data/test/plugin/test_out_forward.rb

@@ -920,4 +920,213 @@ EOL
       assert_equal(['test', time, records[1]], events[1])
     end
   end
+
+  test 'Create new connection per send_data' do
+    target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+    output_conf = CONFIG
+    d = create_driver(output_conf)
+    d.instance_start
+
+    begin
+      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+      mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.twice
+
+      target_input_driver.run(timeout: 15) do
+        d.run(shutdown: false) do
+          node = d.instance.nodes.first
+          2.times do
+            node.send_data('test', chunk) rescue nil
+          end
+        end
+      end
+    ensure
+      d.instance_shutdown
+    end
+  end
+
+  sub_test_case 'keepalive' do
+    test 'Do not create connection per send_data' do
+      target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+      output_conf = CONFIG + %[
+        keepalive true
+        keepalive_timeout 2
+      ]
+      d = create_driver(output_conf)
+      d.instance_start
+
+      begin
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
+
+        target_input_driver.run(timeout: 15) do
+          d.run(shutdown: false) do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
+            end
+          end
+        end
+      ensure
+        d.instance_shutdown
+      end
+    end
+
+    sub_test_case 'with require_ack_response' do
+      test 'Do not create connection per send_data' do
+        target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+        output_conf = CONFIG + %[
+          require_ack_response true
+          keepalive true
+          keepalive_timeout 2
+        ]
+        d = create_driver(output_conf)
+        d.instance_start
+
+        begin
+          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+          mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
+
+          target_input_driver.run(timeout: 15) do
+            d.run(shutdown: false) do
+              node = d.instance.nodes.first
+              2.times do
+                node.send_data('test', chunk) rescue nil
+              end
+            end
+          end
+        ensure
+          d.instance_shutdown
+        end
+      end
+    end
+  end
+
+  sub_test_case 'SocketCache' do
+    sub_test_case 'fetch_or' do
+      test 'when gived key does not exist' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        sock = mock!.open { 1 }.subject
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+
+      test 'when given key exists' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        assert_equal(1, c.fetch_or { 1 })
+
+        sock = dont_allow(mock!).open
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+
+      test "when given key's value was expired" do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(0, Logger.new(nil))
+        assert_equal(1, c.fetch_or { 1 })
+
+        sock = mock!.open { 1 }.subject
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+    end
+
+    test 'revoke' do
+      c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+      c.fetch_or { 1 }
+      c.revoke
+
+      sock = mock!.open { 1 }.subject
+      assert_equal(1, c.fetch_or { sock.open })
+    end
+
+    test 'revoke_by_value' do
+      c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+      c.fetch_or { 1 }
+      c.revoke_by_value(1)
+
+      sock = mock!.open { 1 }.subject
+      assert_equal(1, c.fetch_or { sock.open })
+    end
+
+    sub_test_case 'dec_ref' do
+      test 'when value exists in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.dec_ref
+
+        assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
+      end
+
+      test 'when value exists in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.revoke
+        c.dec_ref
+        assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
+      end
+    end
+
+    sub_test_case 'dec_ref_by_value' do
+      test 'when value exists in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.dec_ref_by_value(1)
+
+        assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
+      end
+
+      test 'when value exists in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.revoke
+        c.dec_ref_by_value(1)
+        assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
+      end
+    end
+
+    sub_test_case 'clear' do
+      test 'when value is in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+
+        c.clear
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+
+      test 'when value is in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        c.revoke
+        assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.clear
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+    end
+
+    sub_test_case 'purge_obsolete_socks' do
+      test 'delete key in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        c.revoke
+        assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.purge_obsolete_socks
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+
+      test 'move key from active_socks to inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = dont_allow(mock!).close
+        stub(m).inspect         # for log
+        c.fetch_or { m }
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+        assert_true(c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.purge_obsolete_socks
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+        assert_true(c.instance_variable_get(:@inactive_socks).empty?)
+      end
+    end
+  end
 end

data/test/plugin/test_parser_json.rb

@@ -111,4 +111,28 @@ class JsonParserTest < ::Test::Unit::TestCase
       assert_equal text, record['time']
     end
   end
+
+  def test_yajl_parse_io_with_buffer_smaller_than_input
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+    parser.configure(
+                     'keep_time_key' => 'true',
+                     'json_parser' => 'yajl',
+                     'stream_buffer_size' => 1,
+                     )
+    text = "100"
+
+    waiting(5) do
+      rd, wr = IO.pipe
+      wr.write "{\"time\":\"#{text}\"}"
+
+      parser.instance.parse_io(rd) do |time, record|
+        assert_equal text.to_i, time.sec
+        assert_equal text, record['time']
+
+        # Once a record has been received the 'write' end of the pipe must be
+        # closed, otherwise the test will block waiting for more input.
+        wr.close
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-02 00:00:00.000000000 Z
+date: 2019-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -772,9 +772,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.14.1