fluentd 1.16.5-x86-mingw32 → 1.17.1-x86-mingw32

data/lib/fluent/plugin/in_sample.rb

@@ -39,6 +39,8 @@ module Fluent::Plugin
     config_param :auto_increment_key, :string, default: nil
     desc "The boolean to suspend-and-resume incremental value after restart"
     config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
+    desc "Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change"
+    config_param :reuse_record, :bool, default: false
     desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
       begin
@@ -117,15 +119,19 @@ module Fluent::Plugin
       end
     end
 
-    def generate
-      d = @sample[@sample_index]
-      unless d
-        @sample_index = 0
-        d = @sample[@sample_index]
-      end
+    def next_sample
+      d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup
       @sample_index += 1
+      return d if d
+
+      @sample_index = 0
+      next_sample
+    end
+
+    def generate
+      d = next_sample
       if @auto_increment_key
-        d = d.dup
+        d = d.dup if @reuse_record
         d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
       end
       d

data/lib/fluent/plugin/in_tail.rb

@@ -36,7 +36,7 @@ module Fluent::Plugin
     helpers :timer, :event_loop, :parser, :compat_parameters
 
     RESERVED_CHARS = ['/', '*', '%'].freeze
-    MetricsInfo = Struct.new(:opened, :closed, :rotated)
+    MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled)
 
     class WatcherSetupError < StandardError
       def initialize(msg)
@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -141,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -198,7 +208,8 @@ module Fluent::Plugin
       opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
       closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
       rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
-      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled")
+      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)
     end
 
     def configure_tag
@@ -288,6 +299,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -297,7 +330,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -332,7 +365,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}
@@ -534,7 +567,7 @@ module Fluent::Plugin
       if @follow_inodes && new_inode.nil?
         # nil inode means the file disappeared, so we only need to stop it.
         @tails.delete(tail_watcher.path)
-        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
         # Because of this problem, log duplication can occur during `rotate_wait`.
         # Need to set `rotate_wait 0` for a workaround.
         # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
@@ -664,14 +697,6 @@ module Fluent::Plugin
 
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
-      lines = lines.reject do |line|
-        skip_line = @max_line_size ? line.bytesize > @max_line_size : false
-        if skip_line
-          log.warn "received line length is longer than #{@max_line_size}"
-          log.debug "skipped line: #{line.chomp}"
-        end
-        skip_line
-      end
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
         tag = if @tag_prefix || @tag_suffix
@@ -769,6 +794,7 @@ module Fluent::Plugin
           'opened_file_count' => @metrics.opened.get,
           'closed_file_count' => @metrics.closed.get,
           'rotated_file_count' => @metrics.rotated.get,
+          'throttled_log_count' => @metrics.throttled.get,
         })
       }
       stats
@@ -787,6 +813,7 @@ module Fluent::Plugin
         from_encoding: @from_encoding,
         encoding: @encoding,
         metrics: @metrics,
+        max_line_size: @max_line_size,
         &method(:receive_lines)
       )
     end
@@ -979,15 +1006,19 @@ module Fluent::Plugin
       end
 
       class FIFO
-        def initialize(from_encoding, encoding)
+        def initialize(from_encoding, encoding, log, max_line_size=nil)
           @from_encoding = from_encoding
           @encoding = encoding
           @need_enc = from_encoding != encoding
           @buffer = ''.force_encoding(from_encoding)
           @eol = "\n".encode(from_encoding).freeze
+          @max_line_size = max_line_size
+          @skip_current_line = false
+          @skipping_current_line_bytesize = 0
+          @log = log
         end
 
-        attr_reader :from_encoding, :encoding, :buffer
+        attr_reader :from_encoding, :encoding, :buffer, :max_line_size
 
         def <<(chunk)
           # Although "chunk" is most likely transient besides String#force_encoding itself
@@ -1019,6 +1050,7 @@ module Fluent::Plugin
 
         def read_lines(lines)
           idx = @buffer.index(@eol)
+          has_skipped_line = false
 
           until idx.nil?
             # Using freeze and slice is faster than slice!
@@ -1027,11 +1059,47 @@ module Fluent::Plugin
             rbuf = @buffer.slice(0, idx + 1)
             @buffer = @buffer.slice(idx + 1, @buffer.size)
             idx = @buffer.index(@eol)
+
+            is_long_line = @max_line_size && (
+              @skip_current_line || rbuf.bytesize > @max_line_size
+            )
+
+            if is_long_line
+              @log.warn "received line length is longer than #{@max_line_size}"
+              if @skip_current_line
+                @log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
+              else
+                @log.debug("skipped line: ") { convert(rbuf).chomp }
+              end
+              has_skipped_line = true
+              @skip_current_line = false
+              @skipping_current_line_bytesize = 0
+              next
+            end
+
             lines << convert(rbuf)
           end
+
+          is_long_current_line = @max_line_size && (
+            @skip_current_line || @buffer.bytesize > @max_line_size
+          )
+
+          if is_long_current_line
+            @log.debug(
+              "The continuing current line length is longer than #{@max_line_size}." +
+              " The received data will be discarded until this line is finished." +
+              " Discarded data: "
+            ) { convert(@buffer).chomp }
+            @skip_current_line = true
+            @skipping_current_line_bytesize += @buffer.bytesize
+            @buffer.clear
+          end
+
+          return has_skipped_line
         end
 
-        def bytesize
+        def reading_bytesize
+          return @skipping_current_line_bytesize if @skip_current_line
           @buffer.bytesize
         end
       end
@@ -1042,14 +1110,14 @@ module Fluent::Plugin
 
         attr_accessor :shutdown_timeout
 
-        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
           @read_bytes_limit_per_second = read_bytes_limit_per_second
           @receive_lines = receive_lines
           @open_on_every_update = open_on_every_update
-          @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
+          @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT, log, max_line_size)
           @iobuf = ''.force_encoding('ASCII-8BIT')
           @lines = []
           @io = nil
@@ -1126,12 +1194,15 @@ module Fluent::Plugin
         end
 
         def handle_notify
-          return if limit_bytes_per_second_reached?
-          return if group_watcher&.limit_lines_reached?(@path)
+          if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
+            @metrics.throttled.inc
+            return
+          end
 
           with_io do |io|
             begin
               read_more = false
+              has_skipped_line = false
 
               if !io.nil? && @lines.empty?
                 begin
@@ -1145,7 +1216,7 @@ module Fluent::Plugin
                     @fifo << data
 
                     n_lines_before_read = @lines.size
-                    @fifo.read_lines(@lines)
+                    has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
                     group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
 
                     group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
@@ -1153,6 +1224,7 @@ module Fluent::Plugin
 
                     if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
                       # Just get out from tailing loop.
+                      @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
                       read_more = false
                       break
                     end
@@ -1168,9 +1240,11 @@ module Fluent::Plugin
                 end
               end
 
-              unless @lines.empty?
+              if @lines.empty?
+                @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
+              else
                 if @receive_lines.call(@lines, @watcher)
-                  @watcher.pe.update_pos(io.pos - @fifo.bytesize)
+                  @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
                   @lines.clear
                 else
                   read_more = false
@@ -1182,12 +1256,12 @@ module Fluent::Plugin
 
         def open
           io = Fluent::FileWrapper.open(@path)
-          io.seek(@watcher.pe.read_pos + @fifo.bytesize)
+          io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
           @metrics.opened.inc
           io
         rescue RangeError
           io.close if io
-          raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
+          raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
         rescue Errno::EACCES => e
           @log.warn "#{e}"
           nil

data/lib/fluent/plugin/out_copy.rb

@@ -89,7 +89,7 @@ module Fluent::Plugin
 
       case @copy_mode
       when :no_copy
-         nil
+        nil
       when :shallow
         Proc.new { |es| es.dup }
       when :deep

data/lib/fluent/plugin/out_file.rb

@@ -172,6 +172,14 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
+          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
+            begin
+              v.validate!
+            rescue Fluent::ConfigError => e
+              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
+            end
+          end
+
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
 
     class RetryableResponse < StandardError; end
 
+    ConnectionCache = Struct.new(:uri, :conn)
+
     helpers :formatter
 
     desc 'The endpoint for HTTP request, e.g. http://example.com/api'
@@ -53,6 +55,8 @@ module Fluent::Plugin
     config_param :headers, :hash, default: nil
     desc 'Additional placeholder based headers for HTTP request'
     config_param :headers_from_placeholders, :hash, default: nil
+    desc 'Compress HTTP request body'
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
 
     desc 'The connection open timeout in seconds'
     config_param :open_timeout, :integer, default: nil
@@ -60,6 +64,8 @@ module Fluent::Plugin
     config_param :read_timeout, :integer, default: nil
     desc 'The TLS timeout in seconds'
     config_param :ssl_timeout, :integer, default: nil
+    desc 'Try to reuse connections'
+    config_param :reuse_connections, :bool, default: false
 
     desc 'The CA certificate path for TLS'
     config_param :tls_ca_cert_path, :string, default: nil
@@ -87,11 +93,29 @@ module Fluent::Plugin
 
     config_section :auth, required: false, multi: false do
       desc 'The method for HTTP authentication'
-      config_param :method, :enum, list: [:basic], default: :basic
+      config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic
       desc 'The username for basic authentication'
       config_param :username, :string, default: nil
       desc 'The password for basic authentication'
       config_param :password, :string, default: nil, secret: true
+      desc 'The AWS service to authenticate against'
+      config_param :aws_service, :string, default: nil
+      desc 'The AWS region to use when authenticating'
+      config_param :aws_region, :string, default: nil
+      desc 'The AWS role ARN to assume when authenticating'
+      config_param :aws_role_arn, :string, default: nil
+    end
+
+    def connection_cache_id_thread_key
+      "#{plugin_id}_connection_cache_id"
+    end
+
+    def connection_cache_id_for_thread
+      Thread.current[connection_cache_id_thread_key]
+    end
+
+    def connection_cache_id_for_thread=(id)
+      Thread.current[connection_cache_id_thread_key] = id
     end
 
     def initialize
@@ -100,11 +124,23 @@ module Fluent::Plugin
       @uri = nil
       @proxy_uri = nil
       @formatter = nil
+
+      @connection_cache = []
+      @connection_cache_id_mutex = Mutex.new
+      @connection_cache_next_id = 0
+    end
+
+    def close
+      super
+
+      @connection_cache.each {|entry| entry.conn.finish if entry.conn&.started? }
     end
 
     def configure(conf)
       super
 
+      @connection_cache = Array.new(actual_flush_thread_count, ConnectionCache.new("", nil)) if @reuse_connections
+
       if @retryable_response_codes.nil?
         log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
         @retryable_response_codes = [503]
@@ -121,6 +157,36 @@ module Fluent::Plugin
         end
         define_singleton_method(:format, method(:format_json_array))
       end
+
+      if @auth and @auth.method == :aws_sigv4
+        begin
+          require 'aws-sigv4'
+          require 'aws-sdk-core'
+        rescue LoadError
+          raise Fluent::ConfigError, "The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'"
+        end
+
+        raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil
+        raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil
+
+        if @auth.aws_role_arn == nil
+          aws_credentials = Aws::CredentialProviderChain.new.resolve
+        else
+          aws_credentials = Aws::AssumeRoleCredentials.new(
+            client: Aws::STS::Client.new(
+              region: @auth.aws_region
+            ),
+            role_arn: @auth.aws_role_arn,
+            role_session_name: "fluentd"
+          )
+        end
+
+        @aws_signer = Aws::Sigv4::Signer.new(
+          service: @auth.aws_service,
+          region: @auth.aws_region,
+          credentials_provider: aws_credentials
+        )
+      end
     end
 
     def multi_workers_ready?
@@ -215,7 +281,7 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req, chunk)
+    def set_headers(req, uri, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
@@ -226,9 +292,34 @@ module Fluent::Plugin
           req[k] = extract_placeholders(v, chunk)
         end
       end
+      if @compress == :gzip
+        req['Content-Encoding'] = "gzip"
+      end
       req['Content-Type'] = @content_type
     end
 
+    def set_auth(req, uri)
+      return unless @auth
+
+      if @auth.method == :basic
+        req.basic_auth(@auth.username, @auth.password)
+      elsif @auth.method == :aws_sigv4
+        signature = @aws_signer.sign_request(
+          http_method: req.method,
+          url: uri.request_uri,
+          headers: {
+            'Content-Type' => @content_type,
+            'Host' => uri.host
+          },
+          body: req.body
+        )
+        req.add_field('x-amz-date', signature.headers['x-amz-date'])
+        req.add_field('x-amz-security-token', signature.headers['x-amz-security-token'])
+        req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256'])
+        req.add_field('authorization', signature.headers['authorization'])
+      end
+    end
+
     def create_request(chunk, uri)
       req = case @http_method
             when :post
@@ -236,23 +327,56 @@ module Fluent::Plugin
             when :put
               Net::HTTP::Put.new(uri.request_uri)
             end
-      if @auth
-        req.basic_auth(@auth.username, @auth.password)
-      end
-      set_headers(req, chunk)
+      set_headers(req, uri, chunk)
+
       req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
+
+      if @compress == :gzip
+        gz = Zlib::GzipWriter.new(StringIO.new)
+        gz << req.body
+        req.body = gz.close.string
+      end
+
+      # At least one authentication method requires the body and other headers, so the order of this call matters
+      set_auth(req, uri)
       req
     end
 
+    def make_request_cached(uri, req)
+      id = self.connection_cache_id_for_thread
+      if id.nil?
+        @connection_cache_id_mutex.synchronize {
+          id = @connection_cache_next_id
+          @connection_cache_next_id += 1
+        }
+        self.connection_cache_id_for_thread = id
+      end
+      uri_str = uri.to_s
+      if @connection_cache[id].uri != uri_str
+        @connection_cache[id].conn.finish if @connection_cache[id].conn&.started?
+        http =  if @proxy_uri
+                  Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt)
+                else
+                  Net::HTTP.start(uri.host, uri.port, @http_opt)
+                end
+        @connection_cache[id] = ConnectionCache.new(uri_str, http)
+      end
+      @connection_cache[id].conn.request(req)
+    end
+
+    def make_request(uri, req, &block)
+      if @proxy_uri
+        Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt, &block)
+      else
+        Net::HTTP.start(uri.host, uri.port, @http_opt, &block)
+      end
+    end
+
     def send_request(uri, req)
-      res = if @proxy_uri
-              Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
-                http.request(req)
-              }
+      res = if @reuse_connections
+              make_request_cached(uri, req)
             else
-              Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
-                http.request(req)
-              }
+              make_request(uri, req) { |http| http.request(req) }
             end
 
       if res.is_a?(Net::HTTPSuccess)

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -21,7 +21,6 @@ module Fluent
         @_owner = plugin
 
         @_plugin_id = plugin.plugin_id
-        @_plugin_id_configured = plugin.plugin_id_configured?
 
         @log = plugin.log
       end

data/lib/fluent/plugin/parser_json.rb

@@ -50,36 +50,45 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          raise LoadError unless Fluent::OjOptions.available?
-          [Oj.method(:load), Oj::ParseError]
+          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
+
+          log&.info "Oj is not installed, and failing back to Yajl for json parser"
+          configure_json_parser(:yajl)
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError => ex
-        name = :yajl
-        if log
-          if /\boj\z/.match?(ex.message)
-            log.info "Oj is not installed, and failing back to Yajl for json parser"
-          else
-            log.warn ex.message
-          end
-        end
-        retry
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-        time = parse_time(record)
-        if @execute_convert_values
-          time, record = convert_values(time, record)
+        parsed_json = @load_proc.call(text)
+
+        if parsed_json.is_a?(Hash)
+          time, record = parse_one_record(parsed_json)
+          yield time, record
+        elsif parsed_json.is_a?(Array)
+          parsed_json.each do |record|
+            unless record.is_a?(Hash)
+              yield nil, nil
+              next
+            end
+            time, parsed_record = parse_one_record(record)
+            yield time, parsed_record
+          end
+        else
+          yield nil, nil
         end
-        yield time, record
+
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil
       end
 
+      def parse_one_record(record)
+        time = parse_time(record)
+        convert_values(time, record)
+      end
+
       def parser_type
         :text
       end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -31,9 +31,9 @@ module Fluent
         :binary
       end
 
-      def parse(data)
+      def parse(data, &block)
         @unpacker.feed_each(data) do |obj|
-          yield convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
         end
       end
       alias parse_partial_data parse
@@ -41,8 +41,29 @@ module Fluent
       def parse_io(io, &block)
         u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
         u.each do |obj|
-          time, record = convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
+        end
+      end
+
+      def parse_unpacked_data(data)
+        if data.is_a?(Hash)
+          time, record = convert_values(parse_time(data), data)
           yield time, record
+          return
+        end
+
+        unless data.is_a?(Array)
+          yield nil, nil
+          return
+        end
+
+        data.each do |record|
+          unless record.is_a?(Hash)
+            yield nil, nil
+            next
+          end
+          time, converted_record = convert_values(parse_time(record), record)
+          yield time, converted_record
         end
       end
     end