fluentd 1.6.3 → 1.7.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf4c49c147ccaa5c7ad0c4585a4e2a0e156b525ff5eb0a4a163df1c20c1c8e1d
-  data.tar.gz: 813578f6a63b400fadad9fead1b0edd323be8f92e31b1522133a886a28c9978b
+  metadata.gz: f5ce0f9135cf13b32692d41e65dbb869738702a0d328ee1decec0b8848000feb
+  data.tar.gz: c803e8f6ece1fa166c1a988867f04c660f0f7dd821c5d7f09dbd1a96d478b9c7
 SHA512:
-  metadata.gz: fbe1fbbe488245d3bda830a5ec3a0419e347ce7d6a685d5fd5f1d933c71382792cb047a45ffdbe982a763cdb336c1589958e9f84536c67452d1d608f9fea0db3
-  data.tar.gz: d9d2370bf3ce08c0500aa99f83c16af9ef3ebb97b15e8992bf88a0b76834b1eaad82a1bab27d3f39b07be90cf20fdf8646d31ee401bdf98a6709b65496417c5d
+  metadata.gz: 18909799465547cfe11c00f701e5aad6845a346be63a29eddf872d56d3ae680ad967e852af5f468a451d1decb051cffce5d8d5e68d8ab3c97307137ccf22a9fa
+  data.tar.gz: 6fbed649b9f164f3f699667e57dd2706e98f11c69b2c31210767f520a4f62310e34a14ac12c5ddde5244f5232c83f6b7ad83dc925a096054ec71abd493ceeecd

data/.drone.yml

@@ -0,0 +1,35 @@
+kind: pipeline
+name: fluentd-test-arm64-2-6-3
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-2-6-3
+  image: arm64v8/ruby:2.6.3
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test
+---
+kind: pipeline
+name: fluentd-test-arm64-latest
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-latest
+  image: arm64v8/ruby:latest
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,6 +22,8 @@ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CON
 - Operating system: `cat /etc/os-release`
 - Kernel version: `uname -r`
 
+If you hit the problem with older fluentd version, try latest version first.
+
 **Your Configuration**
 
 ```

data/README.md

@@ -10,6 +10,10 @@ GitLab CI:
 
 [![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
 
+Drone CI for Arm64:
+
+[![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
+
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
@@ -78,7 +82,7 @@ You can run specified test via `TEST` environment variable:
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
-- Copyright: 2011-2018 Fluentd Authors
+- Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
 ## Contributors:

data/fluentd.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo", ["~> 2.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
   gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])

data/lib/fluent/clock.rb

@@ -43,6 +43,10 @@ module Fluent
       Process.clock_gettime(CLOCK_ID)
     end
 
+    def self.real_now(unit = :second)
+      Process.clock_gettime(Process::CLOCK_REALTIME, unit)
+    end
+
     def self.dst_clock_from_time(time)
       diff_sec = Time.now - time
       now_raw - diff_sec

data/lib/fluent/compat/output.rb

@@ -352,7 +352,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -363,7 +363,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -373,7 +373,7 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += size }
+        @counter_mutex.synchronize{ @emit_records += size }
         [meta]
       end
 

data/lib/fluent/compat/socket_util.rb

@@ -52,7 +52,7 @@ module Fluent
       end
 
       class TcpHandler < Coolio::Socket
-        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+        PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
 
         def initialize(io, log, delimiter, callback)
           super(io)

data/lib/fluent/config/element.rb

@@ -31,7 +31,7 @@ module Fluent
         @unused = unused || attrs.keys
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
-        @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+        @unused_in = nil # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
 
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
@@ -110,13 +110,13 @@ module Fluent
       end
 
       def has_key?(key)
-        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
+        @unused_in = [] # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
         @unused.delete(key)
         super
       end
 
       def [](key)
-        @unused_in = false # ditto
+        @unused_in = [] # ditto
         @unused.delete(key)
 
         if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])

data/lib/fluent/config/literal_parser.rb

@@ -112,7 +112,7 @@ module Fluent
           elsif s = scan(/./)
             string << s
           else
-            parse_error! "unexpected end of file in a signle quoted string"
+            parse_error! "unexpected end of file in a single quoted string"
           end
         end
       end

data/lib/fluent/config/section.rb

@@ -207,8 +207,11 @@ module Fluent
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          next if e.unused_in && e.unused_in.empty? # the section is used at least once
 
-          unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            e.unused_in = []
+          else
             parent_name = if conf.arg.empty?
                             conf.name
                           else

data/lib/fluent/error.rb

@@ -27,4 +27,8 @@ module Fluent
 
   class InvalidRootDirectory < UnrecoverableError
   end
+
+  # For internal use
+  class UncatchableError < Exception
+  end
 end

data/lib/fluent/event.rb

@@ -50,30 +50,30 @@ module Fluent
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def each(&block)
+    def each(unapcker: nil, &block)
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def to_msgpack_stream(time_int: false)
-      return to_msgpack_stream_forced_integer if time_int
-      out = msgpack_packer
+    def to_msgpack_stream(time_int: false, packer: nil)
+      return to_msgpack_stream_forced_integer(packer: packer) if time_int
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
-      out.to_s
+      out.full_pack
     end
 
-    def to_compressed_msgpack_stream(time_int: false)
-      packed = to_msgpack_stream(time_int: time_int)
+    def to_compressed_msgpack_stream(time_int: false, packer: nil)
+      packed = to_msgpack_stream(time_int: time_int, packer: packer)
       compress(packed)
     end
 
-    def to_msgpack_stream_forced_integer
-      out = msgpack_packer
+    def to_msgpack_stream_forced_integer(packer: nil)
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time.to_i,record])
       }
-      out.to_s
+      out.full_pack
     end
   end
 
@@ -87,6 +87,10 @@ module Fluent
       OneEventStream.new(@time, @record.dup)
     end
 
+    def empty?
+      false
+    end
+
     def size
       1
     end
@@ -103,7 +107,7 @@ module Fluent
       end
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       block.call(@time, @record)
       nil
     end
@@ -139,7 +143,7 @@ module Fluent
       ArrayEventStream.new(@entries.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       @entries.each(&block)
       nil
     end
@@ -186,7 +190,7 @@ module Fluent
       MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       time_array = @time_array
       record_array = @record_array
       for i in 0..time_array.length-1
@@ -230,11 +234,11 @@ module Fluent
       true
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       return if @unpacked_times && @unpacked_records
       @unpacked_times = []
       @unpacked_records = []
-      msgpack_unpacker.feed_each(@data) do |time, record|
+      (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
         @unpacked_times << time
         @unpacked_records << record
       end
@@ -250,7 +254,7 @@ module Fluent
       MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       if @unpacked_times
         @unpacked_times.each_with_index do |time, i|
           block.call(time, @unpacked_records[i])
@@ -258,7 +262,7 @@ module Fluent
       else
         @unpacked_times = []
         @unpacked_records = []
-        msgpack_unpacker.feed_each(@data) do |time, record|
+        (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
           @unpacked_times << time
           @unpacked_records << record
           block.call(time, record)
@@ -268,7 +272,7 @@ module Fluent
       nil
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because @data is always packed binary in this class
       @data
     end
@@ -286,17 +290,17 @@ module Fluent
       super
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       ensure_decompressed!
       super
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       ensure_decompressed!
       super
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       ensure_decompressed!
       super
     end
@@ -318,15 +322,15 @@ module Fluent
     include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)
     #  => chunk.each{|time, record| ... }
-    def each(&block)
+    def each(unpacker: nil, &block)
       open do |io|
-        msgpack_unpacker(io).each(&block)
+        (unpacker || msgpack_unpacker(io)).each(&block)
       end
       nil
     end
     alias :msgpack_each :each
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because data is already packed and written in chunk
       read
     end

data/lib/fluent/event_router.rb

@@ -17,6 +17,7 @@
 require 'fluent/match'
 require 'fluent/event'
 require 'fluent/filter'
+require 'fluent/msgpack_factory'
 
 module Fluent
   #
@@ -182,7 +183,7 @@ module Fluent
 
         def optimized_filter_stream(tag, es)
           new_es = MultiEventStream.new
-          es.each do |time, record|
+          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
             filtered_record = record
             filtered_time = time
 

data/lib/fluent/log.rb

@@ -606,7 +606,7 @@ module Fluent
     end
   end
 
-  # This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
+  # This class delegates some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
   # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
   class LogDeviceIO < ::Logger::LogDevice
     def flush

data/lib/fluent/msgpack_factory.rb

@@ -58,5 +58,13 @@ module Fluent
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
       @@engine_factory = factory
     end
+
+    def self.thread_local_msgpack_packer
+      Thread.current[:local_msgpack_packer] ||= MessagePackFactory.engine_factory.packer
+    end
+
+    def self.thread_local_msgpack_unpacker
+      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+    end
   end
 end

data/lib/fluent/plugin/bare_output.rb

@@ -40,7 +40,7 @@ module Fluent
 
       def initialize
         super
-        @counters_monitor = Monitor.new
+        @counter_mutex = Mutex.new
         # TODO: well organized counters
         @num_errors = 0
         @emit_count = 0
@@ -48,12 +48,12 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counters_monitor.synchronize{ @emit_count += 1 }
+        @counter_mutex.synchronize{ @emit_count += 1 }
         begin
           process(tag, es)
-          @counters_monitor.synchronize{ @emit_records += es.size }
+          @counter_mutex.synchronize{ @emit_records += es.size }
         rescue
-          @counters_monitor.synchronize{ @num_errors += 1 }
+          @counter_mutex.synchronize{ @num_errors += 1 }
           raise
         end
       end

data/lib/fluent/plugin/buf_file_single.rb

@@ -0,0 +1,211 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/buffer/file_single_chunk'
+require 'fluent/system_config'
+
+module Fluent
+  module Plugin
+    class FileSingleBuffer < Fluent::Plugin::Buffer
+      Plugin.register_buffer('file_single', self)
+
+      include SystemConfig::Mixin
+
+      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB
+      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB
+
+      PATH_SUFFIX = ".#{Fluent::Plugin::Buffer::FileSingleChunk::PATH_EXT}"
+      DIR_PERMISSION = 0755
+
+      desc 'The path where buffer chunks are stored.'
+      config_param :path, :string, default: nil
+      desc 'Calculate the number of record in chunk during resume'
+      config_param :calc_num_records, :bool, default: true
+      desc 'The format of chunk. This is used to calculate the number of record'
+      config_param :chunk_format, :enum, list: [:msgpack, :text, :auto], default: :auto
+
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
+      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
+
+      desc 'The permission of chunk file. If no specified, <system> setting or 0644 is used'
+      config_param :file_permission, :string, default: nil
+      desc 'The permission of chunk directory. If no specified, <system> setting or 0755 is used'
+      config_param :dir_permission, :string, default: nil
+
+      @@buffer_paths = {}
+
+      def initialize
+        super
+
+        @multi_workers_available = false
+        @additional_resume_path = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @chunk_format == :auto
+          @chunk_format = owner.formatted_to_msgpack_binary? ? :msgpack : :text
+        end
+
+        @key_in_path = nil
+        if owner.chunk_keys.empty?
+          log.debug "use event tag for buffer key"
+        else
+          if owner.chunk_key_tag
+            raise Fluent::ConfigError, "chunk keys must be tag or one field"
+          elsif owner.chunk_keys.size > 1
+            raise Fluent::ConfigError, "2 or more chunk keys is not allowed"
+          else
+            @key_in_path = owner.chunk_keys.first.to_sym
+          end
+        end
+
+        multi_workers_configured = owner.system_config.workers > 1
+        using_plugin_root_dir = false
+        unless @path
+          if root_dir = owner.plugin_root_dir
+            @path = File.join(root_dir, 'buffer')
+            using_plugin_root_dir = true # plugin_root_dir path contains worker id
+          else
+            raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
+          end
+        end
+
+        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
+          type_using_this_path = @@buffer_paths[@path]
+          raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
+        end
+
+        @@buffer_paths[@path] = type_of_owner
+
+        specified_directory_exists = File.exist?(@path) && File.directory?(@path)
+        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
+
+        if specified_directory_exists || unexisting_path_for_directory # directory
+          if using_plugin_root_dir || !multi_workers_configured
+            @path = File.join(@path, "fsb.*#{PATH_SUFFIX}")
+          else
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "fsb.*#{PATH_SUFFIX}")
+            if fluentd_worker_id == 0
+              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "fsb.*#{PATH_SUFFIX}")
+            end
+          end
+          @multi_workers_available = true
+        else # specified path is file path
+          if File.basename(@path).include?('.*.')
+            new_path = File.join(File.dirname(@path), "fsb.*#{PATH_SUFFIX}")
+            log.warn "file_single doesn't allow user specified 'prefix.*.suffix' style path. Use '#{new_path}' for file instead: #{@path}"
+            @path = new_path
+          elsif File.basename(@path).end_with?('.*')
+            @path = @path + PATH_SUFFIX
+          else
+            # existing file will be ignored
+            @path = @path + ".*#{PATH_SUFFIX}"
+          end
+          @multi_workers_available = false
+        end
+
+        @dir_permission = if @dir_permission
+                            @dir_permission.to_i(8)
+                          else
+                            system_config.dir_permission || DIR_PERMISSION
+                          end
+      end
+
+      # This method is called only when multi worker is configured
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "file_single buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def start
+        FileUtils.mkdir_p(File.dirname(@path), mode: @dir_permission)
+
+        super
+      end
+
+      def persistent?
+        true
+      end
+
+      def resume
+        stage = {}
+        queue = []
+
+        patterns = [@path]
+        patterns.unshift @additional_resume_path if @additional_resume_path
+        Dir.glob(patterns) do |path|
+          next unless File.file?(path)
+
+          log.debug { "restoring buffer file: path = #{path}" }
+
+          m = new_metadata() # this metadata will be updated in FileSingleChunk.new
+          mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
+          if mode == :unknown
+            log.debug "unknown state chunk found", path: path
+            next
+          end
+
+          begin
+            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path)
+            chunk.restore_size(@chunk_format) if @calc_num_records
+          rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
+            handle_broken_files(path, mode, e)
+            next
+          end
+
+          case chunk.state
+          when :staged
+            stage[chunk.metadata] = chunk
+          when :queued
+            queue << chunk
+          end
+        end
+
+        queue.sort_by!(&:modified_at)
+
+        return stage, queue
+      end
+
+      def generate_chunk(metadata)
+        # FileChunk generates real path with unique_id
+        if @file_permission
+          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: @file_permission, compress: @compress)
+        else
+          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, compress: @compress)
+        end
+
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
+
+        chunk
+      end
+
+      def handle_broken_files(path, mode, e)
+        log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
+        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        File.unlink(path) rescue nil
+      end
+    end
+  end
+end