fluentd 1.6.3 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf4c49c147ccaa5c7ad0c4585a4e2a0e156b525ff5eb0a4a163df1c20c1c8e1d
-  data.tar.gz: 813578f6a63b400fadad9fead1b0edd323be8f92e31b1522133a886a28c9978b
+  metadata.gz: aa26c04f10203245130ee2457e56f1ea064df188d91c3610aa338f5f66273663
+  data.tar.gz: 48031fcf5dc26c8d1e3a6f6fc4a5f629451c6dc9a06c4a95043c3ba56da031b1
 SHA512:
-  metadata.gz: fbe1fbbe488245d3bda830a5ec3a0419e347ce7d6a685d5fd5f1d933c71382792cb047a45ffdbe982a763cdb336c1589958e9f84536c67452d1d608f9fea0db3
-  data.tar.gz: d9d2370bf3ce08c0500aa99f83c16af9ef3ebb97b15e8992bf88a0b76834b1eaad82a1bab27d3f39b07be90cf20fdf8646d31ee401bdf98a6709b65496417c5d
+  metadata.gz: ac003c0a88bfd623a9e1caf2989db9e45b38841827dc312e4e266ff38cc1ac25001a58fda0fc5b7eebb9b4efa4bfc9cbf0266ad36314aa5ed518303040e09727
+  data.tar.gz: b84fb3fe1b06ee99fcdad4cbc37d897b507b5b916d5ad8b539863827e6d1875ca6c026ae54b6b05898750e84357db51583e59872ee76bc814e3bc7e29e2e3fc2

data/.drone.yml

@@ -0,0 +1,35 @@
+kind: pipeline
+name: fluentd-test-arm64-2-6-3
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-2-6-3
+  image: arm64v8/ruby:2.6.3
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test
+---
+kind: pipeline
+name: fluentd-test-arm64-latest
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-latest
+  image: arm64v8/ruby:latest
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,6 +22,8 @@ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CON
 - Operating system: `cat /etc/os-release`
 - Kernel version: `uname -r`
 
+If you hit the problem with older fluentd version, try latest version first.
+
 **Your Configuration**
 
 ```

data/CHANGELOG.md

@@ -1,3 +1,61 @@
+# v1.7
+
+## Release v1.7.0 - 2019/08/20
+
+### New feature
+
+* buffer: Add file_single buffer
+  https://github.com/fluent/fluentd/pull/2479
+
+### Enhancement
+
+* buffer: Improve the performance of buffer routine
+  https://github.com/fluent/fluentd/pull/2560
+  https://github.com/fluent/fluentd/pull/2563
+  https://github.com/fluent/fluentd/pull/2564
+* output: Use Mutext instead of Monitor
+  https://github.com/fluent/fluentd/pull/2561
+* event: Add `OneEvent#empty?` method
+  https://github.com/fluent/fluentd/pull/2565
+* thread: Set thread name for ruby 2.3 or later
+  https://github.com/fluent/fluentd/pull/2574
+* core: Cache msgpack packer/unpacker to avoid the object allocation
+  https://github.com/fluent/fluentd/pull/2559
+* time: Use faster way to get sec and nsec
+  https://github.com/fluent/fluentd/pull/2557
+* buf_file: Reduce IO flush by removing `IO#truncate`
+  https://github.com/fluent/fluentd/pull/2551
+* in_tcp: Improve the performance for multiple event case
+  https://github.com/fluent/fluentd/pull/2567
+* in_syslog: support `source_hostname_key` and `source_address_key` for unmatched event
+  https://github.com/fluent/fluentd/pull/2553
+* formatter_csv: Improve the format performance.
+  https://github.com/fluent/fluentd/pull/2529
+* parser_csv: Add fast parser for typical cases
+  https://github.com/fluent/fluentd/pull/2535
+* out_forward: Refactor code
+  https://github.com/fluent/fluentd/pull/2516
+  https://github.com/fluent/fluentd/pull/2532
+
+### Bug fixes
+
+* output: fix data lost on decompression
+  https://github.com/fluent/fluentd/pull/2547
+* out_exec_filter: fix non-ascii encoding issue
+  https://github.com/fluent/fluentd/pull/2539
+* in_tail: Don't call parser's configure twice
+  https://github.com/fluent/fluentd/pull/2569
+* Fix unused message handling for <section> parameters
+  https://github.com/fluent/fluentd/pull/2578
+* Fix comment/message typos
+  https://github.com/fluent/fluentd/pull/2549
+  https://github.com/fluent/fluentd/pull/2554
+  https://github.com/fluent/fluentd/pull/2556
+  https://github.com/fluent/fluentd/pull/2566
+  https://github.com/fluent/fluentd/pull/2573
+  https://github.com/fluent/fluentd/pull/2576
+  https://github.com/fluent/fluentd/pull/2583
+
 # v1.6
 
 ## Release v1.6.3 - 2019/07/29

data/README.md

@@ -10,6 +10,10 @@ GitLab CI:
 
 [![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
 
+Drone CI for Arm64:
+
+[![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
+
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
@@ -78,7 +82,7 @@ You can run specified test via `TEST` environment variable:
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
-- Copyright: 2011-2018 Fluentd Authors
+- Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
 ## Contributors:

data/fluentd.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo", ["~> 2.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
   gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])

data/lib/fluent/clock.rb

@@ -43,6 +43,10 @@ module Fluent
       Process.clock_gettime(CLOCK_ID)
     end
 
+    def self.real_now(unit = :second)
+      Process.clock_gettime(Process::CLOCK_REALTIME, unit)
+    end
+
     def self.dst_clock_from_time(time)
       diff_sec = Time.now - time
       now_raw - diff_sec

data/lib/fluent/compat/output.rb

@@ -352,7 +352,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -363,7 +363,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -373,7 +373,7 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += size }
+        @counter_mutex.synchronize{ @emit_records += size }
         [meta]
       end
 

data/lib/fluent/compat/socket_util.rb

@@ -52,7 +52,7 @@ module Fluent
       end
 
       class TcpHandler < Coolio::Socket
-        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+        PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
 
         def initialize(io, log, delimiter, callback)
           super(io)

data/lib/fluent/config/element.rb

@@ -31,7 +31,7 @@ module Fluent
         @unused = unused || attrs.keys
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
-        @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+        @unused_in = nil # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
 
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
@@ -110,13 +110,13 @@ module Fluent
       end
 
       def has_key?(key)
-        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
+        @unused_in = [] # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
         @unused.delete(key)
         super
       end
 
       def [](key)
-        @unused_in = false # ditto
+        @unused_in = [] # ditto
         @unused.delete(key)
 
         if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])

data/lib/fluent/config/literal_parser.rb

@@ -112,7 +112,7 @@ module Fluent
           elsif s = scan(/./)
             string << s
           else
-            parse_error! "unexpected end of file in a signle quoted string"
+            parse_error! "unexpected end of file in a single quoted string"
           end
         end
       end

data/lib/fluent/config/section.rb

@@ -207,8 +207,11 @@ module Fluent
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          next if e.unused_in && e.unused_in.empty? # the section is used at least once
 
-          unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            e.unused_in = []
+          else
             parent_name = if conf.arg.empty?
                             conf.name
                           else

data/lib/fluent/error.rb

@@ -27,4 +27,8 @@ module Fluent
 
   class InvalidRootDirectory < UnrecoverableError
   end
+
+  # For internal use
+  class UncatchableError < Exception
+  end
 end

data/lib/fluent/event.rb

@@ -50,30 +50,30 @@ module Fluent
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def each(&block)
+    def each(unapcker: nil, &block)
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def to_msgpack_stream(time_int: false)
-      return to_msgpack_stream_forced_integer if time_int
-      out = msgpack_packer
+    def to_msgpack_stream(time_int: false, packer: nil)
+      return to_msgpack_stream_forced_integer(packer: packer) if time_int
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
-      out.to_s
+      out.full_pack
     end
 
-    def to_compressed_msgpack_stream(time_int: false)
-      packed = to_msgpack_stream(time_int: time_int)
+    def to_compressed_msgpack_stream(time_int: false, packer: nil)
+      packed = to_msgpack_stream(time_int: time_int, packer: packer)
       compress(packed)
     end
 
-    def to_msgpack_stream_forced_integer
-      out = msgpack_packer
+    def to_msgpack_stream_forced_integer(packer: nil)
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time.to_i,record])
       }
-      out.to_s
+      out.full_pack
     end
   end
 
@@ -87,6 +87,10 @@ module Fluent
       OneEventStream.new(@time, @record.dup)
     end
 
+    def empty?
+      false
+    end
+
     def size
       1
     end
@@ -103,7 +107,7 @@ module Fluent
       end
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       block.call(@time, @record)
       nil
     end
@@ -139,7 +143,7 @@ module Fluent
       ArrayEventStream.new(@entries.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       @entries.each(&block)
       nil
     end
@@ -186,7 +190,7 @@ module Fluent
       MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       time_array = @time_array
       record_array = @record_array
       for i in 0..time_array.length-1
@@ -230,11 +234,11 @@ module Fluent
       true
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       return if @unpacked_times && @unpacked_records
       @unpacked_times = []
       @unpacked_records = []
-      msgpack_unpacker.feed_each(@data) do |time, record|
+      (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
         @unpacked_times << time
         @unpacked_records << record
       end
@@ -250,7 +254,7 @@ module Fluent
       MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       if @unpacked_times
         @unpacked_times.each_with_index do |time, i|
           block.call(time, @unpacked_records[i])
@@ -258,7 +262,7 @@ module Fluent
       else
         @unpacked_times = []
         @unpacked_records = []
-        msgpack_unpacker.feed_each(@data) do |time, record|
+        (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
           @unpacked_times << time
           @unpacked_records << record
           block.call(time, record)
@@ -268,7 +272,7 @@ module Fluent
       nil
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because @data is always packed binary in this class
       @data
     end
@@ -286,17 +290,17 @@ module Fluent
       super
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       ensure_decompressed!
       super
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       ensure_decompressed!
       super
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       ensure_decompressed!
       super
     end
@@ -318,15 +322,15 @@ module Fluent
     include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)
     #  => chunk.each{|time, record| ... }
-    def each(&block)
+    def each(unpacker: nil, &block)
       open do |io|
-        msgpack_unpacker(io).each(&block)
+        (unpacker || msgpack_unpacker(io)).each(&block)
       end
       nil
     end
     alias :msgpack_each :each
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because data is already packed and written in chunk
       read
     end

data/lib/fluent/event_router.rb

@@ -17,6 +17,7 @@
 require 'fluent/match'
 require 'fluent/event'
 require 'fluent/filter'
+require 'fluent/msgpack_factory'
 
 module Fluent
   #
@@ -182,7 +183,7 @@ module Fluent
 
         def optimized_filter_stream(tag, es)
           new_es = MultiEventStream.new
-          es.each do |time, record|
+          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
             filtered_record = record
             filtered_time = time
 

data/lib/fluent/log.rb

@@ -606,7 +606,7 @@ module Fluent
     end
   end
 
-  # This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
+  # This class delegates some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
   # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
   class LogDeviceIO < ::Logger::LogDevice
     def flush

data/lib/fluent/msgpack_factory.rb

@@ -58,5 +58,13 @@ module Fluent
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
       @@engine_factory = factory
     end
+
+    def self.thread_local_msgpack_packer
+      Thread.current[:local_msgpack_packer] ||= MessagePackFactory.engine_factory.packer
+    end
+
+    def self.thread_local_msgpack_unpacker
+      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+    end
   end
 end