fluentd 1.6.3-x86-mingw32 → 1.7.0-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb2e03cd7a40b2276b0ef281ee57e27af4a16204b8da766d1a97c9ae145a1b02
-  data.tar.gz: 91066de64539ac2a06fa97a730702ce41fb53ecab5818824c26630cacaa756e7
+  metadata.gz: 5b90cc36e403fd5085808b25ad55df73934177e67ea0451d8b2f9c8d3654826f
+  data.tar.gz: 1987375e47fbefce01117be8a01c21ffc19add05f0215114a394664e9a639f1d
 SHA512:
-  metadata.gz: 7db0784aeb7136fbc9e6057855ff1c01235acfe4bea6b526d1ca82d01ce40e379a7be9c23534904ee14f036c18c5f3dbeb863bdd141742875b3494057f2a373d
-  data.tar.gz: d1b838408dcc95821fe4cd8487690a1ea7e684157189d743d9e9f94dcfebe2d799b3321ecb3aeb05c08c87d2264a98377fb020af8f3b6d62f78323c5d6f90c03
+  metadata.gz: 4854391dcd5ab6b15df7ad2fecfed383c5e4a01be956a49d2d1dda7eaba43aa0032ad9fa50da9f06484102ac5be0bab83c3771cd0cda9526c8cb317b6304b892
+  data.tar.gz: 38bc9a17665bf0571e4f6c0fd404ac31545bba44427f6b3fa92cd5e116c7c397fca479775a1e71fe970fb3925bc0131d92125fcc070bba900944f3f3e4c68caa

data/.drone.yml

@@ -0,0 +1,35 @@
+kind: pipeline
+name: fluentd-test-arm64-2-6-3
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-2-6-3
+  image: arm64v8/ruby:2.6.3
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test
+---
+kind: pipeline
+name: fluentd-test-arm64-latest
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: fluentd-test-arm64-latest
+  image: arm64v8/ruby:latest
+  commands:
+  - apt update
+  - apt -y install libgmp3-dev
+  - export BUNDLE_GEMFILE=$PWD/Gemfile
+  - gem update --system
+  - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}
+  - bundle exec rake test

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,6 +22,8 @@ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CON
 - Operating system: `cat /etc/os-release`
 - Kernel version: `uname -r`
 
+If you hit the problem with older fluentd version, try latest version first.
+
 **Your Configuration**
 
 ```

data/CHANGELOG.md

@@ -1,3 +1,61 @@
+# v1.7
+
+## Release v1.7.0 - 2019/08/20
+
+### New feature
+
+* buffer: Add file_single buffer
+  https://github.com/fluent/fluentd/pull/2479
+
+### Enhancement
+
+* buffer: Improve the performance of buffer routine
+  https://github.com/fluent/fluentd/pull/2560
+  https://github.com/fluent/fluentd/pull/2563
+  https://github.com/fluent/fluentd/pull/2564
+* output: Use Mutext instead of Monitor
+  https://github.com/fluent/fluentd/pull/2561
+* event: Add `OneEvent#empty?` method
+  https://github.com/fluent/fluentd/pull/2565
+* thread: Set thread name for ruby 2.3 or later
+  https://github.com/fluent/fluentd/pull/2574
+* core: Cache msgpack packer/unpacker to avoid the object allocation
+  https://github.com/fluent/fluentd/pull/2559
+* time: Use faster way to get sec and nsec
+  https://github.com/fluent/fluentd/pull/2557
+* buf_file: Reduce IO flush by removing `IO#truncate`
+  https://github.com/fluent/fluentd/pull/2551
+* in_tcp: Improve the performance for multiple event case
+  https://github.com/fluent/fluentd/pull/2567
+* in_syslog: support `source_hostname_key` and `source_address_key` for unmatched event
+  https://github.com/fluent/fluentd/pull/2553
+* formatter_csv: Improve the format performance.
+  https://github.com/fluent/fluentd/pull/2529
+* parser_csv: Add fast parser for typical cases
+  https://github.com/fluent/fluentd/pull/2535
+* out_forward: Refactor code
+  https://github.com/fluent/fluentd/pull/2516
+  https://github.com/fluent/fluentd/pull/2532
+
+### Bug fixes
+
+* output: fix data lost on decompression
+  https://github.com/fluent/fluentd/pull/2547
+* out_exec_filter: fix non-ascii encoding issue
+  https://github.com/fluent/fluentd/pull/2539
+* in_tail: Don't call parser's configure twice
+  https://github.com/fluent/fluentd/pull/2569
+* Fix unused message handling for <section> parameters
+  https://github.com/fluent/fluentd/pull/2578
+* Fix comment/message typos
+  https://github.com/fluent/fluentd/pull/2549
+  https://github.com/fluent/fluentd/pull/2554
+  https://github.com/fluent/fluentd/pull/2556
+  https://github.com/fluent/fluentd/pull/2566
+  https://github.com/fluent/fluentd/pull/2573
+  https://github.com/fluent/fluentd/pull/2576
+  https://github.com/fluent/fluentd/pull/2583
+
 # v1.6
 
 ## Release v1.6.3 - 2019/07/29

data/README.md

@@ -10,6 +10,10 @@ GitLab CI:
 
 [![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
 
+Drone CI for Arm64:
+
+[![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
+
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
@@ -78,7 +82,7 @@ You can run specified test via `TEST` environment variable:
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
-- Copyright: 2011-2018 Fluentd Authors
+- Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
 ## Contributors:

data/fluentd.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo", ["~> 2.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
   gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])

data/lib/fluent/clock.rb

@@ -43,6 +43,10 @@ module Fluent
       Process.clock_gettime(CLOCK_ID)
     end
 
+    def self.real_now(unit = :second)
+      Process.clock_gettime(Process::CLOCK_REALTIME, unit)
+    end
+
     def self.dst_clock_from_time(time)
       diff_sec = Time.now - time
       now_raw - diff_sec

data/lib/fluent/compat/output.rb

@@ -352,7 +352,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -363,7 +363,7 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counters_monitor.synchronize{ @emit_records += size }
+          @counter_mutex.synchronize{ @emit_records += size }
           return [meta]
         end
 
@@ -373,7 +373,7 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += size }
+        @counter_mutex.synchronize{ @emit_records += size }
         [meta]
       end
 

data/lib/fluent/compat/socket_util.rb

@@ -52,7 +52,7 @@ module Fluent
       end
 
       class TcpHandler < Coolio::Socket
-        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+        PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
 
         def initialize(io, log, delimiter, callback)
           super(io)

data/lib/fluent/config/element.rb

@@ -31,7 +31,7 @@ module Fluent
         @unused = unused || attrs.keys
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
-        @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+        @unused_in = nil # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
 
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
@@ -110,13 +110,13 @@ module Fluent
       end
 
       def has_key?(key)
-        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
+        @unused_in = [] # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
         @unused.delete(key)
         super
       end
 
       def [](key)
-        @unused_in = false # ditto
+        @unused_in = [] # ditto
         @unused.delete(key)
 
         if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])

data/lib/fluent/config/literal_parser.rb

@@ -112,7 +112,7 @@ module Fluent
           elsif s = scan(/./)
             string << s
           else
-            parse_error! "unexpected end of file in a signle quoted string"
+            parse_error! "unexpected end of file in a single quoted string"
           end
         end
       end

data/lib/fluent/config/section.rb

@@ -207,8 +207,11 @@ module Fluent
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          next if e.unused_in && e.unused_in.empty? # the section is used at least once
 
-          unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            e.unused_in = []
+          else
             parent_name = if conf.arg.empty?
                             conf.name
                           else

data/lib/fluent/error.rb

@@ -27,4 +27,8 @@ module Fluent
 
   class InvalidRootDirectory < UnrecoverableError
   end
+
+  # For internal use
+  class UncatchableError < Exception
+  end
 end

data/lib/fluent/event.rb

@@ -50,30 +50,30 @@ module Fluent
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def each(&block)
+    def each(unapcker: nil, &block)
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def to_msgpack_stream(time_int: false)
-      return to_msgpack_stream_forced_integer if time_int
-      out = msgpack_packer
+    def to_msgpack_stream(time_int: false, packer: nil)
+      return to_msgpack_stream_forced_integer(packer: packer) if time_int
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
-      out.to_s
+      out.full_pack
     end
 
-    def to_compressed_msgpack_stream(time_int: false)
-      packed = to_msgpack_stream(time_int: time_int)
+    def to_compressed_msgpack_stream(time_int: false, packer: nil)
+      packed = to_msgpack_stream(time_int: time_int, packer: packer)
       compress(packed)
     end
 
-    def to_msgpack_stream_forced_integer
-      out = msgpack_packer
+    def to_msgpack_stream_forced_integer(packer: nil)
+      out = packer || msgpack_packer
       each {|time,record|
         out.write([time.to_i,record])
       }
-      out.to_s
+      out.full_pack
     end
   end
 
@@ -87,6 +87,10 @@ module Fluent
       OneEventStream.new(@time, @record.dup)
     end
 
+    def empty?
+      false
+    end
+
     def size
       1
     end
@@ -103,7 +107,7 @@ module Fluent
       end
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       block.call(@time, @record)
       nil
     end
@@ -139,7 +143,7 @@ module Fluent
       ArrayEventStream.new(@entries.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       @entries.each(&block)
       nil
     end
@@ -186,7 +190,7 @@ module Fluent
       MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       time_array = @time_array
       record_array = @record_array
       for i in 0..time_array.length-1
@@ -230,11 +234,11 @@ module Fluent
       true
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       return if @unpacked_times && @unpacked_records
       @unpacked_times = []
       @unpacked_records = []
-      msgpack_unpacker.feed_each(@data) do |time, record|
+      (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
         @unpacked_times << time
         @unpacked_records << record
       end
@@ -250,7 +254,7 @@ module Fluent
       MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       if @unpacked_times
         @unpacked_times.each_with_index do |time, i|
           block.call(time, @unpacked_records[i])
@@ -258,7 +262,7 @@ module Fluent
       else
         @unpacked_times = []
         @unpacked_records = []
-        msgpack_unpacker.feed_each(@data) do |time, record|
+        (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
           @unpacked_times << time
           @unpacked_records << record
           block.call(time, record)
@@ -268,7 +272,7 @@ module Fluent
       nil
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because @data is always packed binary in this class
       @data
     end
@@ -286,17 +290,17 @@ module Fluent
       super
     end
 
-    def ensure_unpacked!
+    def ensure_unpacked!(unpacker: nil)
       ensure_decompressed!
       super
     end
 
-    def each(&block)
+    def each(unpacker: nil, &block)
       ensure_decompressed!
       super
     end
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       ensure_decompressed!
       super
     end
@@ -318,15 +322,15 @@ module Fluent
     include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)
     #  => chunk.each{|time, record| ... }
-    def each(&block)
+    def each(unpacker: nil, &block)
       open do |io|
-        msgpack_unpacker(io).each(&block)
+        (unpacker || msgpack_unpacker(io)).each(&block)
       end
       nil
     end
     alias :msgpack_each :each
 
-    def to_msgpack_stream(time_int: false)
+    def to_msgpack_stream(time_int: false, packer: nil)
       # time_int is always ignored because data is already packed and written in chunk
       read
     end

data/lib/fluent/event_router.rb

@@ -17,6 +17,7 @@
 require 'fluent/match'
 require 'fluent/event'
 require 'fluent/filter'
+require 'fluent/msgpack_factory'
 
 module Fluent
   #
@@ -182,7 +183,7 @@ module Fluent
 
         def optimized_filter_stream(tag, es)
           new_es = MultiEventStream.new
-          es.each do |time, record|
+          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
             filtered_record = record
             filtered_time = time
 

data/lib/fluent/log.rb

@@ -606,7 +606,7 @@ module Fluent
     end
   end
 
-  # This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
+  # This class delegates some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
   # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
   class LogDeviceIO < ::Logger::LogDevice
     def flush

data/lib/fluent/msgpack_factory.rb

@@ -58,5 +58,13 @@ module Fluent
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
       @@engine_factory = factory
     end
+
+    def self.thread_local_msgpack_packer
+      Thread.current[:local_msgpack_packer] ||= MessagePackFactory.engine_factory.packer
+    end
+
+    def self.thread_local_msgpack_unpacker
+      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+    end
   end
 end