fluentd 1.8.1 → 1.9.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7df1551f31fc37657b4a757f55ca11bfa66c9a925662512d36ac87309c07bae6
-  data.tar.gz: 3aec1319bee2aa7a07f5a2da46925880cd701a2f004eaff8616ee42077c3f437
+  metadata.gz: 05d0842de47c79ef5a898ea8777454fe9c9863b4d5b6e5161f05c9d5e30bf1c7
+  data.tar.gz: 59fc0341af8411eb6e5830005f097c3f03851df9117d64ea838c91d36e9a7ff6
 SHA512:
-  metadata.gz: 76490ade18c60a06ead5f3626d94eccff81c356278841832b4007d232c31db72a1a89e8f4ef4ff12005ee3425ee75c5484bdd2b38ce347d21f22cd53331d6e86
-  data.tar.gz: c273fe0a21377a313f5a29fbf68c15731bc8dc4fe3cb6330b6b9cf37a8921a716265ee1459d2ef322bd886e3db88411dc88efac7cc50dd498b7f797c404bda22
+  metadata.gz: aee7aaba37e954564671aa0790f6af069d583670bd02819578342d35b94061a225e5714fc221b1631d099000d36eb3df04e014edad39a5c7ac3156733cbf172b
+  data.tar.gz: 1400809d32318726727359177ecf25e6393154dc0a9e5659516b6f9a73c6f87665c138f7c9f71359047818efa5dcf19f1c620faf998bead64edea5d6f17af3db

data/.travis.yml

@@ -7,12 +7,6 @@ cache: bundler
 # See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c
 matrix:
   include:
-    - rvm: 2.1.10
-      os: linux
-    - rvm: 2.2.10
-      os: linux
-    - rvm: 2.2.10
-      os: linux-ppc64le
     - rvm: 2.4.9
       os: linux
     - rvm: 2.4.9
@@ -25,13 +19,12 @@ matrix:
       dist: xenial
     - rvm: 2.6.5
       os: linux
+    - rvm: 2.7.0
+      os: linux
     - rvm: ruby-head
       os: linux
     - rvm: ruby-head
       os: linux-ppc64le
-    - rvm: 2.1.10
-      os: osx
-      osx_image: xcode8.3 # OSX 10.12
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3 # OSX 10.12
@@ -39,9 +32,6 @@ matrix:
       os: osx
       osx_image: xcode8.3 # OSX 10.12
   allow_failures:
-    - rvm: 2.1.10
-      os: osx
-      osx_image: xcode8.3
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
@@ -54,11 +44,6 @@ matrix:
 branches:
   only:
     - master
-    - v0.12
-    - v0.14
-
-before_install:
-  - gem update --system=2.7.8
 
 sudo: false
 dist: trusty # for TLSv1.2 support

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+# v1.9
+
+## Unreleased
+
+### New feature
+
+* New light-weight config reload mechanizm
+  https://github.com/fluent/fluentd/pull/2716
+* Drop ruby 2.1/2.2/2.3 support
+  https://github.com/fluent/fluentd/pull/2750
+
+### Enhancement
+
+* output: Show better message for secondary warning
+  https://github.com/fluent/fluentd/pull/2751
+* Clean up code for ruby 2.7
+  https://github.com/fluent/fluentd/pull/2753
+
+### Bug fixes
+
+* outut/buffer: Fix stage size computation
+  https://github.com/fluent/fluentd/pull/2734
+
 # v1.8
 
 ## Release v1.8.1 - 2019/12/26

data/Gemfile

@@ -2,10 +2,7 @@ source 'https://rubygems.org/'
 
 gemspec
 
-# https://github.com/socketry/async-io/blob/v1.23.1/async-io.gemspec#L21
-if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.3.0')
-  gem 'async-http', '~> 0.42'
-end
+gem 'async-http', '~> 0.42'
 
 local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
 if File.exist?(local_gemfile)

data/README.md

@@ -42,11 +42,11 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 ### Branch
 
 - master: For v1 development.
-- v0.12: For v0.12. This is security maintenance mode. Only security fix is accepted.
+- v0.12: For v0.12. This is deprecated version. we already stopped supporting (See https://www.fluentd.org/blog/drop-schedule-announcement-in-2019).
 
 ### Prerequisites
 
-- Ruby 2.1 or later
+- Ruby 2.4 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.1'
+  gem.required_ruby_version = '>= 2.4'
 
   gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
@@ -27,7 +27,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
-  gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -40,7 +39,7 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
   end
 
-  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("rake", ["~> 12.0"])
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])

data/lib/fluent/command/plugin_generator.rb

@@ -281,7 +281,7 @@ HELP
       @text = ""
       @preamble_source = ""
       @preamble = nil
-      open(LICENSE_URL) do |io|
+      URI.open(LICENSE_URL) do |io|
         @text = io.read
       end
       @preamble_source = @text[/^(\s*Copyright.+)/m, 1]

data/lib/fluent/config.rb

@@ -20,6 +20,25 @@ require 'fluent/configurable'
 
 module Fluent
   module Config
+    # @param config_path [String] config file path
+    # @param encoding [String] encoding of config file
+    # @param additional_config [String] config which is added to last of config body
+    # @param use_v1_config [Bool] config is formatted with v1 or not
+    # @return [Fluent::Config]
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+      config_fname = File.basename(config_path)
+      config_basedir = File.dirname(config_path)
+      config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
+        s = f.read
+        if additional_config
+          c = additional_config.gsub("\\n", "\n")
+          s += "\n#{c}"
+        end
+        s
+      end
+      Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
+    end
+
     def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
       parser = if fname =~ /\.rb$/ || syntax == :ruby
                  :ruby

data/lib/fluent/config/literal_parser.rb

@@ -19,7 +19,7 @@ require 'stringio'
 require 'json'
 require 'yajl'
 require 'socket'
-require 'irb/ruby-lex'  # RubyLex
+require 'ripper'
 
 require 'fluent/config/basic_parser'
 
@@ -155,15 +155,20 @@ module Fluent
       end
 
       def scan_embedded_code
-        rlex = RubyLex.new
-        src = '"#{'+@ss.rest+"\n=end\n}"
+        src = '"#{'+@ss.rest+"\n=begin\n=end\n}"
 
-        input = StringIO.new(src)
-        input.define_singleton_method(:encoding) { external_encoding }
-        rlex.set_input(input)
+        seek = -1
+        while (seek = src.index('}', seek + 1))
+          unless Ripper.sexp(src[0..seek] + '"').nil? # eager parsing until valid expression
+            break
+          end
+        end
+
+        unless seek
+          raise Fluent::ConfigParseError, @ss.rest
+        end
 
-        tk = rlex.token
-        code = src[3,tk.seek-3]
+        code = src[3, seek-3]
 
         if @ss.rest.length < code.length
           @ss.pos += @ss.rest.length

data/lib/fluent/engine.rb

@@ -23,6 +23,7 @@ require 'fluent/time'
 require 'fluent/system_config'
 require 'fluent/plugin'
 require 'fluent/fluent_log_event_router'
+require 'fluent/static_config_analysis'
 
 module Fluent
   class EngineClass
@@ -157,14 +158,47 @@ module Fluent
         raise
       end
 
-      unless @log_event_verbose
-        $log.enable_event(false)
-        @fluent_log_event_router.graceful_stop
+      stop_phase(@root_agent)
+    end
+
+    # @param conf [Fluent::Config]
+    # @param supervisor [Bool]
+    # @reutrn nil
+    def reload_config(conf, supervisor: false)
+      # configure first to reduce down time while restarting
+      new_agent = RootAgent.new(log: log, system_config: @system_config)
+      ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
+
+      ret.all_plugins.each do |plugin|
+        if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
+          raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+        end
       end
-      $log.info "shutting down fluentd worker", worker: worker_id
-      shutdown
 
-      @fluent_log_event_router.stop
+      # Assign @root_agent to new root_agent
+      # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
+      old_agent, @root_agent = @root_agent, new_agent
+      begin
+        @root_agent.configure(conf)
+      rescue
+        @root_agent = old_agent
+        raise
+      end
+
+      unless @suppress_config_dump
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+      end
+
+      # supervisor doesn't handle actual data. so the following code is unnecessary.
+      if supervisor
+        old_agent.shutdown      # to close thread created in #configure
+        return
+      end
+
+      stop_phase(old_agent)
+
+      $log.info 'restart fluentd worker', worker: worker_id
+      start_phase(new_agent)
     end
 
     def stop
@@ -189,12 +223,29 @@ module Fluent
     end
 
     private
-    def start
+
+    def stop_phase(root_agent)
+      unless @log_event_verbose
+        $log.enable_event(false)
+        @fluent_log_event_router.graceful_stop
+      end
+      $log.info 'shutting down fluentd worker', worker: worker_id
+      root_agent.shutdown
+
+      @fluent_log_event_router.stop
+    end
+
+    def start_phase(root_agent)
+      @fluent_log_event_router = FluentLogEventRouter.build(root_agent)
+      if @fluent_log_event_router.emittable?
+        $log.enable_event(true)
+      end
+
       @root_agent.start
     end
 
-    def shutdown
-      @root_agent.shutdown
+    def start
+      @root_agent.start
     end
   end
 

data/lib/fluent/plugin/base.rb

@@ -187,6 +187,11 @@ module Fluent
         #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788
         "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
       end
+
+      def reloadable_plugin?
+        # Engine can't capture all class variables. so it's forbbiden to use class variables in each plugins if enabling reload.
+        self.class.class_variables.empty?
+      end
     end
   end
 end

data/lib/fluent/plugin/buf_file.rb

@@ -19,6 +19,7 @@ require 'fileutils'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/file_chunk'
 require 'fluent/system_config'
+require 'fluent/variable_store'
 
 module Fluent
   module Plugin
@@ -43,19 +44,20 @@ module Fluent
       config_param :file_permission, :string, default: nil # '0644'
       config_param :dir_permission,  :string, default: nil # '0755'
 
-      @@buffer_paths = {}
-
       def initialize
         super
         @symlink_path = nil
         @multi_workers_available = false
         @additional_resume_path = nil
         @buffer_path = nil
+        @variable_store = nil
       end
 
       def configure(conf)
         super
 
+        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
+
         multi_workers_configured = owner.system_config.workers > 1 ? true : false
 
         using_plugin_root_dir = false
@@ -69,13 +71,13 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !called_in_test?
-          type_using_this_path = @@buffer_paths[@path]
+        if @variable_store.has_key?(@path) && !called_in_test?
+          type_using_this_path = @variable_store[@path]
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
         @buffer_path = @path
-        @@buffer_paths[@buffer_path] = type_of_owner
+        @variable_store[@buffer_path] = type_of_owner
 
         specified_directory_exists = File.exist?(@path) && File.directory?(@path)
         unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
@@ -125,7 +127,9 @@ module Fluent
       end
 
       def stop
-        @@buffer_paths.delete(@buffer_path)
+        if @variable_store
+          @variable_store.delete(@buffer_path)
+        end
 
         super
       end

data/lib/fluent/plugin/buf_file_single.rb

@@ -19,6 +19,7 @@ require 'fileutils'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/file_single_chunk'
 require 'fluent/system_config'
+require 'fluent/variable_store'
 
 module Fluent
   module Plugin
@@ -48,18 +49,19 @@ module Fluent
       desc 'The permission of chunk directory. If no specified, <system> setting or 0755 is used'
       config_param :dir_permission, :string, default: nil
 
-      @@buffer_paths = {}
-
       def initialize
         super
 
         @multi_workers_available = false
         @additional_resume_path = nil
+        @variable_store = nil
       end
 
       def configure(conf)
         super
 
+        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file_single)
+
         if @chunk_format == :auto
           @chunk_format = owner.formatted_to_msgpack_binary? ? :msgpack : :text
         end
@@ -117,12 +119,12 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !called_in_test?
-          type_using_this_path = @@buffer_paths[@path]
+        if @variable_store.has_key?(@path) && !called_in_test?
+          type_using_this_path = @variable_store[@path]
           raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
-        @@buffer_paths[@path] = type_of_owner
+        @variable_store[@path] = type_of_owner
         @dir_permission = if @dir_permission
                             @dir_permission.to_i(8)
                           else
@@ -145,7 +147,9 @@ module Fluent
       end
 
       def stop
-        @@buffer_paths.delete(@path)
+        if @variable_store
+          @variable_store.delete(@path)
+        end
 
         super
       end

data/lib/fluent/plugin/buffer.rb

@@ -266,10 +266,10 @@ module Fluent
 
         log.on_trace { log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size }
 
-        staged_bytesize = 0
         operated_chunks = []
         unstaged_chunks = {} # metadata => [chunk, chunk, ...]
         chunks_to_enqueue = []
+        staged_bytesizes_by_chunk = {}
 
         begin
           # sort metadata to get lock of chunks in same order with other threads
@@ -279,7 +279,13 @@ module Fluent
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
               if chunk.staged?
-                staged_bytesize += adding_bytesize
+                #
+                # https://github.com/fluent/fluentd/issues/2712
+                # write_once is supposed to write to a chunk only once
+                # but this block **may** run multiple times from write_step_by_step and previous write may be rollbacked
+                # So we should be counting the stage_size only for the last successful write
+                #
+                staged_bytesizes_by_chunk[chunk] = adding_bytesize
               elsif chunk.unstaged?
                 unstaged_chunks[metadata] ||= []
                 unstaged_chunks[metadata] << chunk
@@ -326,27 +332,37 @@ module Fluent
 
           # All locks about chunks are released.
 
-          synchronize do
-            # At here, staged chunks may be enqueued by other threads.
-            @stage_size += staged_bytesize
-
-            chunks_to_enqueue.each do |c|
-              if c.staged? && (enqueue || chunk_size_full?(c))
-                m = c.metadata
-                enqueue_chunk(m)
-                if unstaged_chunks[m]
-                  u = unstaged_chunks[m].pop
+          #
+          # Now update the stage, stage_size with proper locking
+          # FIX FOR stage_size miscomputation - https://github.com/fluent/fluentd/issues/2712
+          #
+          staged_bytesizes_by_chunk.each do |chunk, bytesize|
+            chunk.synchronize do
+              synchronize { @stage_size += bytesize }
+              log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
+            end
+          end
+
+          chunks_to_enqueue.each do |c|
+            if c.staged? && (enqueue || chunk_size_full?(c))
+              m = c.metadata
+              enqueue_chunk(m)
+              if unstaged_chunks[m]
+                u = unstaged_chunks[m].pop
+                u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)
-                    @stage[m] = u.staged!
-                    @stage_size += u.bytesize
+                    synchronize {
+                      @stage[m] = u.staged!
+                      @stage_size += u.bytesize
+                    }
                   end
                 end
-              elsif c.unstaged?
-                enqueue_unstaged_chunk(c)
-              else
-                # previously staged chunk is already enqueued, closed or purged.
-                # no problem.
               end
+            elsif c.unstaged?
+              enqueue_unstaged_chunk(c)
+            else
+              # previously staged chunk is already enqueued, closed or purged.
+              # no problem.
             end
           end