fluentd 1.12.0.rc2 → 1.12.4

data/fluentd.gemspec

@@ -28,12 +28,14 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -44,11 +46,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
-  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("rr", ["~> 3.0"])
   gem.add_development_dependency("timecop", ["~> 0.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
-  gem.add_development_dependency("ext_monitor", [">= 0.1.2", "< 0.2"])
-  gem.add_development_dependency("async-http")
+  gem.add_development_dependency("async-http", ">= 0.50.0")
 end

data/lib/fluent/command/bundler_injection.rb

@@ -40,6 +40,6 @@ else
     File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
   ] + ARGV
 
-  exec *cmdline
+  exec(*cmdline)
   exit! 127
 end

data/lib/fluent/command/ca_generate.rb

@@ -75,6 +75,8 @@ HELP
 
       digest = OpenSSL::Digest::SHA256.new
 
+      factory = OpenSSL::X509::ExtensionFactory.new
+
       cert = OpenSSL::X509::Certificate.new
       cert.not_before = Time.at(0)
       cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
@@ -82,7 +84,7 @@ HELP
       cert.serial = 1
       cert.issuer = issuer
       cert.subject = subject
-      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+      cert.add_extension(factory.create_extension('basicConstraints', 'CA:TRUE'))
       cert.sign(key, digest)
 
       return cert, key
@@ -111,8 +113,9 @@ HELP
       cert.issuer = issuer
       cert.subject = subject
 
-      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-      cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+      factory = OpenSSL::X509::ExtensionFactory.new
+      server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))
+      server_cert.add_extension(factory.create_extension('nsCertType', 'server'))
 
       cert.sign ca_key, digest
 

data/lib/fluent/command/cat.rb

@@ -101,7 +101,6 @@ rescue
   usage $!.to_s
 end
 
-require 'thread'
 require 'socket'
 require 'yajl'
 require 'msgpack'

data/lib/fluent/command/fluentd.rb

@@ -163,6 +163,10 @@ op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
   opts[:conf_encoding] = s
 }
 
+op.on('--disable-shared-socket', "Don't open shared socket for multiple workers") { |b|
+  opts[:disable_shared_socket] = b
+}
+
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library

data/lib/fluent/command/plugin_config_formatter.rb

@@ -44,6 +44,7 @@ class FluentPluginConfigFormatter
     @verbose = false
     @libs = []
     @plugin_dirs = []
+    @table = false
     @options = {}
 
     prepare_option_parser
@@ -162,9 +163,20 @@ class FluentPluginConfigFormatter
     else
       sections, params = base_section.partition {|_name, value| value[:section] }
     end
+    if @table && (not params.empty?)
+      dumped << "### Configuration\n\n"
+      dumped << "|parameter|type|description|default|\n"
+      dumped << "|---|---|---|---|\n"
+    end
     params.each do |name, config|
       next if name == :section
-      template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
+      template_name = if @compact
+                        "param.md-compact.erb"
+                      elsif @table
+                        "param.md-table.erb"
+                      else
+                        "param.md.erb"
+                      end
       template = template_path(template_name).read
       dumped <<
         if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
@@ -257,6 +269,9 @@ BANNER
     @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
       @plugin_dirs << s
     end
+    @parser.on("-t", "--table", "Use table syntax to dump parameters") do
+      @table = true
+    end
   end
 
   def parse_options!

data/lib/fluent/command/plugin_generator.rb

@@ -105,7 +105,7 @@ Generate a project skeleton for creating a Fluentd plugin
 
 Arguments:
 \ttype: #{SUPPORTED_TYPES.join(",")}
-\tname: Your plugin name
+\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
 
 Options:
 BANNER
@@ -151,6 +151,36 @@ BANNER
     underscore_name
   end
 
+  def gem_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile"))
+  end
+
+  def lock_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile.lock"))
+  end
+
+  def locked_gem_version(gem_name)
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+  end
+
+  def rake_version
+    locked_gem_version("rake")
+  end
+
+  def test_unit_version
+    locked_gem_version("test-unit")
+  end
+
+  def bundler_version
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.bundler_version.version
+  end
+
   def class_name
     "#{capitalized_name}#{type.capitalize}"
   end

data/lib/fluent/compat/parser.rb

@@ -244,10 +244,10 @@ module Fluent
         end
 
         def convert_value_to_nil(value)
-          if value and @null_empty_string
+          if value && @null_empty_string
             value = (value == '') ? nil : value
           end
-          if value and @null_value_pattern
+          if value && @null_value_pattern
             value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
           end
           value

data/lib/fluent/config/section.rb

@@ -179,7 +179,7 @@ module Fluent
             end
 
             if section_params[varname].nil?
-              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
                 logger.error "config error in:\n#{conf}" if logger
                 raise ConfigError, "'#{name}' parameter is required but nil is specified"
               end
@@ -247,7 +247,7 @@ module Fluent
       def self.check_unused_section(proxy, conf, plugin_class)
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
-          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
           next if e.unused_in && e.unused_in.empty? # the section is used at least once
 
           if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }

data/lib/fluent/config/types.rb

@@ -186,7 +186,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -213,7 +213,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
               else
                 val
               end

data/lib/fluent/event.rb

@@ -254,19 +254,9 @@ module Fluent
     end
 
     def each(unpacker: nil, &block)
-      if @unpacked_times
-        @unpacked_times.each_with_index do |time, i|
-          block.call(time, @unpacked_records[i])
-        end
-      else
-        @unpacked_times = []
-        @unpacked_records = []
-        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
-          @unpacked_times << time
-          @unpacked_records << record
-          block.call(time, record)
-        end
-        @size = @unpacked_times.size
+      ensure_unpacked!(unpacker: unpacker)
+      @unpacked_times.each_with_index do |time, i|
+        block.call(time, @unpacked_records[i])
       end
       nil
     end

data/lib/fluent/load.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'

data/lib/fluent/plugin/file_wrapper.rb

@@ -46,6 +46,42 @@ module Fluent
     end
   end
 
+  class Win32Error < StandardError
+    require 'windows/error'
+    include Windows::Error
+
+    attr_reader :errcode, :msg
+
+    def initialize(errcode, msg = nil)
+      @errcode = errcode
+      @msg = msg
+    end
+
+    def format_english_message(errcode)
+      buf = 0.chr * 260
+      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
+      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
+      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
+      buf.force_encoding(Encoding.default_external).strip
+    end
+
+    def to_s
+      msg = super
+      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
+      msg << " - #{@msg}" if @msg
+      msg
+    end
+
+    def inspect
+      "#<#{to_s}>"
+    end
+
+    def ==(other)
+      return false if other.class != Win32Error
+      @errcode == other.errcode && @msg == other.msg
+    end
+  end
+
   # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
     require 'windows/file'
@@ -77,11 +113,11 @@ module Fluent
       @file_handle = CreateFile.call(@path, access, sharemode,
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
-        err = GetLastError.call
+        err = Win32::API.last_error
         if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
-          raise SystemCallError.new(2)
+          raise Errno::ENOENT
         end
-        raise SystemCallError.new(err)
+        raise Win32Error.new(err, path)
       end
     end
 

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -27,14 +27,14 @@ module Fluent
 
       config_param :delimiter, :string, default: "\t".freeze
       config_param :label_delimiter, :string, default: ":".freeze
+      config_param :replacement, :string, default: " ".freeze
       config_param :add_newline, :bool, default: true
 
-      # TODO: escaping for \t in values
       def format(tag, time, record)
         formatted = ""
         record.each do |label, value|
           formatted << @delimiter if formatted.length.nonzero?
-          formatted << "#{label}#{@label_delimiter}#{value}"
+          formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
         end
         formatted << @newline if @add_newline
         formatted

data/lib/fluent/plugin/in_http.rb

@@ -503,7 +503,7 @@ module Fluent::Plugin
         # For every incoming request, we check if we have some CORS
         # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?('*') or include_cors_allow_origin
+          unless @cors_allow_origins.include?('*') || include_cors_allow_origin
             send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -338,7 +338,7 @@ module Fluent::Plugin
         obj.merge!(pe.statistics['output'] || {})
       end
 
-      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
+      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
 
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]

data/lib/fluent/plugin/in_tail.rb

@@ -313,11 +313,17 @@ module Fluent::Plugin
       (paths - excluded).select { |path|
         FileTest.exist?(path)
       }.each { |path|
-        target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
-        if @follow_inodes
-          hash[target_info.ino] = target_info
-        else
-          hash[target_info.path] = target_info
+        # Even we just checked for existence, there is a race condition here as
+        # of which stat() might fail with ENOENT. See #3224.
+        begin
+          target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+          if @follow_inodes
+            hash[target_info.ino] = target_info
+          else
+            hash[target_info.path] = target_info
+          end
+        rescue Errno::ENOENT
+          $log.warn "expand_paths: stat() for #{path} failed with ENOENT. Skip file."
         end
       }
       hash
@@ -406,8 +412,16 @@ module Fluent::Plugin
           log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
           next
         end
-        target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
-        @tails[target_info] = tw
+
+        begin
+          target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
+          @tails[target_info] = tw
+        rescue Errno::ENOENT
+          $log.warn "stat() for #{target_info.path} failed with ENOENT. Drop tail watcher for now."
+          # explicitly detach and unwatch watcher `tw`.
+          tw.unwatched = true
+          detach_watcher(tw, target_info.ino, false)
+        end
       }
     end
 
@@ -767,16 +781,22 @@ module Fluent::Plugin
           end
 
           if watcher_needs_update
-            # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-            # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-            if stat
-              target_info = TargetInfo.new(@path, stat.ino)
-              if @follow_inodes
-                # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+            if @follow_inodes
+              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
+              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
+              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              if stat
+                target_info = TargetInfo.new(@path, stat)
                 @update_watcher.call(target_info, @pe)
-              else
-                @update_watcher.call(target_info, swap_state(@pe))
               end
+            else
+              # Permit to handle if stat is nil (file not present).
+              # If a file is mv-ed and a new file is created during
+              # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
+              # and `#stop_watchers()` for the path because `target_paths_hash`
+              # always contains the path.
+              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
+              @update_watcher.call(target_info, swap_state(@pe))
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -248,6 +248,20 @@ module Fluent::Plugin
       end
     end
 
-    TargetInfo = Struct.new(:path, :ino)
+    TargetInfo = Struct.new(:path, :ino) do
+      def ==(other)
+        return false unless other.is_a?(TargetInfo)
+        self.path == other.path
+      end
+
+      def hash
+        self.path.hash
+      end
+
+      def eql?(other)
+        return false unless other.is_a?(TargetInfo)
+        self.path == other.path
+      end
+    end
   end
 end