fluentd 1.12.2-x64-mingw32 → 1.13.2-x64-mingw32

data/MAINTAINERS.md

@@ -2,7 +2,7 @@
 
 - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
 - [Okkez](https://github.com/okkez)
-- [Hiroshi Hatake](https://github.com/cosmo0920), [ClearCode](https://www.clear-code.com/)
+- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://www.calyptia.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
 - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
 - [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)

data/README.md

@@ -1,8 +1,11 @@
 Fluentd: Open-Source Log Collector
 ===================================
-Travis CI:
+GitHub Actions:
 
-[<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
+[![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
+[![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
+[![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
+[![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
@@ -22,9 +25,9 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 
 Use Case | Description | Diagram
 -------- | ------------|:---------:
-Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/assets/img/recipes/elasticsearch-s3-fluentd.png" height="150"/>
+Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
 Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
-Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/assets/img/datasources/asynchronous_logging.png" height="150"/>
+Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
 
 ## Quick Start
 
@@ -74,7 +77,7 @@ You can run specified test via `TEST` environment variable:
 - Website: https://www.fluentd.org/
 - Documentation: https://docs.fluentd.org/
 - Project repository: https://github.com/fluent
-- Discussion: https://groups.google.com/group/fluentd
+- Discussion: https://discuss.fluentd.org/
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)

data/bin/fluentd

@@ -1,7 +1,14 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
 
-RubyVM::InstructionSequence.compile_option = {trace_instruction: false} rescue nil
+# In recent CRuby, RubyVM::InstructionSequence is always defined.
+# This optimization aims to reduce memory (avg 0.5 megabytes) at startup.
+# ref: http://atdot.net/~ko1/activities/2017_fukuoka_rubykaigi_02.pdf
+# But other implementation such as truffleruby does not provide such
+# class.
+if defined?(RubyVM::InstructionSequence)
+  RubyVM::InstructionSequence.compile_option = {trace_instruction: false}
+end
 
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))

data/example/counter.conf

@@ -3,7 +3,7 @@
     scope server1
     bind 127.0.0.1
     port 24321
-    path tmp/back
+    backup_path tmp/back
   </counter_server>
 </system>
 

data/fluentd.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
@@ -34,7 +34,8 @@ Gem::Specification.new do |gem|
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -45,7 +46,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
-  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("rr", ["~> 3.0"])
   gem.add_development_dependency("timecop", ["~> 0.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])

data/lib/fluent/command/cat.rb

@@ -152,14 +152,30 @@ class Writer
     super()
   end
 
+  def secondary_record?(record)
+    record.class != Hash &&
+      record.size == 2 &&
+      record.first.class == Fluent::EventTime &&
+      record.last.class == Hash
+  end
+
   def write(record)
-    if record.class != Hash
-      raise ArgumentError, "Input must be a map (got #{record.class})"
+    unless secondary_record?(record)
+      if record.class != Hash
+        raise ArgumentError, "Input must be a map (got #{record.class})"
+      end
     end
 
     time = Fluent::EventTime.now
     time = time.to_i if @time_as_integer
-    entry = [time, record]
+    entry = if secondary_record?(record)
+              # Even though secondary contains Fluent::EventTime in record,
+              # fluent-cat just ignore it and set Fluent::EventTime.now instead.
+              # This specification is adopted to keep consistency.
+              [time, record.last]
+            else
+              [time, record]
+            end
     synchronize {
       unless write_impl([entry])
         # write failed

data/lib/fluent/command/fluentd.rb

@@ -85,9 +85,8 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
-ROTATE_AGE = %w(daily weekly monthly)
 op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
-  if ROTATE_AGE.include?(age)
+  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
     opts[:log_rotate_age] = age
   else
     begin

data/lib/fluent/command/plugin_generator.rb

@@ -27,7 +27,7 @@ class FluentPluginGenerator
   attr_reader :type, :name
   attr_reader :license_name
 
-  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
 
   def initialize(argv = ARGV)
     @argv = argv
@@ -105,7 +105,7 @@ Generate a project skeleton for creating a Fluentd plugin
 
 Arguments:
 \ttype: #{SUPPORTED_TYPES.join(",")}
-\tname: Your plugin name
+\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
 
 Options:
 BANNER
@@ -151,6 +151,46 @@ BANNER
     underscore_name
   end
 
+  def gem_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile"))
+  end
+
+  def lock_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile.lock"))
+  end
+
+  def locked_gem_version(gem_name)
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name(gem_name).version.version
+    end
+  end
+
+  def rake_version
+    locked_gem_version("rake")
+  end
+
+  def test_unit_version
+    locked_gem_version("test-unit")
+  end
+
+  def bundler_version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.bundler_version.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name("bundler").version.version
+    end
+  end
+
   def class_name
     "#{capitalized_name}#{type.capitalize}"
   end

data/lib/fluent/config.rb

@@ -62,7 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
-        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file") if $log
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/config/section.rb

@@ -50,6 +50,11 @@ module Fluent
         "<Fluent::Config::Section #{@params.to_json}>"
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       def nil?
         false
       end

data/lib/fluent/config/types.rb

@@ -20,6 +20,10 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
     def self.size_value(str, opts = {}, name = nil)
       return nil if str.nil?
 
@@ -104,6 +108,16 @@ module Fluent
       Config.string_value(val, opts, name)
     }
 
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
     def self.enum_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
@@ -176,6 +190,7 @@ module Fluent
         when :bool then Config.bool_value(value, opts, name)
         when :time then Config.time_value(value, opts, name)
         when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end

data/lib/fluent/config/v1_parser.rb

@@ -37,6 +37,7 @@ module Fluent
         super(strscan, eval_context)
         @include_basepath = include_basepath
         @fname = fname
+        @logger = defined?($log) ? $log : nil
       end
 
       def parse!
@@ -99,7 +100,7 @@ module Fluent
 
           elsif root_element && skip(/(\@include|include)#{SPACING}/)
             if !prev_match.start_with?('@')
-              $log.warn "'include' is deprecated. Use '@include' instead"
+              @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
             end
             parse_include(attrs, elems)
 
@@ -123,7 +124,7 @@ module Fluent
                     parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
                   else
                     # TODO: This is for backward compatibility. It will throw an error in the future.
-                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
+                    @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
                   end
                 end
 

data/lib/fluent/env.rb

@@ -15,13 +15,14 @@
 #
 
 require 'serverengine/utils'
+require 'fluent/oj_options'
 
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
-  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
+  DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
 

data/lib/fluent/log.rb

@@ -49,6 +49,7 @@ module Fluent
     LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
 
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
+    LOG_ROTATE_AGE = %w(daily weekly monthly)
 
     def self.str_to_level(log_level_str)
       case log_level_str.downcase

data/lib/fluent/oj_options.rb

@@ -0,0 +1,62 @@
+require 'fluent/config/types'
+
+module Fluent
+  class OjOptions
+    OPTIONS = {
+      'bigdecimal_load': :symbol,
+      'max_nesting': :integer,
+      'mode': :symbol,
+      'use_to_json': :bool
+    }
+
+    ALLOWED_VALUES = {
+      'bigdecimal_load': %i[bigdecimal float auto],
+      'mode': %i[strict null compat json rails object custom]
+    }
+
+    DEFAULTS = {
+      'bigdecimal_load': :float,
+      'mode': :compat,
+      'use_to_json': true
+    }
+
+    @@available = false
+
+    def self.available?
+      @@available
+    end
+
+    def self.load_env
+      options = self.get_options
+      begin
+        require 'oj'
+        Oj.default_options = options
+        @@available = true
+      rescue LoadError
+        @@available = false
+      end
+      options
+    end
+
+    private
+
+    def self.get_options
+      options = {}
+      DEFAULTS.each { |key, value| options[key] = value }
+
+      OPTIONS.each do |key, type|
+        env_value = ENV["FLUENT_OJ_OPTION_#{key.upcase}"]
+        next if env_value.nil?
+
+        cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true })
+        next if cast_value.nil?
+
+        next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value)
+
+        options[key.to_sym] = cast_value
+      end
+
+      options
+    end
+  end
+end

data/lib/fluent/plugin/file_wrapper.rb

@@ -46,6 +46,48 @@ module Fluent
     end
   end
 
+  class Win32Error < StandardError
+    require 'windows/error'
+    include Windows::Error
+
+    attr_reader :errcode, :msg
+
+    WSABASEERR = 10000
+
+    def initialize(errcode, msg = nil)
+      @errcode = errcode
+      @msg = msg
+    end
+
+    def format_english_message(errcode)
+      buf = 0.chr * 260
+      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
+      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
+      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
+      buf.force_encoding(Encoding.default_external).strip
+    end
+
+    def to_s
+      msg = super
+      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
+      msg << " - #{@msg}" if @msg
+      msg
+    end
+
+    def inspect
+      "#<#{to_s}>"
+    end
+
+    def ==(other)
+      return false if other.class != Win32Error
+      @errcode == other.errcode && @msg == other.msg
+    end
+
+    def wsaerr?
+      @errcode >= WSABASEERR
+    end
+  end
+
   # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
     require 'windows/file'
@@ -77,11 +119,14 @@ module Fluent
       @file_handle = CreateFile.call(@path, access, sharemode,
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
-        err = GetLastError.call
-        if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
-          raise SystemCallError.new(2)
+        win32err = Win32Error.new(Win32::API.last_error, path)
+        errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
+        if errno == Errno::EINVAL::Errno || win32err.wsaerr?
+          # maybe failed to map
+          raise win32err
+        else
+          raise SystemCallError.new(win32err.message, errno)
         end
-        raise SystemCallError.new(err)
       end
     end