fluentd 1.11.5-x86-mingw32 → 1.12.4-x86-mingw32

data/lib/fluent/command/fluentd.rb

@@ -163,6 +163,10 @@ op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
   opts[:conf_encoding] = s
 }
 
+op.on('--disable-shared-socket', "Don't open shared socket for multiple workers") { |b|
+  opts[:disable_shared_socket] = b
+}
+
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library

data/lib/fluent/command/plugin_config_formatter.rb

@@ -29,7 +29,8 @@ class FluentPluginConfigFormatter
   AVAILABLE_FORMATS = [:markdown, :txt, :json]
   SUPPORTED_TYPES = [
     "input", "output", "filter",
-    "buffer", "parser", "formatter", "storage"
+    "buffer", "parser", "formatter", "storage",
+    "service_discovery"
   ]
 
   DOCS_BASE_URL = "https://docs.fluentd.org/v/1.0"
@@ -43,6 +44,7 @@ class FluentPluginConfigFormatter
     @verbose = false
     @libs = []
     @plugin_dirs = []
+    @table = false
     @options = {}
 
     prepare_option_parser
@@ -161,9 +163,20 @@ class FluentPluginConfigFormatter
     else
       sections, params = base_section.partition {|_name, value| value[:section] }
     end
+    if @table && (not params.empty?)
+      dumped << "### Configuration\n\n"
+      dumped << "|parameter|type|description|default|\n"
+      dumped << "|---|---|---|---|\n"
+    end
     params.each do |name, config|
       next if name == :section
-      template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
+      template_name = if @compact
+                        "param.md-compact.erb"
+                      elsif @table
+                        "param.md-table.erb"
+                      else
+                        "param.md.erb"
+                      end
       template = template_path(template_name).read
       dumped <<
         if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
@@ -256,6 +269,9 @@ BANNER
     @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
       @plugin_dirs << s
     end
+    @parser.on("-t", "--table", "Use table syntax to dump parameters") do
+      @table = true
+    end
   end
 
   def parse_options!

data/lib/fluent/command/plugin_generator.rb

@@ -105,7 +105,7 @@ Generate a project skeleton for creating a Fluentd plugin
 
 Arguments:
 \ttype: #{SUPPORTED_TYPES.join(",")}
-\tname: Your plugin name
+\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
 
 Options:
 BANNER
@@ -151,6 +151,36 @@ BANNER
     underscore_name
   end
 
+  def gem_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile"))
+  end
+
+  def lock_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile.lock"))
+  end
+
+  def locked_gem_version(gem_name)
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+  end
+
+  def rake_version
+    locked_gem_version("rake")
+  end
+
+  def test_unit_version
+    locked_gem_version("test-unit")
+  end
+
+  def bundler_version
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.bundler_version.version
+  end
+
   def class_name
     "#{capitalized_name}#{type.capitalize}"
   end

data/lib/fluent/compat/parser.rb

@@ -244,10 +244,10 @@ module Fluent
         end
 
         def convert_value_to_nil(value)
-          if value and @null_empty_string
+          if value && @null_empty_string
             value = (value == '') ? nil : value
           end
-          if value and @null_value_pattern
+          if value && @null_value_pattern
             value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
           end
           value

data/lib/fluent/config/section.rb

@@ -179,7 +179,7 @@ module Fluent
             end
 
             if section_params[varname].nil?
-              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
                 logger.error "config error in:\n#{conf}" if logger
                 raise ConfigError, "'#{name}' parameter is required but nil is specified"
               end
@@ -247,7 +247,7 @@ module Fluent
       def self.check_unused_section(proxy, conf, plugin_class)
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
-          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
           next if e.unused_in && e.unused_in.empty? # the section is used at least once
 
           if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }

data/lib/fluent/config/types.rb

@@ -186,7 +186,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -213,7 +213,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
               else
                 val
               end

data/lib/fluent/env.rb

@@ -28,4 +28,8 @@ module Fluent
   def self.windows?
     ServerEngine.windows?
   end
+
+  def self.linux?
+    /linux/ === RUBY_PLATFORM
+  end
 end

data/lib/fluent/event.rb

@@ -254,19 +254,9 @@ module Fluent
     end
 
     def each(unpacker: nil, &block)
-      if @unpacked_times
-        @unpacked_times.each_with_index do |time, i|
-          block.call(time, @unpacked_records[i])
-        end
-      else
-        @unpacked_times = []
-        @unpacked_records = []
-        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
-          @unpacked_times << time
-          @unpacked_records << record
-          block.call(time, record)
-        end
-        @size = @unpacked_times.size
+      ensure_unpacked!(unpacker: unpacker)
+      @unpacked_times.each_with_index do |time, i|
+        block.call(time, @unpacked_records[i])
       end
       nil
     end

data/lib/fluent/load.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'

data/lib/fluent/plugin.rb

@@ -121,6 +121,11 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    class << self
+      # This should be defined for fluent-plugin-config-formatter type arguments.
+      alias_method :new_service_discovery, :new_sd
+    end
+
     def self.new_parser(type, parent: nil)
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly

data/lib/fluent/plugin/buffer.rb

@@ -143,33 +143,14 @@ module Fluent
           end
         end
 
-        # timekey should be unixtime as usual.
-        # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
-        # We should check object_id stability to use object_id as optimization for comparing operations.
-        # e.g.)
-        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
-        # => 1596187800
-        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
-        # => true
-        def self.enable_optimize?
-          a1 = 2**30 - 1
-          a2 = 2**30 - 1
-          b1 = 2**62 - 1
-          b2 = 2**62 - 1
-          (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
-        end
-
         # This is an optimization code. Current Struct's implementation is comparing all data.
         # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
-        # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
-        # This difference causes flood of buffer files.
-        # So, this optimization should be enabled on `enable_optimize?` as true platforms.
         def hash
-          timekey.object_id
-        end if enable_optimize?
+          timekey.hash
+        end
       end
 
       # for tests

data/lib/fluent/plugin/file_wrapper.rb

@@ -46,6 +46,42 @@ module Fluent
     end
   end
 
+  class Win32Error < StandardError
+    require 'windows/error'
+    include Windows::Error
+
+    attr_reader :errcode, :msg
+
+    def initialize(errcode, msg = nil)
+      @errcode = errcode
+      @msg = msg
+    end
+
+    def format_english_message(errcode)
+      buf = 0.chr * 260
+      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
+      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
+      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
+      buf.force_encoding(Encoding.default_external).strip
+    end
+
+    def to_s
+      msg = super
+      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
+      msg << " - #{@msg}" if @msg
+      msg
+    end
+
+    def inspect
+      "#<#{to_s}>"
+    end
+
+    def ==(other)
+      return false if other.class != Win32Error
+      @errcode == other.errcode && @msg == other.msg
+    end
+  end
+
   # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
     require 'windows/file'
@@ -77,11 +113,11 @@ module Fluent
       @file_handle = CreateFile.call(@path, access, sharemode,
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
-        err = GetLastError.call
+        err = Win32::API.last_error
         if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
-          raise SystemCallError.new(2)
+          raise Errno::ENOENT
         end
-        raise SystemCallError.new(err)
+        raise Win32Error.new(err, path)
       end
     end
 

data/lib/fluent/plugin/formatter.rb

@@ -63,9 +63,9 @@ module Fluent
           super
           @newline = case newline
                      when :lf
-                       "\n"
+                       "\n".freeze
                      when :crlf
-                       "\r\n"
+                       "\r\n".freeze
                      end
         end
       end

data/lib/fluent/plugin/formatter_csv.rb

@@ -27,7 +27,7 @@ module Fluent
       helpers :record_accessor
 
       config_param :delimiter, default: ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
+        ['\t', 'TAB'].include?(val) ? "\t".freeze : val.freeze
       end
       config_param :force_quotes, :bool, default: true
       # "array" looks good for type of :fields, but this implementation removes tailing comma

data/lib/fluent/plugin/formatter_hash.rb

@@ -27,7 +27,7 @@ module Fluent
 
       def format(tag, time, record)
         line = record.to_s
-        line << @newline.freeze if @add_newline
+        line << @newline if @add_newline
         line
       end
     end

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -25,18 +25,18 @@ module Fluent
 
       # http://ltsv.org/
 
-      config_param :delimiter, :string, default: "\t"
-      config_param :label_delimiter, :string, default: ":"
+      config_param :delimiter, :string, default: "\t".freeze
+      config_param :label_delimiter, :string, default: ":".freeze
+      config_param :replacement, :string, default: " ".freeze
       config_param :add_newline, :bool, default: true
 
-      # TODO: escaping for \t in values
       def format(tag, time, record)
         formatted = ""
         record.each do |label, value|
           formatted << @delimiter if formatted.length.nonzero?
-          formatted << "#{label}#{@label_delimiter}#{value}"
+          formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
         end
-        formatted << @newline.freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -29,9 +29,9 @@ module Fluent
       config_param :output_tag, :bool, default: true
       config_param :delimiter, default: "\t" do |val|
         case val
-        when /SPACE/i then ' '
-        when /COMMA/i then ','
-        else "\t"
+        when /SPACE/i then ' '.freeze
+        when /COMMA/i then ','.freeze
+        else "\t".freeze
         end
       end
       config_set_default :time_type, :string

data/lib/fluent/plugin/formatter_single_value.rb

@@ -23,12 +23,12 @@ module Fluent
 
       Plugin.register_formatter('single_value', self)
 
-      config_param :message_key, :string, default: 'message'
+      config_param :message_key, :string, default: 'message'.freeze
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
-        text << @newline.freeze if @add_newline
+        text << @newline if @add_newline
         text
       end
     end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -26,13 +26,13 @@ module Fluent
       desc 'Field names included in each lines'
       config_param :keys, :array, value_type: :string
       desc 'The delimiter character (or string) of TSV values'
-      config_param :delimiter, :string, default: "\t"
+      config_param :delimiter, :string, default: "\t".freeze
       desc 'The parameter to enable writing to new lines'
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
-        formatted << @newline.freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/in_http.rb

@@ -80,6 +80,8 @@ module Fluent::Plugin
     config_param :use_204_response, :bool, default: false
     desc 'Dump error log or not'
     config_param :dump_error_log, :bool, default: true
+    desc 'Add QUERY_ prefix query params to record'
+    config_param :add_query_params, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -277,7 +279,7 @@ module Fluent::Plugin
     private
 
     def on_server_connect(conn)
-      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
+      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -326,6 +328,14 @@ module Fluent::Plugin
         }
       end
 
+      if @add_query_params
+        params.each_pair { |k, v|
+          if k.start_with?("QUERY_".freeze)
+            record[k] = v
+          end
+        }
+      end
+
       if @add_remote_addr
         record['REMOTE_ADDR'] = params['REMOTE_ADDR']
       end
@@ -346,7 +356,7 @@ module Fluent::Plugin
     class Handler
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
         @io = io
         @km = km
         @callback = callback
@@ -356,6 +366,7 @@ module Fluent::Plugin
         @log = log
         @cors_allow_origins = cors_allow_origins
         @idle = 0
+        @add_query_params = add_query_params
         @km.add(self)
 
         @remote_port, @remote_addr = io.remote_port, io.remote_addr
@@ -492,7 +503,7 @@ module Fluent::Plugin
         # For every incoming request, we check if we have some CORS
         # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?('*') or include_cors_allow_origin
+          unless @cors_allow_origins.include?('*') || include_cors_allow_origin
             send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end
@@ -533,7 +544,17 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
+        if (@add_query_params) 
+
+          query_params = WEBrick::HTTPUtils.parse_query(uri.query)
+
+          query_params.each_pair {|k,v|
+            params["QUERY_#{k.gsub('-','_').upcase}"] = v
+          }
+        end
+
         params.merge!(@env)
+
         @env.clear
 
         code, header, body = @callback.call(path_info, params)