fluentd 1.12.0-x64-mingw32 → 1.13.0-x64-mingw32

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -32,17 +32,23 @@ module Fluent
           @static_config = true
         end
 
-        def configure(opts, parent: nil)
-          opts.each do |opt|
-            sd = Fluent::Plugin.new_sd(opt[:type], parent: parent)
-            sd.configure(opt[:conf])
+        def configure(configs, parent: nil)
+          configs.each do |config|
+            type, conf = if config.has_key?(:conf) # for compatibility with initial API
+                           [config[:type], config[:conf]]
+                         else
+                           [config['@type'], config]
+                         end
+
+            sd = Fluent::Plugin.new_sd(type, parent: parent)
+            sd.configure(conf)
 
             sd.services.each do |s|
               @services[s.discovery_id] = build_service(s)
             end
             @discoveries << sd
 
-            if @static_config && opt[:type] != :static
+            if @static_config && type.to_sym != :static
               @static_config = false
             end
           end

data/lib/fluent/plugin_helper/socket_option.rb

@@ -38,8 +38,8 @@ module Fluent
           end
         end
         if send_keepalive_packet
-          if protocol != :tcp
-            raise ArgumentError, "BUG: send_keepalive_packet is available for tcp"
+          if protocol != :tcp && protocol != :tls
+            raise ArgumentError, "BUG: send_keepalive_packet is available for tcp/tls"
           end
         end
       end

data/lib/fluent/supervisor.rb

@@ -45,6 +45,7 @@ module Fluent
   module ServerModule
     def before_run
       @fluentd_conf = config[:fluentd_conf]
+      @rpc_endpoint = nil
       @rpc_server = nil
       @counter = nil
 
@@ -64,9 +65,13 @@ module Fluent
         run_counter_server(counter)
       end
 
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      if config[:disable_shared_socket]
+        $log.info "shared socket for multiple workers is disabled"
+      else
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
     end
 
     def after_run
@@ -380,7 +385,7 @@ module Fluent
       config_mtime = File.mtime(path)
 
       # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
-      if Time.now - Time.at(pre_loadtime) < 5 and config_mtime == pre_config_mtime
+      if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
         return params['pre_conf']
       end
 
@@ -452,6 +457,7 @@ module Fluent
         config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
+        disable_shared_socket: params['disable_shared_socket']
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -512,6 +518,8 @@ module Fluent
         dl_opts = {}
         # subtract 1 to match serverengine daemon logger side logging severity.
         dl_opts[:log_level] = @level - 1
+        dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
+        dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
         logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
@@ -567,7 +575,8 @@ module Fluent
         supervise: true,
         standalone_worker: false,
         signame: nil,
-        conf_encoding: 'utf-8'
+        conf_encoding: 'utf-8',
+        disable_shared_socket: nil
       }
     end
 
@@ -599,6 +608,19 @@ module Fluent
 
       @cl_opt = opt
       @conf = nil
+      # parse configuration immediately to initialize logger in early stage
+      if @config_path and File.exist?(@config_path)
+        @conf = Fluent::Config.build(config_path: @config_path,
+                                     encoding: @conf_encoding ? @conf_encoding : 'utf-8',
+                                     additional_config: @inline_config ? @inline_config : nil,
+                                     use_v1_config: !!@use_v1_config)
+        @system_config = build_system_config(@conf)
+        if @system_config.log
+          @log_rotate_age ||= @system_config.log.rotate_age
+          @log_rotate_size ||= @system_config.log.rotate_size
+        end
+        @conf = nil
+      end
 
       log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
                   ignore_same_log_interval: opt[:ignore_same_log_interval]}
@@ -795,6 +817,7 @@ module Fluent
         'counter_server' => @system_config.counter_server,
         'log_format' => @system_config.log.format,
         'log_time_format' => @system_config.log.time_format,
+        'disable_shared_socket' => @system_config.disable_shared_socket
       }
 
       se = ServerEngine.create(ServerModule, WorkerModule){

data/lib/fluent/system_config.rb

@@ -27,7 +27,7 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
     ]
 
     config_param :workers,   :integer, default: 1
@@ -45,6 +45,7 @@ module Fluent
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
+    config_param :disable_shared_socket, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -54,6 +55,20 @@ module Fluent
     config_section :log, required: false, init: true, multi: false do
       config_param :format, :enum, list: [:text, :json], default: :text
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+      config_param :rotate_age, default: nil do |v|
+        if Fluent::Log::LOG_ROTATE_AGE.include?(v)
+          v.to_sym
+        else
+          begin
+            Integer(v)
+          rescue ArgumentError => e
+            raise Fluent::ConfigError, e.message
+          else
+            v.to_i
+          end
+        end
+      end
+      config_param :rotate_size, :size, default: nil
     end
 
     config_section :counter_server, multi: false do

data/lib/fluent/time.rb

@@ -132,13 +132,14 @@ module Fluent
   end
 
   module TimeMixin
-    TIME_TYPES = ['string', 'unixtime', 'float']
+    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
 
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
       [:utc,       :bool, {default: false}], # to turn :localtime false
       [:timezone, :string, {default: nil}],
+      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
@@ -170,6 +171,12 @@ module Fluent
           raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
         end
 
+        if conf.has_key?('time_type') and @time_type == :mixed
+          if @time_format.nil? and @time_format_fallbacks.empty?
+            raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
+          end
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -180,6 +187,7 @@ module Fluent
       end
 
       def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
         return NumericTimeParser.new(type) if type != :string
         return TimeParser.new(format, true, nil) if force_localtime
 
@@ -452,4 +460,52 @@ module Fluent
       end
     end
   end
+
+  # MixedTimeParser is available when time_type is set to :mixed
+  #
+  # Use Case 1: primary format is specified explicitly in time_format
+  #  time_type mixed
+  #  time_format %iso8601
+  #  time_format_fallbacks unixtime
+  # Use Case 2: time_format is omitted
+  #  time_type mixed
+  #  time_format_fallbacks %iso8601, unixtime
+  #
+  class MixedTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
+      @parsers = []
+      fallbacks.unshift(format).each do |fallback|
+        next unless fallback
+        case fallback
+        when 'unixtime', 'float'
+          @parsers << NumericTimeParser.new(fallback, localtime, timezone)
+        else
+          if force_localtime
+            @parsers << TimeParser.new(fallback, true, nil)
+          else
+            localtime = localtime && (timezone.nil? && !utc)
+            @parsers << TimeParser.new(fallback, localtime, timezone)
+          end
+        end
+      end
+    end
+
+    def parse(value)
+      @parsers.each do |parser|
+        begin
+          Float(value) if parser.class == Fluent::NumericTimeParser
+        rescue
+          next
+        end
+        begin
+          return parser.parse(value)
+        rescue
+          # skip TimeParseError
+        end
+      end
+      fallback_class = @parsers.collect do |parser| parser.class end.join(",")
+      raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
+    end
+  end
+
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.12.0'
+  VERSION = '1.13.0'
 
 end

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 12.0"
-  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
+  spec.add_development_dependency "rake", "~> <%= rake_version %>"
+  spec.add_development_dependency "test-unit", "~> <%= test_unit_version %>"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/templates/plugin_config_formatter/param.md-table.erb

@@ -0,0 +1,10 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|

data/test/command/test_cat.rb

@@ -0,0 +1,96 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'open3'
+require 'fluent/plugin/output'
+require 'fluent/plugin/in_forward'
+require 'fluent/plugin/out_secondary_file'
+require 'fluent/test/driver/output'
+require 'fluent/test/driver/input'
+
+class TestFluentCat < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.mkdir_p(TMP_DIR)
+    @record = { 'key' => 'value' }
+    @time = event_time
+    @es = Fluent::OneEventStream.new(@time, @record)
+    @primary = create_primary
+    metadata = @primary.buffer.new_metadata
+    @chunk = create_chunk(@primary, metadata, @es)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}")
+  FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + "/../../bin/fluent-cat")
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+  ]
+
+  SECONDARY_CONFIG = %[
+    directory #{TMP_DIR}
+  ]
+
+  class DummyOutput < Fluent::Plugin::Output
+    def write(chunk); end
+  end
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
+  end
+
+  def create_primary(buffer_cofig = config_element('buffer'))
+    DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_cofig]))
+  end
+
+  def create_secondary_driver(conf=SECONDARY_CONFIG)
+    c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
+    c.instance.acts_as_secondary(@primary)
+    c.configure(conf)
+  end
+
+  def create_chunk(primary, metadata, es)
+    primary.buffer.generate_chunk(metadata).tap do |c|
+      c.concat(es.to_msgpack_stream, es.size)
+      c.commit
+    end
+  end
+
+  sub_test_case "json" do
+    def test_cat_json
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} json") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_equal([1, "json", @record],
+                   [d.events.size, event.first, event.last])
+    end
+  end
+
+  sub_test_case "msgpack" do
+    def test_cat_secondary_file
+      d = create_secondary_driver
+      path = d.instance.write(@chunk)
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} --format msgpack secondary") do |stdin|
+          stdin.write(File.read(path))
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_equal([1, "secondary", @record],
+                   [d.events.size, event.first, event.last])
+    end
+  end
+end

data/test/command/test_fluentd.rb

@@ -878,6 +878,8 @@ CONF
     end
 
     test "without RUBYOPT" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -889,6 +891,8 @@ CONF
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
       assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'invalid values are set to RUBYOPT' do
@@ -912,6 +916,8 @@ CONF
 
     # https://github.com/fluent/fluentd/issues/2915
     test "ruby path contains spaces" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -940,6 +946,8 @@ CONF
         'spawn command to main:',
         '-Eascii-8bit:ascii-8bit'
       )
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
@@ -1065,4 +1073,34 @@ CONF
                          "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
     end
   end
+
+  sub_test_case 'sahred socket options' do
+    test 'enable shared socket by default' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         patterns_not_match: ["shared socket for multiple workers is disabled"])
+    end
+
+    test 'disable shared socket by command line option' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+
+    test 'disable shared socket by system config' do
+      conf = <<CONF
+<system>
+  disable_shared_socket
+</system>
+CONF
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+  end
 end