fluentd 1.12.0.rc2 → 1.12.4

data/lib/fluent/system_config.rb

@@ -27,7 +27,7 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
     ]
 
     config_param :workers,   :integer, default: 1
@@ -45,6 +45,7 @@ module Fluent
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
+    config_param :disable_shared_socket, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/time.rb

@@ -50,6 +50,7 @@ module Fluent
     def to_int
       @sec
     end
+    alias :to_i :to_int
 
     def to_f
       @sec + @nsec / 1_000_000_000.0
@@ -131,13 +132,14 @@ module Fluent
   end
 
   module TimeMixin
-    TIME_TYPES = ['string', 'unixtime', 'float']
+    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
 
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
       [:utc,       :bool, {default: false}], # to turn :localtime false
       [:timezone, :string, {default: nil}],
+      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
@@ -169,6 +171,12 @@ module Fluent
           raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
         end
 
+        if conf.has_key?('time_type') and @time_type == :mixed
+          if @time_format.nil? and @time_format_fallbacks.empty?
+            raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
+          end
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -179,6 +187,7 @@ module Fluent
       end
 
       def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
         return NumericTimeParser.new(type) if type != :string
         return TimeParser.new(format, true, nil) if force_localtime
 
@@ -451,4 +460,52 @@ module Fluent
       end
     end
   end
+
+  # MixedTimeParser is available when time_type is set to :mixed
+  #
+  # Use Case 1: primary format is specified explicitly in time_format
+  #  time_type mixed
+  #  time_format %iso8601
+  #  time_format_fallbacks unixtime
+  # Use Case 2: time_format is omitted
+  #  time_type mixed
+  #  time_format_fallbacks %iso8601, unixtime
+  #
+  class MixedTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
+      @parsers = []
+      fallbacks.unshift(format).each do |fallback|
+        next unless fallback
+        case fallback
+        when 'unixtime', 'float'
+          @parsers << NumericTimeParser.new(fallback, localtime, timezone)
+        else
+          if force_localtime
+            @parsers << TimeParser.new(fallback, true, nil)
+          else
+            localtime = localtime && (timezone.nil? && !utc)
+            @parsers << TimeParser.new(fallback, localtime, timezone)
+          end
+        end
+      end
+    end
+
+    def parse(value)
+      @parsers.each do |parser|
+        begin
+          Float(value) if parser.class == Fluent::NumericTimeParser
+        rescue
+          next
+        end
+        begin
+          return parser.parse(value)
+        rescue
+          # skip TimeParseError
+        end
+      end
+      fallback_class = @parsers.collect do |parser| parser.class end.join(",")
+      raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
+    end
+  end
+
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.12.0.rc2'
+  VERSION = '1.12.4'
 
 end

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 12.0"
-  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
+  spec.add_development_dependency "rake", "~> <%= rake_version %>"
+  spec.add_development_dependency "test-unit", "~> <%= test_unit_version %>"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/templates/plugin_config_formatter/param.md-table.erb

@@ -0,0 +1,10 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|

data/test/command/test_fluentd.rb

@@ -878,6 +878,8 @@ CONF
     end
 
     test "without RUBYOPT" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -889,6 +891,8 @@ CONF
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
       assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'invalid values are set to RUBYOPT' do
@@ -912,6 +916,8 @@ CONF
 
     # https://github.com/fluent/fluentd/issues/2915
     test "ruby path contains spaces" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -940,6 +946,8 @@ CONF
         'spawn command to main:',
         '-Eascii-8bit:ascii-8bit'
       )
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
@@ -1065,4 +1073,34 @@ CONF
                          "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
     end
   end
+
+  sub_test_case 'sahred socket options' do
+    test 'enable shared socket by default' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         patterns_not_match: ["shared socket for multiple workers is disabled"])
+    end
+
+    test 'disable shared socket by command line option' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+
+    test 'disable shared socket by system config' do
+      conf = <<CONF
+<system>
+  disable_shared_socket
+</system>
+CONF
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+  end
 end

data/test/command/test_plugin_config_formatter.rb

@@ -225,6 +225,30 @@ TEXT
 path to something
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "input simple (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "input", "simple"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
+
+## TestFluentPluginConfigFormatter::SimpleInput
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|path|string (required)|path to something||
+
 TEXT
       assert_equal(expected, dumped_config)
     end
@@ -298,6 +322,49 @@ Default value: `normal`.
 
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
+
+## TestFluentPluginConfigFormatter::ComplexOutput
+
+
+### \\<authentication\\> section (required) (single)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|username|string (required)|username||
+|password|string (required)|password||
+
+
+### \\<parent\\> section (optional) (multiple)
+
+
+#### \\<child\\> section (optional) (multiple)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|names|array (required)|names||
+|difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|
+
+
+
 TEXT
       assert_equal(expected, dumped_config)
     end

data/test/config/test_configurable.rb

@@ -1453,7 +1453,7 @@ module Fluent::Config
         @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
-        assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
+        assert_raise NoMethodError do
           @example.name
         end
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -85,7 +85,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 5, lines.size
     end
 
-    test 'update seek postion of remained position entry' do
+    test 'update seek position of remained position entry' do
       pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
@@ -147,7 +147,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
@@ -177,7 +177,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
 
       f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
@@ -193,7 +193,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
-      inode1 = Fluent::FileWrapper.stat(@file).ino
+      inode1 = File.stat(@file).ino
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
       p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
@@ -201,7 +201,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      inode2 = Fluent::FileWrapper.stat(@file).ino
+      inode2 = File.stat(@file).ino
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
       p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
@@ -282,4 +282,58 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 2, f.read_inode
     end
   end
+
+  sub_test_case "TargetInfo equality rules" do
+    sub_test_case "== operator" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
+
+        assert_equal t1, t2
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1, t2
+      end
+    end
+
+    sub_test_case "eql? method" do
+      def test_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
+
+        assert do
+          t1.eql? t2
+        end
+      end
+
+      def test_not_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
+
+        assert do
+          !t1.eql? t2
+        end
+      end
+    end
+
+    sub_test_case "hash" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
+
+        assert_equal t1.hash, t2.hash
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1.hash, t2.hash
+      end
+    end
+  end
 end

data/test/plugin/test_file_wrapper.rb

@@ -0,0 +1,105 @@
+require_relative '../helper'
+require 'fluent/plugin/file_wrapper'
+
+class FileWrapperTest < Test::Unit::TestCase
+  require 'windows/file'
+  require 'windows/error'
+  include Windows::File
+  include Windows::Error
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
+
+  def setup
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  sub_test_case 'Win32Error' do
+    test 'equal' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
+    end
+
+    test 'different error code' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'different error message' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
+                       Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
+    end
+
+    test 'different class' do
+      assert_not_equal(Errno::EPIPE,
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'ERROR_SHARING_VIOLATION message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
+    end
+
+    test 'ERROR_SHARING_VIOLATION with a message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
+                   " - cannot open the file")
+    end
+
+    test 'to_s' do
+      assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
+    end
+
+    test 'inspect' do
+      assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
+    end
+  end
+
+  sub_test_case 'WindowsFile exceptions' do
+    test 'nothing raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb") do |f|
+        end
+        assert_nothing_raised do
+          file2 = Fluent::WindowsFile.new(path)
+        ensure
+          file2.close
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+
+    test 'Errno::ENOENT raised' do
+      path = "#{TMP_DIR}/nofile.txt"
+      file = nil
+      assert_raise(Errno::ENOENT) do
+        file = Fluent::WindowsFile.new(path)
+      ensure
+        file.close if file
+      end
+    end
+
+    test 'ERROR_SHARING_VIOLATION raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb")
+        assert_raise(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)) do
+          file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
+        ensure
+          file2.close if file2
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+  end
+end if Fluent.windows?