fluentd 1.6.3 → 1.7.1

data/lib/fluent/plugin_helper/server.rb

@@ -377,7 +377,7 @@ module Fluent
       end
 
       # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
-      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+      PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
 
       class CallbackSocket
         def initialize(server_type, sock, enabled_events = [], close_socket: true)
@@ -722,7 +722,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/plugin_helper/socket.rb

@@ -17,6 +17,9 @@
 require 'socket'
 require 'ipaddr'
 require 'openssl'
+if Fluent.windows?
+  require 'certstore'
+end
 
 require_relative 'socket_option'
 
@@ -96,7 +99,9 @@ module Fluent
           host, port,
           version: TLS_DEFAULT_VERSION, ciphers: CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
           enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
-          cert_path: nil, private_key_path: nil, private_key_passphrase: nil, **kwargs, &block)
+          cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
+          cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
+          **kwargs, &block)
 
         host_is_ipaddress = IPAddr.new(host) rescue false
         fqdn ||= host unless host_is_ipaddress
@@ -113,6 +118,14 @@ module Fluent
           end
           begin
             if enable_system_cert_store
+              if Fluent.windows? && cert_logical_store_name
+                log.trace "loading Windows system certificate store"
+                loader = Certstore::OpenSSL::Loader.new(log, cert_store, cert_logical_store_name,
+                                                        enterprise: cert_use_enterprise_store)
+                loader.load_cert_store
+                cert_store = loader.cert_store
+                context.cert = loader.get_certificate(cert_thumbprint) if cert_thumbprint
+              end
               log.trace "loading system default certificate store"
               cert_store.set_default_paths
             end

data/lib/fluent/plugin_helper/thread.rb

@@ -92,6 +92,7 @@ module Fluent
           end
         end
         thread.abort_on_exception = true
+        thread.name = title.to_s if thread.respond_to?(:name)
         @_threads_mutex.synchronize do
           @_threads[thread.object_id] = thread
         end

data/lib/fluent/root_agent.rb

@@ -201,7 +201,7 @@ module Fluent
     def start
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
-        # Input#start sometimes emits lots of evetns with in_tail/`read_from_head true` case
+        # Input#start sometimes emits lots of events with in_tail/`read_from_head true` case
         # and it causes deadlock for small buffer/queue output. To avoid such problem,
         # buffer related output threads should be run before `Input#start`.
         # This is why after_start should be called immediately after start call.

data/lib/fluent/time.rb

@@ -78,7 +78,7 @@ module Fluent
       end
     rescue
       def to_time
-        Time.at(@sec, @nsec / 1000.0)
+        Time.at(Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000))
       end
     end
 
@@ -111,7 +111,9 @@ module Fluent
     end
 
     def self.now
-      from_time(Time.now)
+      # This method is called many time. so call Process.clock_gettime directly instead of Fluent::Clock.real_now
+      now = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
+      Fluent::EventTime.new(now / 1_000_000_000, now % 1_000_000_000)
     end
 
     def self.parse(*args)

data/lib/fluent/timezone.rb

@@ -18,6 +18,15 @@ require 'tzinfo'
 
 require 'fluent/config/error'
 
+# For v0.12. Will be removed after v2
+module IntegerExt
+  refine Integer do
+    def to_time
+      Time.at(self)
+    end
+  end
+end
+
 module Fluent
   class Timezone
     # [+-]HH:MM, [+-]HHMM, [+-]HH
@@ -84,6 +93,8 @@ module Fluent
       end
     end
 
+    using IntegerExt
+
     # Create a formatter for a timezone and optionally a format.
     #
     # An Proc object is returned. If the given timezone is invalid,
@@ -100,15 +111,15 @@ module Fluent
         case
         when format.is_a?(String)
           return Proc.new {|time|
-            Time.at(time).localtime(offset).strftime(format)
+            time.to_time.localtime(offset).strftime(format)
           }
         when format.is_a?(Strftime)
           return Proc.new {|time|
-            format.exec(Time.at(time).localtime(offset))
+            format.exec(time.to_time.localtime(offset))
           }
         else
           return Proc.new {|time|
-            Time.at(time).localtime(offset).iso8601
+            time.to_time.localtime(offset).iso8601
           }
         end
       end
@@ -124,15 +135,18 @@ module Fluent
         case
         when format.is_a?(String)
           return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
+            time = time.to_time
+            time.localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
           }
         when format.is_a?(Strftime)
           return Proc.new {|time|
-            format.exec(Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset))
+            time = time.to_time
+            format.exec(time.localtime(tz.period_for_utc(time).utc_total_offset))
           }
         else
           return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601
+            time = time.to_time
+            time.localtime(tz.period_for_utc(time).utc_total_offset).iso8601
           }
         end
       end
@@ -149,7 +163,7 @@ module Fluent
       when NAME_PATTERN
         tz = TZInfo::Timezone.get(timezone)
         ->(time) {
-          tz.period_for_utc(time).utc_total_offset
+          tz.period_for_utc(time.to_time).utc_total_offset
         }
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.6.3'
+  VERSION = '1.7.1'
 
 end

data/test/command/test_fluentd.rb

@@ -807,7 +807,7 @@ CONF
       )
     end
 
-    test 'success to start workers when configured plugins as a chidren of MultiOutput only for specific worker do not support multi worker configuration' do
+    test 'success to start workers when configured plugins as a children of MultiOutput only for specific worker do not support multi worker configuration' do
       script = <<-EOC
 require 'fluent/plugin/output'
 module Fluent::Plugin

data/test/command/test_plugin_generator.rb

@@ -16,6 +16,14 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
     FileUtils.rm_rf(TEMP_DIR)
   end
 
+  def stub_git_process(target)
+    stub(target).spawn do |cmd, arg1, arg2|
+      assert_equal %w[git init .], [cmd, arg1, arg2]
+      -1
+    end
+    stub(Process).wait { |pid| assert_equal(pid, -1) }
+  end
+
   sub_test_case "generate plugin" do
     data(input: ["input", "in"],
          output: ["output", "out"],
@@ -23,8 +31,10 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          parser: ["parser", "parser"],
          formatter: ["formatter", "formatter"])
     test "generate plugin" do |(type, part)|
+      generator = FluentPluginGenerator.new([type, "fake"])
+      stub_git_process(generator)
       capture_stdout do
-        FluentPluginGenerator.new([type, "fake"]).call
+        generator.call
       end
       plugin_base_dir = Pathname("fluent-plugin-fake")
       assert { plugin_base_dir.directory? }
@@ -49,8 +59,10 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
     end
 
     test "no license" do
+      generator = FluentPluginGenerator.new(["--no-license", "filter", "fake"])
+      stub_git_process(generator)
       capture_stdout do
-        FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call
+        generator.call
       end
       assert { !Pathname("fluent-plugin-fake/LICENSE").exist? }
       assert { Pathname("fluent-plugin-fake/Gemfile").exist? }
@@ -72,6 +84,8 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          "dash" => ["rewrite-tag-filter", "rewrite_tag_filter"])
     test "plugin_name" do |(name, plugin_name)|
       generator = FluentPluginGenerator.new(["filter", name])
+      stub_git_process(generator)
+      stub(Process).wait { |pid| assert_equal(pid, -1) }
       capture_stdout do
         generator.call
       end
@@ -83,6 +97,8 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          "dash" => ["rewrite-tag-filter", "fluent-plugin-rewrite-tag-filter"])
     test "gem_name" do |(name, gem_name)|
       generator = FluentPluginGenerator.new(["output", name])
+      stub_git_process(generator)
+      stub(Process).wait { |pid| assert_equal(pid, -1) }
       capture_stdout do
         generator.call
       end

data/test/config/test_configurable.rb

@@ -152,7 +152,7 @@ module ConfigurableSpec
     config_param :arrayvalue, :array
   end
 
-  class Example1
+  class ExampleWithAlias
     include Fluent::Configurable
 
     config_param :name, :string, alias: :fullname
@@ -166,22 +166,7 @@ module ConfigurableSpec
     end
   end
 
-  class Example3
-    include Fluent::Configurable
-
-    config_param :age, :integer, default: 10
-
-    config_section :appendix, required: true, multi: false, final: true do
-      config_param :type, :string
-      config_param :name, :string, default: "x"
-    end
-
-    def get_all
-      [@name, @detail]
-    end
-  end
-
-  class Example5
+  class ExampleWithSecret
     include Fluent::Configurable
 
     config_param :normal_param, :string
@@ -193,17 +178,42 @@ module ConfigurableSpec
     end
   end
 
-  class Example6
+  class ExampleWithDefaultHashAndArray
     include Fluent::Configurable
     config_param :obj1, :hash, default: {}
     config_param :obj2, :array, default: []
   end
 
-  class Example7
+  class ExampleWithSkipAccessor
     include Fluent::Configurable
     config_param :name, :string, default: 'example7', skip_accessor: true
   end
 
+  class ExampleWithCustomSection
+    include Fluent::Configurable
+    config_param :name_param, :string
+    config_section :normal_section  do
+      config_param :normal_section_param, :string
+    end
+
+    class CustomSection
+      include Fluent::Configurable
+      config_param :custom_section_param, :string
+    end
+
+    class AnotherElement
+      include Fluent::Configurable
+    end
+
+    def configure(conf)
+      super
+      conf.elements.each do |e|
+        next if e.name != 'custom_section'
+        CustomSection.new.configure(e)
+      end
+    end
+  end
+
   module Overwrite
     class Base
       include Fluent::Configurable
@@ -564,12 +574,12 @@ module Fluent::Config
 
         sub_test_case 'default values should be duplicated before touched in plugin code' do
           test 'default object should be dupped for cases configured twice' do
-            x6a = ConfigurableSpec::Example6.new
+            x6a = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6a.configure(config_element("")) }
             assert_equal({}, x6a.obj1)
             assert_equal([], x6a.obj2)
 
-            x6b = ConfigurableSpec::Example6.new
+            x6b = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6b.configure(config_element("")) }
             assert_equal({}, x6b.obj1)
             assert_equal([], x6b.obj2)
@@ -577,7 +587,7 @@ module Fluent::Config
             assert { x6a.obj1.object_id != x6b.obj1.object_id }
             assert { x6a.obj2.object_id != x6b.obj2.object_id }
 
-            x6c = ConfigurableSpec::Example6.new
+            x6c = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6c.configure(config_element("")) }
             assert_equal({}, x6c.obj1)
             assert_equal([], x6c.obj2)
@@ -1184,7 +1194,7 @@ module Fluent::Config
     sub_test_case 'class defined with config_param/config_section having :alias' do
       sub_test_case '#initialize' do
         test 'does not create methods for alias' do
-          ex1 = ConfigurableSpec::Example1.new
+          ex1 = ConfigurableSpec::ExampleWithAlias.new
           assert_nothing_raised { ex1.name }
           assert_raise(NoMethodError) { ex1.fullname }
           assert_nothing_raised { ex1.bool }
@@ -1196,7 +1206,7 @@ module Fluent::Config
 
       sub_test_case '#configure' do
         test 'provides accessible data for alias attribute keys' do
-          ex1 = ConfigurableSpec::Example1.new
+          ex1 = ConfigurableSpec::ExampleWithAlias.new
           conf = config_element('ROOT', '', {
                                   "fullname" => "foo bar",
                                   "bool" => false
@@ -1288,7 +1298,7 @@ module Fluent::Config
                                  'secret_param' => 'secret'
                                },
                                [config_element('section', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
-        @example = ConfigurableSpec::Example5.new
+        @example = ConfigurableSpec::ExampleWithSecret.new
         @example.configure(@conf)
       end
 
@@ -1311,20 +1321,6 @@ module Fluent::Config
         }
       end
 
-      test 'get plugin name when found unknown section' do
-        @conf = config_element('ROOT', '',
-                               {
-                                 'normal_param' => 'normal',
-                                 'secret_param' => 'secret'
-                               },
-                               [config_element('unknown', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
-        @example = ConfigurableSpec::Example5.new
-        @example.configure(@conf)
-        @conf.elements.each { |e|
-          assert_equal(['ROOT', nil], e.unused_in)
-        }
-      end
-
       def assert_secret_param(key, value)
         case key
         when 'normal_param', 'normal_param2'
@@ -1335,9 +1331,51 @@ module Fluent::Config
       end
     end
 
+    sub_test_case 'unused section' do
+      test 'get plugin name when found unknown section' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('unknown', '', {'name_param' => 'normal'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        conf.elements.each { |e|
+          assert_equal(['ROOT', nil], e.unused_in)
+        }
+      end
+
+      test 'get an empty array when the section is defined without using config_section' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('custom_section', '', {'custom_section_param' => 'custom'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        conf.elements.each { |e|
+          assert_equal([], e.unused_in)
+        }
+      end
+
+      test 'get an empty array when the configuration is used in another element without any sections' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('normal_section', '', {'normal_section_param' => 'normal'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        ConfigurableSpec::ExampleWithCustomSection::AnotherElement.new.configure(conf)
+        conf.elements.each { |e|
+          assert_equal([], e.unused_in)
+        }
+      end
+    end
+
     sub_test_case ':skip_accessor option' do
       test 'it does not create accessor methods for parameters' do
-        @example = ConfigurableSpec::Example7.new
+        @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
         assert_raise NoMethodError.new("undefined method `name' for #{@example}") do

data/test/counter/test_store.rb

@@ -14,7 +14,7 @@ class CounterStoreTest < ::Test::Unit::TestCase
     @now = Fluent::EventTime.now
   end
 
-  shutdown do
+  teardown do
     Timecop.return
   end