fluentd 1.6.3 → 1.7.0

data/lib/fluent/timezone.rb

@@ -18,6 +18,15 @@ require 'tzinfo'
 
 require 'fluent/config/error'
 
+# For v0.12. Will be removed after v2
+module IntegerExt
+  refine Integer do
+    def to_time
+      Time.at(self)
+    end
+  end
+end
+
 module Fluent
   class Timezone
     # [+-]HH:MM, [+-]HHMM, [+-]HH
@@ -84,6 +93,8 @@ module Fluent
       end
     end
 
+    using IntegerExt
+
     # Create a formatter for a timezone and optionally a format.
     #
     # An Proc object is returned. If the given timezone is invalid,
@@ -100,15 +111,15 @@ module Fluent
         case
         when format.is_a?(String)
           return Proc.new {|time|
-            Time.at(time).localtime(offset).strftime(format)
+            time.to_time.localtime(offset).strftime(format)
           }
         when format.is_a?(Strftime)
           return Proc.new {|time|
-            format.exec(Time.at(time).localtime(offset))
+            format.exec(time.to_time.localtime(offset))
           }
         else
           return Proc.new {|time|
-            Time.at(time).localtime(offset).iso8601
+            time.to_time.localtime(offset).iso8601
           }
         end
       end
@@ -124,15 +135,18 @@ module Fluent
         case
         when format.is_a?(String)
           return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
+            time = time.to_time
+            time.localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
           }
         when format.is_a?(Strftime)
           return Proc.new {|time|
-            format.exec(Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset))
+            time = time.to_time
+            format.exec(time.localtime(tz.period_for_utc(time).utc_total_offset))
           }
         else
           return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601
+            time = time.to_time
+            time.localtime(tz.period_for_utc(time).utc_total_offset).iso8601
           }
         end
       end
@@ -149,7 +163,7 @@ module Fluent
       when NAME_PATTERN
         tz = TZInfo::Timezone.get(timezone)
         ->(time) {
-          tz.period_for_utc(time).utc_total_offset
+          tz.period_for_utc(time.to_time).utc_total_offset
         }
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.6.3'
+  VERSION = '1.7.0'
 
 end

data/test/command/test_fluentd.rb

@@ -807,7 +807,7 @@ CONF
       )
     end
 
-    test 'success to start workers when configured plugins as a chidren of MultiOutput only for specific worker do not support multi worker configuration' do
+    test 'success to start workers when configured plugins as a children of MultiOutput only for specific worker do not support multi worker configuration' do
       script = <<-EOC
 require 'fluent/plugin/output'
 module Fluent::Plugin

data/test/command/test_plugin_generator.rb

@@ -16,6 +16,14 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
     FileUtils.rm_rf(TEMP_DIR)
   end
 
+  def stub_git_process(target)
+    stub(target).spawn do |cmd, arg1, arg2|
+      assert_equal %w[git init .], [cmd, arg1, arg2]
+      -1
+    end
+    stub(Process).wait { |pid| assert_equal(pid, -1) }
+  end
+
   sub_test_case "generate plugin" do
     data(input: ["input", "in"],
          output: ["output", "out"],
@@ -23,8 +31,10 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          parser: ["parser", "parser"],
          formatter: ["formatter", "formatter"])
     test "generate plugin" do |(type, part)|
+      generator = FluentPluginGenerator.new([type, "fake"])
+      stub_git_process(generator)
       capture_stdout do
-        FluentPluginGenerator.new([type, "fake"]).call
+        generator.call
       end
       plugin_base_dir = Pathname("fluent-plugin-fake")
       assert { plugin_base_dir.directory? }
@@ -49,8 +59,10 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
     end
 
     test "no license" do
+      generator = FluentPluginGenerator.new(["--no-license", "filter", "fake"])
+      stub_git_process(generator)
       capture_stdout do
-        FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call
+        generator.call
       end
       assert { !Pathname("fluent-plugin-fake/LICENSE").exist? }
       assert { Pathname("fluent-plugin-fake/Gemfile").exist? }
@@ -72,6 +84,8 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          "dash" => ["rewrite-tag-filter", "rewrite_tag_filter"])
     test "plugin_name" do |(name, plugin_name)|
       generator = FluentPluginGenerator.new(["filter", name])
+      stub_git_process(generator)
+      stub(Process).wait { |pid| assert_equal(pid, -1) }
       capture_stdout do
         generator.call
       end
@@ -83,6 +97,8 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
          "dash" => ["rewrite-tag-filter", "fluent-plugin-rewrite-tag-filter"])
     test "gem_name" do |(name, gem_name)|
       generator = FluentPluginGenerator.new(["output", name])
+      stub_git_process(generator)
+      stub(Process).wait { |pid| assert_equal(pid, -1) }
       capture_stdout do
         generator.call
       end

data/test/config/test_configurable.rb

@@ -152,7 +152,7 @@ module ConfigurableSpec
     config_param :arrayvalue, :array
   end
 
-  class Example1
+  class ExampleWithAlias
     include Fluent::Configurable
 
     config_param :name, :string, alias: :fullname
@@ -166,22 +166,7 @@ module ConfigurableSpec
     end
   end
 
-  class Example3
-    include Fluent::Configurable
-
-    config_param :age, :integer, default: 10
-
-    config_section :appendix, required: true, multi: false, final: true do
-      config_param :type, :string
-      config_param :name, :string, default: "x"
-    end
-
-    def get_all
-      [@name, @detail]
-    end
-  end
-
-  class Example5
+  class ExampleWithSecret
     include Fluent::Configurable
 
     config_param :normal_param, :string
@@ -193,17 +178,42 @@ module ConfigurableSpec
     end
   end
 
-  class Example6
+  class ExampleWithDefaultHashAndArray
     include Fluent::Configurable
     config_param :obj1, :hash, default: {}
     config_param :obj2, :array, default: []
   end
 
-  class Example7
+  class ExampleWithSkipAccessor
     include Fluent::Configurable
     config_param :name, :string, default: 'example7', skip_accessor: true
   end
 
+  class ExampleWithCustomSection
+    include Fluent::Configurable
+    config_param :name_param, :string
+    config_section :normal_section  do
+      config_param :normal_section_param, :string
+    end
+
+    class CustomSection
+      include Fluent::Configurable
+      config_param :custom_section_param, :string
+    end
+
+    class AnotherElement
+      include Fluent::Configurable
+    end
+
+    def configure(conf)
+      super
+      conf.elements.each do |e|
+        next if e.name != 'custom_section'
+        CustomSection.new.configure(e)
+      end
+    end
+  end
+
   module Overwrite
     class Base
       include Fluent::Configurable
@@ -564,12 +574,12 @@ module Fluent::Config
 
         sub_test_case 'default values should be duplicated before touched in plugin code' do
           test 'default object should be dupped for cases configured twice' do
-            x6a = ConfigurableSpec::Example6.new
+            x6a = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6a.configure(config_element("")) }
             assert_equal({}, x6a.obj1)
             assert_equal([], x6a.obj2)
 
-            x6b = ConfigurableSpec::Example6.new
+            x6b = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6b.configure(config_element("")) }
             assert_equal({}, x6b.obj1)
             assert_equal([], x6b.obj2)
@@ -577,7 +587,7 @@ module Fluent::Config
             assert { x6a.obj1.object_id != x6b.obj1.object_id }
             assert { x6a.obj2.object_id != x6b.obj2.object_id }
 
-            x6c = ConfigurableSpec::Example6.new
+            x6c = ConfigurableSpec::ExampleWithDefaultHashAndArray.new
             assert_nothing_raised { x6c.configure(config_element("")) }
             assert_equal({}, x6c.obj1)
             assert_equal([], x6c.obj2)
@@ -1184,7 +1194,7 @@ module Fluent::Config
     sub_test_case 'class defined with config_param/config_section having :alias' do
       sub_test_case '#initialize' do
         test 'does not create methods for alias' do
-          ex1 = ConfigurableSpec::Example1.new
+          ex1 = ConfigurableSpec::ExampleWithAlias.new
           assert_nothing_raised { ex1.name }
           assert_raise(NoMethodError) { ex1.fullname }
           assert_nothing_raised { ex1.bool }
@@ -1196,7 +1206,7 @@ module Fluent::Config
 
       sub_test_case '#configure' do
         test 'provides accessible data for alias attribute keys' do
-          ex1 = ConfigurableSpec::Example1.new
+          ex1 = ConfigurableSpec::ExampleWithAlias.new
           conf = config_element('ROOT', '', {
                                   "fullname" => "foo bar",
                                   "bool" => false
@@ -1288,7 +1298,7 @@ module Fluent::Config
                                  'secret_param' => 'secret'
                                },
                                [config_element('section', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
-        @example = ConfigurableSpec::Example5.new
+        @example = ConfigurableSpec::ExampleWithSecret.new
         @example.configure(@conf)
       end
 
@@ -1311,20 +1321,6 @@ module Fluent::Config
         }
       end
 
-      test 'get plugin name when found unknown section' do
-        @conf = config_element('ROOT', '',
-                               {
-                                 'normal_param' => 'normal',
-                                 'secret_param' => 'secret'
-                               },
-                               [config_element('unknown', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
-        @example = ConfigurableSpec::Example5.new
-        @example.configure(@conf)
-        @conf.elements.each { |e|
-          assert_equal(['ROOT', nil], e.unused_in)
-        }
-      end
-
       def assert_secret_param(key, value)
         case key
         when 'normal_param', 'normal_param2'
@@ -1335,9 +1331,51 @@ module Fluent::Config
       end
     end
 
+    sub_test_case 'unused section' do
+      test 'get plugin name when found unknown section' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('unknown', '', {'name_param' => 'normal'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        conf.elements.each { |e|
+          assert_equal(['ROOT', nil], e.unused_in)
+        }
+      end
+
+      test 'get an empty array when the section is defined without using config_section' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('custom_section', '', {'custom_section_param' => 'custom'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        conf.elements.each { |e|
+          assert_equal([], e.unused_in)
+        }
+      end
+
+      test 'get an empty array when the configuration is used in another element without any sections' do
+        conf = config_element('ROOT', '',
+                              {
+                                'name_param' => 'name',
+                              },
+                              [config_element('normal_section', '', {'normal_section_param' => 'normal'} )])
+        example = ConfigurableSpec::ExampleWithCustomSection.new
+        example.configure(conf)
+        ConfigurableSpec::ExampleWithCustomSection::AnotherElement.new.configure(conf)
+        conf.elements.each { |e|
+          assert_equal([], e.unused_in)
+        }
+      end
+    end
+
     sub_test_case ':skip_accessor option' do
       test 'it does not create accessor methods for parameters' do
-        @example = ConfigurableSpec::Example7.new
+        @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
         assert_raise NoMethodError.new("undefined method `name' for #{@example}") do

data/test/counter/test_store.rb

@@ -14,7 +14,7 @@ class CounterStoreTest < ::Test::Unit::TestCase
     @now = Fluent::EventTime.now
   end
 
-  shutdown do
+  teardown do
     Timecop.return
   end
 

data/test/helper.rb

@@ -50,6 +50,7 @@ require 'fluent/msgpack_factory'
 require 'fluent/time'
 require 'serverengine'
 require 'helpers/fuzzy_assert'
+require 'helpers/process_extenstion'
 
 module Fluent
   module Plugin

data/test/helpers/process_extenstion.rb

@@ -0,0 +1,33 @@
+require 'timecop'
+
+module Process
+  class << self
+    alias_method :clock_gettime_original, :clock_gettime
+
+    def clock_gettime(clock_id, unit = :float_second)
+      # now only support CLOCK_REALTIME
+      if Process::CLOCK_REALTIME == clock_id
+        t = Time.now
+
+        case unit
+        when :float_second
+          t.to_i + t.nsec / 1_000_000_000.0
+        when :float_millisecond
+          t.to_i * 1_000 + t.nsec / 1_000_000.0
+        when :float_microsecond
+          t.to_i * 1_000_000 + t.nsec / 1_000.0
+        when :second
+          t.to_i
+        when :millisecond
+          t.to_i * 1000 + t.nsec / 1_000_000
+        when :microsecond
+          t.to_i * 1_000_000 + t.nsec / 1_000
+        when :nanosecond
+          t.to_i * 1_000_000_000 + t.nsec
+        end
+      else
+        Process.clock_gettime_original(clock_id, unit)
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_ack_handler.rb

@@ -0,0 +1,101 @@
+require_relative '../../helper'
+require 'fluent/test/driver/output'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/ack_handler'
+
+class AckHandlerTest < Test::Unit::TestCase
+  data(
+    'chunk_id is matched' => [MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }), Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS],
+    'chunk_id is not matched' => [MessagePack.pack({ 'ack' => 'unmatched' }), Fluent::Plugin::ForwardOutput::AckHandler::Result::CHUNKID_UNMATCHED],
+    'chunk_id is empty' => ['', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED],
+  )
+  test 'returns chunk_id, node, sock and result status' do |args|
+    receved, state = args
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything) { |_| receved } # IO does not have recv
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_equal chunk_id, a1
+      assert_equal node, a2
+      assert_equal r, a3
+      assert_equal state, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+
+  test 'returns nil if raise an error' do
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything) { |_| raise 'unexpected error' } # IO does not have recv
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_nil a1
+      assert_nil a2
+      assert_nil a3
+      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+
+  test 'when ack is expired' do
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything).never
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_equal chunk_id, a1
+      assert_equal node, a2
+      assert_equal r, a3
+      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+end