fluentd 0.14.12 → 0.14.13

data/test/config/test_configure_proxy.rb

@@ -400,45 +400,54 @@ module Fluent::Config
         end
       end
 
-      sub_test_case '#dump' do
+      sub_test_case '#dump_config_definition' do
         setup do
           @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)
         end
 
         test 'empty proxy' do
-          assert_equal("", @proxy.dump)
+          assert_equal({}, @proxy.dump_config_definition)
         end
 
         test 'plain proxy w/o default value' do
           @proxy.config_param(:name, :string)
-          assert_equal(<<CONFIG, @proxy.dump)
-name: string: <nil>
-CONFIG
+          expected = {
+            name: { type: :string, required: true }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
         end
 
         test 'plain proxy w/ default value' do
           @proxy.config_param(:name, :string, default: "name1")
-          assert_equal(<<CONFIG, @proxy.dump)
-name: string: <"name1">
-CONFIG
+          expected = {
+            name: { type: :string, default: "name1", required: false }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
         end
 
         test 'plain proxy w/ default value using config_set_default' do
           @proxy.config_param(:name, :string)
           @proxy.config_set_default(:name, "name1")
-          assert_equal(<<CONFIG, @proxy.dump)
-name: string: <"name1">
-CONFIG
+          expected = {
+            name: { type: :string, default: "name1", required: false }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
         end
 
         test 'single sub proxy' do
           @proxy.config_section(:sub) do
             config_param(:name, :string, default: "name1")
           end
-          assert_equal(<<CONFIG, @proxy.dump)
-sub
- name: string: <"name1">
-CONFIG
+          expected = {
+            sub: {
+              alias: nil,
+              multi: true,
+              required: false,
+              section: true,
+              name: { type: :string, default: "name1", required: false }
+            }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
         end
 
         test 'nested sub proxy' do
@@ -450,30 +459,43 @@ CONFIG
               config_param(:name4, :string, default: "name4")
             end
           end
-          assert_equal(<<CONFIG, @proxy.dump)
-sub
- name1: string: <"name1">
- name2: string: <"name2">
- sub2
-  name3: string: <"name3">
-  name4: string: <"name4">
-CONFIG
+          expected = {
+            sub: {
+              alias: nil,
+              multi: true,
+              required: false,
+              section: true,
+              name1: { type: :string, default: "name1", required: false },
+              name2: { type: :string, default: "name2", required: false },
+              sub2: {
+                alias: nil,
+                multi: true,
+                required: false,
+                section: true,
+                name3: { type: :string, default: "name3", required: false },
+                name4: { type: :string, default: "name4", required: false },
+              }
+            }
+          }
+          assert_equal(expected, @proxy.dump_config_definition)
         end
 
         sub_test_case 'w/ description' do
           test 'single proxy' do
             @proxy.config_param(:name, :string, desc: "description for name")
-          assert_equal(<<CONFIG, @proxy.dump)
-name: string: <nil> # description for name
-CONFIG
+            expected = {
+              name: { type: :string, desc: "description for name", required: true }
+            }
+            assert_equal(expected, @proxy.dump_config_definition)
           end
 
           test 'single proxy using config_set_desc' do
             @proxy.config_param(:name, :string)
             @proxy.config_set_desc(:name, "description for name")
-          assert_equal(<<CONFIG, @proxy.dump)
-name: string: <nil> # description for name
-CONFIG
+            expected = {
+              name: { type: :string, desc: "description for name", required: true }
+            }
+            assert_equal(expected, @proxy.dump_config_definition)
           end
 
           test 'sub proxy' do
@@ -485,14 +507,25 @@ CONFIG
                 config_param(:name4, :string, default: "name4", desc: "desc4")
               end
             end
-            assert_equal(<<CONFIG, @proxy.dump)
-sub
- name1: string: <"name1"> # desc1
- name2: string: <"name2"> # desc2
- sub2
-  name3: string: <"name3">
-  name4: string: <"name4"> # desc4
-CONFIG
+            expected = {
+              sub: {
+                alias: nil,
+                multi: true,
+                required: false,
+                section: true,
+                name1: { type: :string, default: "name1", desc: "desc1", required: false },
+                name2: { type: :string, default: "name2", desc: "desc2", required: false },
+                sub2: {
+                  alias: nil,
+                  multi: true,
+                  required: false,
+                  section: true,
+                  name3: { type: :string, default: "name3", required: false },
+                  name4: { type: :string, default: "name4", desc: "desc4", required: false },
+                }
+              }
+            }
+            assert_equal(expected, @proxy.dump_config_definition)
           end
 
           test 'sub proxy w/ desc method' do
@@ -506,14 +539,25 @@ CONFIG
                 config_param(:name4, :string, default: "name4")
               end
             end
-            assert_equal(<<CONFIG, @proxy.dump)
-sub
- name1: string: <"name1"> # desc1
- name2: string: <"name2"> # desc2
- sub2
-  name3: string: <"name3">
-  name4: string: <"name4"> # desc4
-CONFIG
+            expected = {
+              sub: {
+                alias: nil,
+                multi: true,
+                required: false,
+                section: true,
+                name1: { type: :string, default: "name1", desc: "desc1", required: false },
+                name2: { type: :string, default: "name2", desc: "desc2", required: false },
+                sub2: {
+                  alias: nil,
+                  multi: true,
+                  required: false,
+                  section: true,
+                  name3: { type: :string, default: "name3", required: false },
+                  name4: { type: :string, default: "name4", desc: "desc4", required: false },
+                }
+              }
+            }
+            assert_equal(expected, @proxy.dump_config_definition)
           end
         end
       end

data/test/plugin/test_in_tail.rb

@@ -12,14 +12,7 @@ class TailInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    FileUtils.rm_rf(TMP_DIR, secure: true)
-    if File.exist?(TMP_DIR)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
-      FileUtils.remove_entry_secure(TMP_DIR, true)
-    end
-    FileUtils.mkdir_p(TMP_DIR)
+    cleanup_directory(TMP_DIR)
   end
 
   def teardown
@@ -27,6 +20,17 @@ class TailInputTest < Test::Unit::TestCase
     Fluent::Engine.stop
   end
 
+  def cleanup_directory(path)
+    FileUtils.rm_rf(path, secure: true)
+    if File.exist?(path)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(path, true)
+    end
+    FileUtils.mkdir_p(path)
+  end
+
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = config_element("ROOT", "", {
@@ -847,6 +851,67 @@ class TailInputTest < Test::Unit::TestCase
       plugin = create_driver(exclude_config, false).instance
       assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
     end
+
+    def test_log_file_without_extension
+      expected_files = [
+        'test/plugin/data/log/bar',
+        'test/plugin/data/log/foo/bar.log',
+        'test/plugin/data/log/foo/bar2',
+        'test/plugin/data/log/test.log'
+      ]
+
+      config = config_element("", "", {
+        "tag" => "tail",
+        "path" => "test/plugin/data/log/**/*",
+        "format" => "none",
+        "pos_file" => "#{TMP_DIR}/tail.pos"
+      })
+
+      plugin = create_driver(config, false).instance
+      assert_equal expected_files, plugin.expand_paths.sort
+    end
+
+    # For https://github.com/fluent/fluentd/issues/1455
+    # This test is fragile because test content depends on internal implementaion.
+    # So if you modify in_tail internal, this test may break.
+    def test_unwatched_files_should_be_removed
+      config = config_element("", "", {
+                                "tag" => "tail",
+                                "path" => "#{TMP_DIR}/*.txt",
+                                "format" => "none",
+                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                "read_from_head" => true,
+                                "refresh_interval" => 1,
+                              })
+      d = create_driver(config, false)
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
+      end
+
+      assert_equal 1, d.instance.instance_variable_get(:@tails).keys.size
+      cleanup_directory(TMP_DIR)
+      waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
+      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
+
+      # Previous implementaion has an infinite watcher creation bug.
+      # Following code checks such unexpected bug by couting  actual object allocation.
+      base_num = count_timer_object
+      2.times {
+        sleep 1
+        num = count_timer_object
+        assert_equal base_num, num
+      }
+
+      d.instance_shutdown
+    end
+
+    def count_timer_object
+      num = 0
+      ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|
+        num += 1
+      }
+      num
+    end
   end
 
   def test_z_refresh_watchers
@@ -1109,4 +1174,28 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
     end
   end
+
+  def test_limit_recently_modified
+    now = Time.new(2010, 1, 2, 3, 4, 5)
+    FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
+    FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
+    FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
+
+    config = config_element('', '', {
+      'tag' => 'tail',
+      'path' => "#{TMP_DIR}/*.txt",
+      'format' => 'none',
+      'limit_recently_modified' => '3600s'
+    })
+
+    expected_files = [
+      "#{TMP_DIR}/tail_watch1.txt",
+      "#{TMP_DIR}/tail_watch2.txt"
+    ]
+
+    Timecop.freeze(now) do
+      plugin = create_driver(config, false).instance
+      assert_equal expected_files, plugin.expand_paths.sort
+    end
+  end
 end

data/test/plugin/test_output.rb

@@ -706,6 +706,24 @@ class OutputTest < Test::Unit::TestCase
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
     end
 
+    test 'flush_interval is ignored when flush_mode is not interval' do
+      mock(@i.log).warn("'flush_interval' is ignored because default 'flush_mode' is not 'interval': 'lazy'")
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'flush_interval' => 10})]))
+    end
+
+    data(:lazy => 'lazy', :immediate => 'immediate')
+    test 'flush_interval and non-interval flush_mode is exclusive ' do |mode|
+      assert_raise Fluent::ConfigError.new("'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{mode}'") do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'flush_mode' => mode, 'flush_interval' => 10})]))
+      end
+    end
+
+    test 'flush_mode is set to interval when flush_interval with v0.12 configuration is given' do
+      mock(@i.log).info("'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour")
+      @i.configure(config_element('ROOT', '', {'flush_interval' => 60}, []))
+      assert_equal :interval, @i.instance_variable_get(:@flush_mode)
+    end
+
     test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
       o = create_output(:buffered)
       mock(o.log).warn("secondary type should be same with primary one",

data/test/plugin/test_output_as_buffered.rb

@@ -339,6 +339,41 @@ class BufferedOutputTest < Test::Unit::TestCase
       end
     end
 
+    data(:handle_stream_simple => '',
+         :handle_stream_with_custom_format => 'tag,message')
+    test 'plugin using custom format can skip record chunk when format return nil' do |chunk_keys|
+      events_from_chunk = []
+      @i = create_output(:custom)
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', chunk_keys, @hash)]))
+      @i.register(:prefer_delayed_commit) { false }
+      @i.register(:format) { |tag, time, record|
+        if record['message'] == 'test1'
+          nil
+        else
+          [tag,time,record].to_msgpack
+        end
+      }
+      @i.register(:format_type_is_msgpack) { true }
+      @i.register(:write){ |chunk| e = []; chunk.each { |ta, t, r| e << [ta, t, r] }; events_from_chunk << [:write, e] }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "test1"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "test2"}],
+      ]
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      waiting(5) { sleep 0.1 until events_from_chunk.size == 1 }
+
+      assert_equal 1, events_from_chunk.size
+      assert_equal :write, events_from_chunk[0][0]
+      each_pushed = events_from_chunk[0][1]
+      assert_equal 1, each_pushed.size
+      assert_equal 'test.tag', each_pushed[0][0]
+      assert_equal "test2", each_pushed[0][2]['message']
+    end
+
     test 'plugin using custom format can iterate chunk in #try_write if #format returns msgpack' do
       events_from_chunk = []
       @i = create_output(:custom)

data/test/plugin_helper/test_compat_parameters.rb

@@ -106,6 +106,7 @@ class CompatParameterTest < Test::Unit::TestCase
       assert_equal [], @i.buffer_config.chunk_keys
       assert_equal 8, @i.buffer_config.flush_thread_count
       assert_equal 10, @i.buffer_config.flush_interval
+      assert_equal :default, @i.buffer_config.flush_mode
       assert @i.buffer_config.flush_at_shutdown
 
       assert_equal 8*1024*1024, @i.buffer.chunk_limit_size
@@ -131,6 +132,7 @@ class CompatParameterTest < Test::Unit::TestCase
       assert @i.buffer_config.retry_forever
       assert_equal 60*60, @i.buffer_config.retry_max_interval
       assert_equal :block, @i.buffer_config.overflow_action
+      assert_equal :default, @i.buffer_config.flush_mode
 
       assert !@i.chunk_key_tag
       assert_equal [], @i.chunk_keys

data/test/plugin_helper/test_retry_state.rb

@@ -178,6 +178,16 @@ class RetryStateHelperTest < Test::Unit::TestCase
     assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
   end
 
+  test 'periodic retries with secondary and max_steps' do
+    s = @d.retry_state_create(:t3, :periodic, 3, 100, max_steps: 5, randomize: false, secondary: true)
+    dummy_current_time = s.start
+    override_current_time(s, dummy_current_time)
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + 3 * 5 * 0.8), s.secondary_transition_at
+  end
+
   test 'exponential backoff forever without randomization' do
     s = @d.retry_state_create(:t11, :exponential_backoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2)
     dummy_current_time = s.start
@@ -396,4 +406,17 @@ class RetryStateHelperTest < Test::Unit::TestCase
     assert_equal (dummy_current_time + 100), s.timeout_at
     assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at
   end
+
+  test 'exponential backoff retries with secondary and max_steps' do
+    s = @d.retry_state_create(:t15, :exponential_backoff, 1, 100, randomize: false, max_steps: 5, backoff_base: 2, secondary: true) # threshold 0.8
+    dummy_current_time = s.start
+    override_current_time(s, dummy_current_time)
+
+    timeout = 0
+    5.times { |i| timeout += 1.0 * (2 ** (i - 1)) }
+
+    assert_equal dummy_current_time, s.current_time
+    assert_equal (dummy_current_time + 100), s.timeout_at
+    assert_equal (dummy_current_time + timeout * 0.8), s.secondary_transition_at
+  end
 end

data/test/test_output.rb

@@ -2,6 +2,7 @@ require_relative 'helper'
 require 'fluent/test'
 require 'fluent/output'
 require 'fluent/output_chain'
+require 'fluent/plugin/buffer'
 require 'timecop'
 require 'flexmock/test_unit'
 
@@ -142,6 +143,10 @@ module FluentOutputTest
 
       assert_not_nil d.instance.instance_variable_get(:@secondary).router
     end
+
+    test 'BufferQueueLimitError compatibility' do
+      assert_equal Fluent::Plugin::Buffer::BufferOverflowError, Fluent::BufferQueueLimitError
+    end
   end
 
   class ObjectBufferedOutputTest < ::Test::Unit::TestCase