fluentd 0.14.1 → 0.14.2

data/lib/fluent/test/driver/filter.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/test/driver/base'
+require 'fluent/test/driver/base_owner'
 require 'fluent/test/driver/event_feeder'
 
 require 'fluent/plugin/filter'
@@ -22,12 +22,34 @@ require 'fluent/plugin/filter'
 module Fluent
   module Test
     module Driver
-      class Filter < Base
+      class Filter < BaseOwner
         include EventFeeder
 
+        attr_reader :filtered
+
         def initialize(klass, opts: {}, &block)
           super
           raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Filter" unless @instance.is_a? Fluent::Plugin::Filter
+          @filtered = []
+        end
+
+        def filtered_records
+          @filtered.map {|_time, record| record }
+        end
+
+        def instance_hook_after_started
+          super
+          filter_hook = ->(time, record) { @filtered << [time, record] }
+          m = Module.new do
+            define_method(:filter_stream) do |tag, es|
+              new_es = super(tag, es)
+              new_es.each do |time, record|
+                filter_hook.call(time, record)
+              end
+              new_es
+            end
+          end
+          @instance.singleton_class.prepend(m)
         end
       end
     end

data/lib/fluent/test/driver/input.rb

@@ -14,13 +14,13 @@
 #    limitations under the License.
 #
 
-require 'fluent/test/driver/base'
+require 'fluent/test/driver/base_owner'
 require 'fluent/plugin/input'
 
 module Fluent
   module Test
     module Driver
-      class Input < Base
+      class Input < BaseOwner
         def initialize(klass, opts: {}, &block)
           super
           raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Input" unless @instance.is_a? Fluent::Plugin::Input

data/lib/fluent/test/driver/multi_output.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/test/driver/base'
+require 'fluent/test/driver/base_owner'
 require 'fluent/test/driver/event_feeder'
 
 require 'fluent/plugin/multi_output'
@@ -22,7 +22,7 @@ require 'fluent/plugin/multi_output'
 module Fluent
   module Test
     module Driver
-      class MultiOutput < Base
+      class MultiOutput < BaseOwner
         include EventFeeder
 
         def initialize(klass, opts: {}, &block)

data/lib/fluent/test/driver/output.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/test/driver/base'
+require 'fluent/test/driver/base_owner'
 require 'fluent/test/driver/event_feeder'
 
 require 'fluent/plugin/output'
@@ -22,7 +22,7 @@ require 'fluent/plugin/output'
 module Fluent
   module Test
     module Driver
-      class Output < Base
+      class Output < BaseOwner
         include EventFeeder
 
         def initialize(klass, opts: {}, &block)
@@ -67,9 +67,7 @@ module Fluent
                 result
               end
             end
-            @instance.singleton_class.module_eval do
-              prepend m
-            end
+            @instance.singleton_class.prepend m
           end
         end
       end

data/lib/fluent/test/helpers.rb

@@ -37,6 +37,22 @@ module Fluent
         end
       end
 
+      def time2str(time, localtime: false, format: nil)
+        if format
+          if localtime
+            Time.at(time).strftime(format)
+          else
+            Time.at(time).utc.strftime(format)
+          end
+        else
+          if localtime
+            Time.at(time).iso8601
+          else
+            Time.at(time).utc.iso8601
+          end
+        end
+      end
+
       def msgpack(type)
         case type
         when :factory
@@ -49,6 +65,15 @@ module Fluent
           raise ArgumentError, "unknown msgpack object type '#{type}'"
         end
       end
+
+      def capture_log(driver)
+        tmp = driver.instance.log.out
+        driver.instance.log.out = StringIO.new
+        yield
+        return driver.instance.log.out.string
+      ensure
+        driver.instance.log.out = tmp
+      end
     end
   end
 end

data/lib/fluent/test/input_test.rb

@@ -110,12 +110,12 @@ module Fluent
 
       def run(num_waits = 10, &block)
         m = method(:emit_stream)
-        (class << Engine; self; end).module_eval do
-          prepend EmitStreamWrapper
+        unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
+          Engine.singleton_class.prepend EmitStreamWrapper
         end
         Engine.emit_stream_callee = m
-        (class << instance.router; self; end).module_eval do
-          prepend EmitStreamWrapper
+        unless instance.router.singleton_class.ancestors.include?(EmitStreamWrapper)
+          instance.router.singleton_class.prepend EmitStreamWrapper
         end
         instance.router.emit_stream_callee = m
 

data/lib/fluent/test/output_test.rb

@@ -84,9 +84,9 @@ module Fluent
           end
 
           chunk = if @instance.instance_eval{ @chunk_key_tag }
-                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil))
+                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil)).staged!
                   else
-                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil))
+                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil)).staged!
                   end
           chunk.concat(buffer, es.size)
 
@@ -140,7 +140,7 @@ module Fluent
           end
 
           lines.keys.each do |meta|
-            chunk = @instance.buffer.generate_chunk(meta)
+            chunk = @instance.buffer.generate_chunk(meta).staged!
             chunk.append(lines[meta])
             begin
               result.push(@instance.write(chunk))

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.1'
+  VERSION = '0.14.2'
 
 end

data/test/config/test_element.rb

@@ -1,17 +1,13 @@
 require_relative '../helper'
 require 'fluent/config/element'
+require 'fluent/config/configure_proxy'
+require 'fluent/configurable'
 
 class TestConfigElement < ::Test::Unit::TestCase
   def element(name = 'ROOT', arg = '', attrs = {}, elements = [], unused = nil)
     Fluent::Config::Element.new(name, arg, attrs, elements, unused)
   end
 
-  sub_test_case 'Most test' do
-    test 'should be written' do
-      pend 'TODO'
-    end
-  end
-
   sub_test_case '#elements=' do
     test 'elements can be set by others' do
       e = element()
@@ -271,4 +267,137 @@ CONF
       assert_equal(dump.chomp, e.inspect)
     end
   end
+
+  sub_test_case 'for sections which has secret parameter' do
+    setup do
+      @type_lookup = ->(type){ Fluent::Configurable.lookup_type(type) }
+      p1 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup)
+      p1.config_param :str1, :string
+      p1.config_param :str2, :string, secret: true
+      p1.config_param :enum1, :enum, list: [:a, :b, :c]
+      p1.config_param :enum2, :enum, list: [:a, :b, :c], secret: true
+      p1.config_param :bool1, :bool
+      p1.config_param :bool2, :bool, secret: true
+      p1.config_param :int1, :integer
+      p1.config_param :int2, :integer, secret: true
+      p1.config_param :float1, :float
+      p1.config_param :float2, :float, secret: true
+      p2 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup)
+      p2.config_param :size1, :size
+      p2.config_param :size2, :size, secret: true
+      p2.config_param :time1, :time
+      p2.config_param :time2, :time, secret: true
+      p2.config_param :array1, :array
+      p2.config_param :array2, :array, secret: true
+      p2.config_param :hash1, :hash
+      p2.config_param :hash2, :hash, secret: true
+      p1.config_section :mysection do
+        config_param :str1, :string
+        config_param :str2, :string, secret: true
+        config_param :enum1, :enum, list: [:a, :b, :c]
+        config_param :enum2, :enum, list: [:a, :b, :c], secret: true
+        config_param :bool1, :bool
+        config_param :bool2, :bool, secret: true
+        config_param :int1, :integer
+        config_param :int2, :integer, secret: true
+        config_param :float1, :float
+        config_param :float2, :float, secret: true
+        config_param :size1, :size
+        config_param :size2, :size, secret: true
+        config_param :time1, :time
+        config_param :time2, :time, secret: true
+        config_param :array1, :array
+        config_param :array2, :array, secret: true
+        config_param :hash1, :hash
+        config_param :hash2, :hash, secret: true
+      end
+      params = {
+        'str1' => 'aaa', 'str2' => 'bbb', 'enum1' => 'a', 'enum2' => 'b', 'bool1' => 'true', 'bool2' => 'yes',
+        'int1' => '1', 'int2' => '2', 'float1' => '1.0', 'float2' => '0.5', 'size1' => '1k', 'size2' => '1m',
+        'time1' => '5m', 'time2' => '3h', 'array1' => 'a,b,c', 'array2' => 'd,e,f',
+        'hash1' => 'a:1,b:2', 'hash2' => 'a:2,b:4',
+        'unknown1' => 'yay', 'unknown2' => 'boo',
+      }
+      e2 = Fluent::Config::Element.new('mysection', '', params.dup, [])
+      e2.corresponding_proxies << p1.sections.values.first
+      @e = Fluent::Config::Element.new('match', '**', params, [e2])
+      @e.corresponding_proxies << p1
+      @e.corresponding_proxies << p2
+    end
+
+    sub_test_case '#to_masked_element' do
+      test 'returns a new element object which has masked values for secret parameters and elements' do
+        e = @e.to_masked_element
+        assert_equal 'aaa',    e['str1']
+        assert_equal 'xxxxxx', e['str2']
+        assert_equal 'a',      e['enum1']
+        assert_equal 'xxxxxx', e['enum2']
+        assert_equal 'true',   e['bool1']
+        assert_equal 'xxxxxx', e['bool2']
+        assert_equal '1',      e['int1']
+        assert_equal 'xxxxxx', e['int2']
+        assert_equal '1.0',    e['float1']
+        assert_equal 'xxxxxx', e['float2']
+        assert_equal '1k',     e['size1']
+        assert_equal 'xxxxxx', e['size2']
+        assert_equal '5m',     e['time1']
+        assert_equal 'xxxxxx', e['time2']
+        assert_equal 'a,b,c',  e['array1']
+        assert_equal 'xxxxxx', e['array2']
+        assert_equal 'a:1,b:2', e['hash1']
+        assert_equal 'xxxxxx',  e['hash2']
+        assert_equal 'yay', e['unknown1']
+        assert_equal 'boo', e['unknown2']
+        e2 = e.elements.first
+        assert_equal 'aaa',    e2['str1']
+        assert_equal 'xxxxxx', e2['str2']
+        assert_equal 'a',      e2['enum1']
+        assert_equal 'xxxxxx', e2['enum2']
+        assert_equal 'true',   e2['bool1']
+        assert_equal 'xxxxxx', e2['bool2']
+        assert_equal '1',      e2['int1']
+        assert_equal 'xxxxxx', e2['int2']
+        assert_equal '1.0',    e2['float1']
+        assert_equal 'xxxxxx', e2['float2']
+        assert_equal '1k',     e2['size1']
+        assert_equal 'xxxxxx', e2['size2']
+        assert_equal '5m',     e2['time1']
+        assert_equal 'xxxxxx', e2['time2']
+        assert_equal 'a,b,c',  e2['array1']
+        assert_equal 'xxxxxx', e2['array2']
+        assert_equal 'a:1,b:2', e2['hash1']
+        assert_equal 'xxxxxx',  e2['hash2']
+        assert_equal 'yay', e2['unknown1']
+        assert_equal 'boo', e2['unknown2']
+      end
+    end
+
+    sub_test_case '#secret_param?' do
+      test 'returns boolean which shows values of given key will be masked' do
+        assert !@e.secret_param?('str1')
+        assert @e.secret_param?('str2')
+        assert !@e.elements.first.secret_param?('str1')
+        assert @e.elements.first.secret_param?('str2')
+      end
+    end
+
+    sub_test_case '#param_type' do
+      test 'returns parameter type which are registered in corresponding proxy' do
+        assert_equal :string, @e.param_type('str1')
+        assert_equal :string, @e.param_type('str2')
+        assert_equal :enum, @e.param_type('enum1')
+        assert_equal :enum, @e.param_type('enum2')
+        assert_nil @e.param_type('unknown1')
+        assert_nil @e.param_type('unknown2')
+      end
+    end
+
+    # sub_test_case '#dump_value'
+    sub_test_case '#dump_value' do
+      test 'dumps parameter_name and values with leading indentation' do
+        assert_equal "str1 aaa\n", @e.dump_value("str1", @e["str1"], "")
+        assert_equal "str2 xxxxxx\n", @e.dump_value("str2", @e["str2"], "")
+      end
+    end
+  end
 end

data/test/plugin/test_buf_file.rb

@@ -181,7 +181,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert c1.is_a? Fluent::Plugin::Buffer::FileChunk
       assert_equal m1, c1.metadata
       assert c1.empty?
-      assert_equal :staged, c1.state
+      assert_equal :unstaged, c1.state
       assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c1.permission
       assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path
       assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
@@ -191,7 +191,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert c2.is_a? Fluent::Plugin::Buffer::FileChunk
       assert_equal m2, c2.metadata
       assert c2.empty?
-      assert_equal :staged, c2.state
+      assert_equal :unstaged, c2.state
       assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission
       assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path
       assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') }
@@ -221,7 +221,7 @@ class FileBufferTest < Test::Unit::TestCase
       assert c.is_a? Fluent::Plugin::Buffer::FileChunk
       assert_equal m, c.metadata
       assert c.empty?
-      assert_equal :staged, c.state
+      assert_equal :unstaged, c.state
       assert_equal 0600, c.permission
       assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path
       assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
@@ -500,4 +500,72 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at
     end
   end
+
+  sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))
+
+      @not_chunk = File.join(@bufdir, 'etest.20160416.log')
+      File.open(@not_chunk, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-16 23:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-16 23:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-16 23:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-16 23:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(@not_chunk, mtime: Time.parse('2016-04-17 00:00:00 -0700'))
+
+      @bufpath = File.join(@bufdir, 'etest.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns queued chunks for files without metadata, while ignoring non-chunk looking files' do
+      assert_equal 0, @p.stage.size
+      assert_equal 1, @p.queue.size
+
+      queue = @p.queue
+
+      m = metadata()
+
+      assert_equal @c1id, queue[0].unique_id
+      assert_equal m, queue[0].metadata
+      assert_equal 0, queue[0].size
+      assert_equal :queued, queue[0].state
+      assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at
+
+      assert File.exist?(@not_chunk)
+    end
+  end
 end