fluentd 0.12.0.pre.1 → 0.12.0.pre.2

data/test/plugin/test_buf_memory.rb

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+require 'helper'
+require 'fluent/test'
+require 'fluent/plugin/buf_memory'
+
+require 'stringio'
+require 'msgpack'
+
+module FluentMemoryBufferTest
+  class MemoryBufferChunkTest < Test::Unit::TestCase
+    def test_init
+      chunk = Fluent::MemoryBufferChunk.new('key')
+      assert_equal 'key', chunk.key
+      assert_equal '', chunk.instance_eval{ @data }
+      assert_equal 'ASCII-8BIT', chunk.instance_eval{ @data }.encoding.to_s
+      assert chunk.unique_id # non nil
+
+      chunk2 = Fluent::MemoryBufferChunk.new('initdata', 'data')
+      assert_equal 'initdata', chunk2.key
+      assert_equal 'data', chunk2.instance_eval{ @data }
+    end
+
+    def test_buffer_chunk_interface
+      chunk = Fluent::BufferChunk.new('key')
+
+      assert chunk.respond_to?(:empty?)
+      assert chunk.respond_to?(:<<)
+      assert chunk.respond_to?(:size)
+      assert chunk.respond_to?(:close)
+      assert chunk.respond_to?(:purge)
+      assert chunk.respond_to?(:read)
+      assert chunk.respond_to?(:open)
+      assert chunk.respond_to?(:write_to)
+      assert chunk.respond_to?(:msgpack_each)
+    end
+
+    def test_empty?
+      chunk = Fluent::MemoryBufferChunk.new('key')
+      assert chunk.empty?
+
+      chunk.instance_eval{ @data = "non empty" }
+      assert !(chunk.empty?)
+    end
+
+    def test_append_data_and_size
+      chunk = Fluent::MemoryBufferChunk.new('key')
+      assert_equal '', chunk.instance_eval{ @data }
+
+      chunk << "foo bar baz\n".force_encoding('UTF-8')
+      assert_equal "foo bar baz\n", chunk.instance_eval{ @data }
+      assert_equal 'ASCII-8BIT', chunk.instance_eval{ @data }.encoding.to_s
+
+      assert_equal 12, chunk.size # bytesize
+
+      chunk << "日本語Japanese\n".force_encoding('UTF-8')
+      assert_equal "foo bar baz\n日本語Japanese\n".force_encoding('ASCII-8BIT'), chunk.instance_eval{ @data }
+      assert_equal 'ASCII-8BIT', chunk.instance_eval{ @data }.encoding.to_s
+
+      assert_equal 30, chunk.size # bytesize
+    end
+
+    def test_close_and_purge_does_nothing
+      chunk = Fluent::MemoryBufferChunk.new('key', 'data')
+      chunk.close
+      chunk.close
+      chunk.close
+      chunk.close
+      chunk.purge
+      chunk.purge
+      chunk.purge
+      chunk.purge
+      assert_equal 'data', chunk.instance_eval{ @data }
+    end
+
+    def test_read_just_returns_data
+      data = "data1\ndata2\n"
+      chunk = Fluent::MemoryBufferChunk.new('key', data)
+      assert_equal data, chunk.read
+      assert_equal data.object_id, chunk.read.object_id
+    end
+
+    def test_open
+      # StringIO.open(@data, &block)
+      chunk = Fluent::MemoryBufferChunk.new('key', 'foo bar baz')
+      chunk.open do |io|
+        assert 'foo bar baz', io.read
+      end
+    end
+
+    def test_write_to
+      chunk = Fluent::MemoryBufferChunk.new('key', 'foo bar baz')
+      dummy_dst = StringIO.new
+      chunk.write_to(dummy_dst)
+      assert_equal 'foo bar baz', dummy_dst.string
+    end
+
+    def test_msgpack_each
+      d0 = MessagePack.pack([[1, "foo"], [2, "bar"], [3, "baz"]])
+      d1 = MessagePack.pack({"key1" => "value1", "key2" => "value2"})
+      d2 = MessagePack.pack("string1")
+      d3 = MessagePack.pack(1)
+      d4 = MessagePack.pack(nil)
+      chunk = Fluent::MemoryBufferChunk.new('key', d0 + d1 + d2 + d3 + d4)
+
+      store = []
+      chunk.msgpack_each do |data|
+        store << data
+      end
+
+      assert_equal 5, store.size
+      assert_equal [[1, "foo"], [2, "bar"], [3, "baz"]], store[0]
+      assert_equal({"key1" => "value1", "key2" => "value2"}, store[1])
+      assert_equal "string1", store[2]
+      assert_equal 1, store[3]
+      assert_equal nil, store[4]
+    end
+  end
+
+  class MemoryBufferTest < Test::Unit::TestCase
+    def test_init_configure
+      buf = Fluent::MemoryBuffer.new
+
+      buf.configure({})
+      assert buf.flush_at_shutdown
+      assert_equal 64, buf.buffer_queue_limit
+    end
+
+    class DummyOutput
+      attr_accessor :written
+
+      def write(chunk)
+        @written ||= []
+        @written.push chunk
+        "return value"
+      end
+    end
+
+    def test_before_shutdown
+      buf = Fluent::MemoryBuffer.new
+      buf.start
+
+      # before_shutdown flushes all chunks in @map and @queue
+
+      c1 = [ buf.new_chunk('k0'), buf.new_chunk('k1'), buf.new_chunk('k2'), buf.new_chunk('k3') ]
+      c2 = [ buf.new_chunk('q0'), buf.new_chunk('q1') ]
+
+      buf.instance_eval do
+        @map = {
+          'k0' => c1[0], 'k1' => c1[1], 'k2' => c1[2], 'k3' => c1[3],
+          'q0' => c2[0], 'q1' => c2[1]
+        }
+      end
+      c1[0] << "data1\ndata2\n"
+      c1[1] << "data1\ndata2\n"
+      c1[2] << "data1\ndata2\n"
+      # k3 chunk is empty!
+
+      c2[0] << "data1\ndata2\n"
+      c2[1] << "data1\ndata2\n"
+      buf.push('q0')
+      buf.push('q1')
+
+      buf.instance_eval do
+        @enqueue_hook_times = 0
+        def enqueue(chunk)
+          @enqueue_hook_times += 1
+        end
+      end
+      assert_equal 0, buf.instance_eval{ @enqueue_hook_times }
+
+      out = DummyOutput.new
+      assert_equal nil, out.written
+
+      buf.before_shutdown(out)
+
+      assert_equal 3, buf.instance_eval{ @enqueue_hook_times } # k0, k1, k2
+      assert_equal 5, out.written.size
+      assert_equal [c2[0], c2[1], c1[0], c1[1], c1[2]], out.written
+    end
+
+    def test_new_chunk
+      buf = Fluent::MemoryBuffer.new
+      chunk = buf.new_chunk('key')
+      assert_equal Fluent::MemoryBufferChunk, chunk.class
+      assert_equal 'key', chunk.key
+      assert chunk.empty?
+    end
+
+    def test_resume
+      buf = Fluent::MemoryBuffer.new
+      resumed_queue, resumed_store = buf.resume
+      assert resumed_queue.empty?
+      assert resumed_store.empty?
+    end
+
+    def test_enqueue_does_nothing # enqueue hook
+      buf = Fluent::MemoryBuffer.new
+      chunk = Fluent::MemoryBufferChunk.new('k', "data1\ndata2\n")
+      assert_equal "data1\ndata2\n", chunk.read
+      buf.enqueue(chunk)
+      assert_equal "data1\ndata2\n", chunk.read
+    end
+  end
+end

data/test/plugin/test_filter_grep.rb

@@ -0,0 +1,124 @@
+require 'helper'
+require 'fluent/test'
+require 'fluent/plugin/filter_grep'
+
+# TODO: Replace with FilterTestDriver
+class GrepFilterTest < Test::Unit::TestCase
+  setup do
+    @filter = Fluent::GrepFilter.new
+    @time = Fluent::Engine.now
+  end
+
+  def configure(instance, conf, use_v1 = false)
+    config = if conf.is_a?(Fluent::Config::Element)
+               str
+             else
+               Fluent::Config.parse(conf, "(test)", "(test_dir)", use_v1)
+             end
+    instance.configure(config)
+    instance
+  end
+
+  sub_test_case 'configure' do
+    test 'check default' do
+      configure(@filter, '')
+      assert_empty(@filter.regexps)
+      assert_empty(@filter.excludes)
+    end
+
+    test "regexpN can contain a space" do
+      configure(@filter, %[regexp1 message  foo])
+      assert_equal(Regexp.compile(/ foo/), @filter.regexps['message'])
+    end
+
+    test "excludeN can contain a space" do
+      configure(@filter, %[exclude1 message  foo])
+      assert_equal(Regexp.compile(/ foo/), @filter.excludes['message'])
+    end
+  end
+
+  sub_test_case 'filter_stream' do
+    def messages
+      [
+        "2013/01/13T07:02:11.124202 INFO GET /ping",
+        "2013/01/13T07:02:13.232645 WARN POST /auth",
+        "2013/01/13T07:02:21.542145 WARN GET /favicon.ico",
+        "2013/01/13T07:02:43.632145 WARN POST /login",
+      ]
+    end
+
+    def emit(config, msgs)
+      es = Fluent::MultiEventStream.new
+      msgs.each { |msg|
+        es.add(@time, {'foo' => 'bar', 'message' => msg})
+      }
+
+      configure(@filter, config)
+      @filter.filter_stream('filter.test', es);
+    end
+
+    test 'empty config' do
+      es = emit('', messages)
+      assert_equal(4, es.instance_variable_get(:@record_array).size)
+    end
+
+    test 'regexpN' do
+      es = emit('regexp1 message WARN', messages)
+      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      assert_block('only WARN logs') do
+        es.all? { |t, r|
+          !r['message'].include?('INFO')
+        }
+      end
+    end
+
+    test 'excludeN' do
+      es = emit('exclude1 message favicon', messages)
+      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      assert_block('remove favicon logs') do
+        es.all? { |t, r|
+          !r['message'].include?('favicon')
+        }
+      end
+    end
+
+    sub_test_case 'with invalid sequence' do
+      def messages
+        [
+          "\xff".force_encoding('UTF-8'),
+        ]
+      end
+
+      test "don't raise an exception" do
+        assert_nothing_raised { 
+          emit(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
+        }
+      end
+    end
+  end
+
+  sub_test_case 'grep non-string jsonable values' do
+    def emit(msg, config = 'regexp1 message 0')
+      es = Fluent::MultiEventStream.new
+      es.add(@time, {'foo' => 'bar', 'message' => msg})
+
+      configure(@filter, config)
+      @filter.filter_stream('filter.test', es);
+    end
+
+    data(
+      'array' => ["0"],
+      'hash' => ["0" => "0"],
+      'integer' => 0,
+      'float' => 0.1)
+    test "value" do |data|
+      es = emit(data)
+      assert_equal(1, es.instance_variable_get(:@record_array).size)
+    end
+
+    test "value boolean" do
+      es = emit(true, %[regexp1 message true])
+      assert_equal(1, es.instance_variable_get(:@record_array).size)
+    end
+  end
+end

data/test/plugin/test_filter_record_transformer.rb

@@ -0,0 +1,251 @@
+require_relative '../helper'
+require 'timecop'
+require 'fluent/test'
+require 'fluent/plugin/filter_record_transformer'
+
+# TODO: Replace with FilterTestDriver
+class RecordTransformerFilterTest < Test::Unit::TestCase
+  setup do
+    @instance = Fluent::RecordTransformerFilter.new
+    @hostname = Socket.gethostname.chomp
+    @tag = 'test.tag'
+    @tag_parts = @tag.split('.')
+    @time = Time.utc(1,2,3,4,5,2010,nil,nil,nil,nil)
+    Timecop.freeze(@time)
+  end
+
+  teardown do
+    Timecop.return
+  end
+
+  def configure(instance, conf, use_v1 = false)
+    config = if conf.is_a?(Fluent::Config::Element)
+               str
+             else
+               Fluent::Config.parse(conf, "(test)", "(test_dir)", use_v1)
+             end
+    instance.configure(config)
+    instance
+  end
+
+  sub_test_case 'configure' do
+    test 'check default' do
+      assert_nothing_raised do
+        configure(@instance, '')
+      end
+    end
+
+    test "keep_keys must be specified together with renew_record true" do
+      assert_raise(Fluent::ConfigError) do
+        configure(@instance, %[keep_keys a])
+      end
+    end
+  end
+
+  sub_test_case "test options" do
+    def emit(config, msgs = [''])
+      es = Fluent::MultiEventStream.new
+      msgs.each do |msg|
+        es.add(@time, {'foo' => 'bar', 'message' => msg})
+      end
+
+      configure(@instance, config)
+      @instance.filter_stream(@tag, es)
+    end
+
+    CONFIG = %[
+      <record>
+        hostname ${hostname}
+        tag ${tag}
+        time ${time}
+        message ${hostname} ${tag_parts[-1]} ${message}
+      </record>
+    ]
+
+    test 'typical usage' do
+      msgs = ['1', '2']
+      es = emit(CONFIG, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal('bar', r['foo'])
+        assert_equal(@hostname, r['hostname'])
+        assert_equal(@tag, r['tag'])
+        assert_equal(@time.to_s, r['time'])
+        assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+      end
+    end
+
+    test 'remove_keys' do
+      config = CONFIG + %[remove_keys foo,message]
+      es = emit(config)
+      es.each_with_index do |(t, r), i|
+        assert_not_include(r, 'foo')
+        assert_equal(@hostname, r['hostname'])
+        assert_equal(@tag, r['tag'])
+        assert_equal(@time.to_s, r['time'])
+        assert_not_include(r, 'message')
+      end
+    end
+
+    test 'renew_record' do
+      config = CONFIG + %[renew_record true]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_not_include(r, 'foo')
+        assert_equal(@hostname, r['hostname'])
+        assert_equal(@tag, r['tag'])
+        assert_equal(@time.to_s, r['time'])
+        assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+      end
+    end
+
+    test 'keep_keys' do
+      config = %[renew_record true\nkeep_keys foo,message]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal('bar', r['foo'])
+        assert_equal(msgs[i], r['message'])
+      end
+    end
+
+    test 'enable_ruby' do
+      config = %[
+        enable_ruby yes
+        <record>
+          message ${hostname} ${tag_parts.last} ${URI.encode(message)}
+        </record>
+      ]
+      msgs = ['1', '2']
+      es = emit(config, msgs)
+      es.each_with_index do |(t, r), i|
+        assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+      end
+    end
+  end
+
+  sub_test_case 'test placeholders' do
+    def emit(config, msgs = [''])
+      es = Fluent::MultiEventStream.new
+      msgs.each do |msg|
+        es.add(@time, {'eventType0' => 'bar', 'message' => msg})
+      end
+
+      configure(@instance, config)
+      @instance.filter_stream(@tag, es)
+    end
+
+    %w[yes no].each do |enable_ruby|
+      test "hostname with enble_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            message ${hostname}
+          </record>
+        ]
+        es = emit(config)
+        es.each do |t, r|
+          assert_equal(@hostname, r['message'])
+        end
+      end
+
+      test "tag with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            message ${tag}
+          </record>
+        ]
+        es = emit(config)
+        es.each do |t, r|
+          assert_equal(@tag, r['message'])
+        end
+      end
+
+      test "tag_parts with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            message ${tag_parts[0]} ${tag_parts[-1]}
+          </record>
+        ]
+        expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
+        es = emit(config)
+        es.each do |t, r|
+          assert_equal(expected, r['message'])
+        end
+      end
+
+      test "${tag_prefix[N]} and ${tag_suffix[N]} with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            message ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]}
+          </record>
+        ]
+        @tag = 'prefix.test.tag.suffix'
+        expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix"
+        es = emit(config)
+        es.each do |t, r|
+          assert_equal(expected, r['message'])
+        end
+      end
+
+      test "time with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            message ${time}
+          </record>
+        ]
+        es = emit(config)
+        es.each do |t, r|
+          assert_equal(@time.to_s, r['message'])
+        end
+      end
+
+      test "record keys with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          remove_keys eventType0
+          <record>
+            message bar ${message}
+            eventtype ${eventType0}
+          </record>
+        ]
+        msgs = ['1', '2']
+        es = emit(config, msgs)
+        es.each_with_index do |(t, r), i|
+          assert_not_include(r, 'eventType0')
+          assert_equal("bar", r['eventtype'])
+          assert_equal("bar #{msgs[i]}", r['message'])
+        end
+      end
+    end
+
+    test 'unknown placeholder (enable_ruby no)' do
+      config = %[
+        enable_ruby no
+        <record>
+          message ${unknown}
+        </record>
+      ]
+      mock(@instance.log).warn("unknown placeholder `${unknown}` found")
+      emit(config)
+    end
+
+    test 'failed to expand (enable_ruby yes)' do
+      config = %[
+        enable_ruby yes
+        <record>
+          message ${unknown['bar']}
+        </record>
+      ]
+      mock(@instance.log).warn("failed to expand `${unknown['bar']}`", anything)
+      es = emit(config)
+      es.each do |t, r|
+        assert_nil(r['message'])
+      end
+    end
+  end
+end