fluentd 0.14.6 → 0.14.7

data/test/plugin_helper/test_extract.rb

@@ -0,0 +1,195 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/extract'
+require 'fluent/time'
+
+class ExtractHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :extract
+  end
+
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :extract
+    config_section :extract do
+      config_set_default :tag_key, 'tag2'
+    end
+  end
+
+  def config_extract_section(hash = {})
+    config_element('ROOT', '', {}, [config_element('extract', '', hash)])
+  end
+
+  setup do
+    Fluent::Test.setup
+    @d = Dummy.new
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can override default parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    d.configure(config_element())
+    assert_nil d.extract_config
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('extract')]))
+    assert d.extract_config
+    assert_equal 'tag2', d.extract_config.tag_key
+  end
+
+  test 'returns nil in default' do
+    @d.configure(config_extract_section())
+    @d.start
+    assert_nil @d.instance_eval{ @_extract_tag_key }
+    assert_nil @d.instance_eval{ @_extract_time_key }
+    assert_nil @d.instance_eval{ @_extract_time_parser }
+
+    time = event_time()
+    record = {"key1" => "value1", "key2" => 2, "tag" => "yay", "time" => Time.now.to_i}
+
+    assert_nil @d.extract_tag_from_record(record)
+    assert_nil @d.extract_time_from_record(record)
+  end
+
+  test 'can be configured as specified' do
+    @d.configure(config_extract_section(
+        "tag_key" => "tag",
+        "time_key" => "time",
+        "time_type" => "unixtime",
+    ))
+
+    assert_equal "tag", @d.instance_eval{ @_extract_tag_key }
+    assert_equal "time", @d.instance_eval{ @_extract_time_key }
+    assert_equal :unixtime, @d.instance_eval{ @extract_config.time_type }
+    assert_not_nil @d.instance_eval{ @_extract_time_parser }
+  end
+
+  sub_test_case 'extract_tag_from_record' do
+    test 'returns tag string from specified tag_key field' do
+      @d.configure(config_extract_section("tag_key" => "tag"))
+      @d.start
+      @d.after_start
+
+      tag = @d.extract_tag_from_record({"tag" => "tag.test.code", "message" => "yay!"})
+      assert_equal "tag.test.code", tag
+    end
+
+    test 'returns tag as string by stringifying values from specified key' do
+      @d.configure(config_extract_section("tag_key" => "tag"))
+      @d.start
+      @d.after_start
+
+      tag = @d.extract_tag_from_record({"tag" => 100, "message" => "yay!"})
+      assert_equal "100", tag
+    end
+  end
+
+  sub_test_case 'extract_time_from_record' do
+    test 'returns EventTime object from specified time_key field, parsed as float in default' do
+      @d.configure(config_extract_section("time_key" => "t"))
+      @d.start
+      @d.after_start
+
+      # 1473135272 => 2016-09-06 04:14:32 UTC
+      t = @d.extract_time_from_record({"t" => 1473135272.5, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)
+
+      t = @d.extract_time_from_record({"t" => "1473135272.5", "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 500_000_000), t)
+    end
+
+    test 'returns EventTime object, parsed as unixtime when configured so' do
+      @d.configure(config_extract_section("time_key" => "t", "time_type" => "unixtime"))
+      @d.start
+      @d.after_start
+
+      t = @d.extract_time_from_record({"t" => 1473135272, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+
+      t = @d.extract_time_from_record({"t" => "1473135272", "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+
+      t = @d.extract_time_from_record({"t" => 1473135272.5, "message" => "yay!"})
+      assert_equal_event_time(Fluent::EventTime.new(1473135272, 0), t)
+    end
+
+    test 'returns EventTime object, parsed by default time parser of ruby with timezone in data' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "2016-09-06 13:27:01 +0900", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0900"), t)
+    end
+
+    test 'returns EventTime object, parsed by default time parser of ruby as localtime' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "2016-09-06 13:27:01", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0200"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format with timezone' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y, %z"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016, -0700", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 -0700"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in localtime without timezone' do
+      t = with_timezone("UTC-02") do
+        @d.configure(config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y"))
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0200"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in utc without timezone, localtime: false' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "localtime" => "false")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 UTC"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in utc without timezone, utc: true' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "utc" => "true")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 UTC"), t)
+    end
+
+    test 'returns EventTime object, parsed as configured time_format in configured timezone' do
+      t = with_timezone("UTC-02") do
+        c = config_extract_section("time_key" => "t", "time_type" => "string", "time_format" => "%H:%M:%S, %m/%d/%Y", "timezone" => "+09:00")
+        @d.configure(c)
+        @d.start
+        @d.after_start
+        @d.extract_time_from_record({"t" => "13:27:01, 09/06/2016", "message" => "yay!"})
+      end
+      assert_equal_event_time(event_time("2016-09-06 13:27:01 +0900"), t)
+    end
+  end
+end

data/test/plugin_helper/test_inject.rb

@@ -410,13 +410,6 @@ class InjectHelperTest < Test::Unit::TestCase
       @time = event_time("2014-09-27 00:00:00 +00:00").to_i
     end
 
-    def with_timezone(tz)
-      oldtz, ENV['TZ'] = ENV['TZ'], tz
-      yield
-    ensure
-      ENV['TZ'] = oldtz
-    end
-
     def format(conf)
       @d.configure(config_inject_section(
           "hostname_key" => "hostnamedata",

data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb

@@ -0,0 +1,7 @@
+module Fluent
+  Plugin.register_formatter(
+    'test1',
+    Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb

@@ -0,0 +1,7 @@
+module Fluent
+  Plugin.register_formatter(
+    'test2',
+    Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/test_event.rb

@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/event'
+require 'fluent/plugin/compressable'
 
 module EventTest
   module DeepCopyAssertion
@@ -23,6 +24,7 @@ module EventTest
   class OneEventStreamTest < ::Test::Unit::TestCase
     include Fluent
     include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
       @time = event_time()
@@ -85,11 +87,28 @@ module EventTest
         assert_equal @record, record
       }
     end
+
+    test 'to_compressed_msgpack_stream' do
+      stream = @es.to_compressed_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      }
+    end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      stream = @es.to_compressed_msgpack_stream(time_int: true)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+        assert_equal @time.to_i, time
+        assert_equal @record, record
+      }
+    end
   end
 
   class ArrayEventStreamTest < ::Test::Unit::TestCase
     include Fluent
     include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
       time = Engine.now
@@ -161,11 +180,34 @@ module EventTest
         i += 1
       }
     end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i].to_i, time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
   end
 
   class MultiEventStreamTest < ::Test::Unit::TestCase
     include Fluent
     include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
       time = Engine.now
@@ -240,11 +282,34 @@ module EventTest
         i += 1
       }
     end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream with time_int argument' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i].to_i, time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
   end
 
   class MessagePackEventStreamTest < ::Test::Unit::TestCase
     include Fluent
     include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
 
     def setup
       pk = Fluent::Engine.msgpack_factory.packer
@@ -325,5 +390,126 @@ module EventTest
         i += 1
       }
     end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      compressed_stream = @es.to_compressed_msgpack_stream
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+  end
+
+  class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+    include Fluent::Plugin::Compressable
+
+    def setup
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
+      @records = [{ 'k' => 'v1', 'n' => 1 }, { 'k' => 'v2', 'n' => 2 }]
+      @packed_record = ''
+      @entries = ''
+      @times.zip(@records).each do |_time, record|
+        v = [_time, record].to_msgpack
+        @packed_record += v
+        @entries += compress(v)
+      end
+      @es = CompressedMessagePackEventStream.new(@entries)
+    end
+
+    def ensure_data_is_decompressed
+      assert_equal @entries, @es.instance_variable_get(:@data)
+      yield
+      assert_equal @packed_record, @es.instance_variable_get(:@data)
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of CompressedMessagePackEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+
+      # After iteration of events (done in assert_duplicated_records),
+      # duplicated event stream still has unpacked objects and correct size
+      dupped = @es.dup
+      assert_equal 2, dupped.instance_eval{ @size }
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'size' do
+      assert_equal 0, CompressedMessagePackEventStream.new('').size
+      ensure_data_is_decompressed { assert_equal 2, @es.size  }
+    end
+
+    test 'each' do
+      i = 0
+      ensure_data_is_decompressed do
+        @es.each do |time, record|
+          assert_equal @times[i], time
+          assert_equal @records[i], record
+          i += 1
+        end
+      end
+    end
+
+    test 'slice' do
+      sliced = nil
+      ensure_data_is_decompressed { sliced = @es.slice(1,1) }
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
+    test 'to_msgpack_stream' do
+      i = 0
+      stream = nil
+      ensure_data_is_decompressed { stream = @es.to_msgpack_stream }
+
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_compressed_msgpack_stream' do
+      i = 0
+      # Do not call ensure_decompressed!
+      assert_equal @entries, @es.instance_variable_get(:@data)
+      compressed_stream = @es.to_compressed_msgpack_stream
+      assert_equal @entries, @es.instance_variable_get(:@data)
+
+      stream = decompress(compressed_stream)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
   end
 end