fluentd 0.14.8 → 0.14.9

data/test/test_output.rb

@@ -178,12 +178,24 @@ module FluentOutputTest
 
     CONFIG = %[
       buffer_path #{TMP_DIR}/foo
-      time_slice_format %Y%m%d
+      time_slice_format %Y%m%d%H
     ]
 
     class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput
+      attr_reader :written_chunk_keys, :errors_in_write
+      def initialize
+        super
+        @written_chunk_keys = []
+        @errors_in_write = []
+      end
       def format(tag, time, record)
-        ''
+        [tag, time, record].to_json + "\n"
+      end
+      def write(chunk)
+        @written_chunk_keys << chunk.key
+        true
+      rescue => e
+        @errors_in_write << e
       end
     end
 
@@ -209,6 +221,16 @@ module FluentOutputTest
           d.instance.emit_events('test', OneEventStream.new('string', 10))
         end
       end
+
+      test "plugin can get key of chunk in #write" do
+        d = create_driver
+        d.instance.start
+        d.instance.after_start
+        d.instance.emit_events('test', OneEventStream.new(event_time("2016-11-08 17:44:30 +0900"), {"message" => "yay"}))
+        d.instance.force_flush
+        assert_equal [], d.instance.errors_in_write
+        assert_equal ["2016110808"], d.instance.written_chunk_keys # default timezone is UTC
+      end
     end
   end
 end

data/test/test_plugin_classes.rb

@@ -110,6 +110,26 @@ module FluentTest
     end
   end
 
+  class FluentTestBufferedOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_buffered', self)
+    def write(chunk)
+      # drop everything
+    end
+  end
+
+  class FluentTestEmitOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_emit', self)
+    helpers :event_emitter
+    def write(chunk)
+      tag = chunk.metadata.tag || 'test'
+      array = []
+      chunk.each do |time, record|
+        array << [time, record]
+      end
+      router.emit_array(tag, array)
+    end
+  end
+
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 

data/test/test_root_agent.rb

@@ -207,6 +207,145 @@ EOC
     end
   end
 
+  sub_test_case 'configured with label and secondary plugin' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_buffered
+    <secondary>
+      @type test_out_emit
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label which the plugin is in' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+
+      assert_equal 1, c1.outputs.size
+      assert !c1.outputs.first.has_router?
+
+      assert c1.outputs.first.secondary
+      assert c1.outputs.first.secondary.has_router?
+      assert_equal c1.event_router, c1.outputs.first.secondary.router
+    end
+  end
+
+  sub_test_case 'configured with label and secondary plugin with @label specifier' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_buffered
+    <secondary>
+      @type test_out_emit
+      @label @route_b
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label specified in secondary section' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+      c2 = @ra.labels['@route_b']
+
+      assert_equal 1, c1.outputs.size
+      assert !c1.outputs.first.has_router?
+
+      assert c1.outputs.first.secondary
+      assert c1.outputs.first.secondary.has_router?
+      assert_equal c2.event_router, c1.outputs.first.secondary.router
+    end
+  end
+
+  sub_test_case 'configured with label and secondary plugin with @label specifier in primary output' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_emit
+    @label @route_b
+    <secondary>
+      @type test_out_emit
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label specified in secondary section' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+      c2 = @ra.labels['@route_b']
+
+      assert_equal 1, c1.outputs.size
+      assert c1.outputs.first.secondary
+
+      p1 = c1.outputs.first
+      assert p1.has_router?
+      assert_equal c1.event_router, p1.context_router
+      assert_equal c2.event_router, p1.router
+
+      s1 = p1.secondary
+      assert s1.has_router?
+      assert_equal c1.event_router, s1.context_router
+      assert_equal c2.event_router, s1.router
+    end
+  end
+
   sub_test_case 'configured with MultiOutput plugins' do
     setup do
       @ra = RootAgent.new(log: $log)

data/test/test_test_drivers.rb

@@ -0,0 +1,132 @@
+require_relative 'helper'
+require 'fluent/plugin/input'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/output'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/filter'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/multi_output'
+require 'fluent/test/driver/multi_output'
+require 'fluent/plugin/parser'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/formatter'
+require 'fluent/test/driver/formatter'
+
+require 'timecop'
+
+class TestDriverTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'plugin test driver' do
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'returns the block value as the return value of #run' do |args|
+      driver_class, plugin_class = args
+      d = driver_class.new(Class.new(plugin_class))
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'raises error for hard timeout' do |args|
+      driver_class, plugin_class = args
+      d = driver_class.new(Class.new(plugin_class))
+      assert_raise Fluent::Test::Driver::TestTimedOut do
+        d.run(timeout: 1) do
+          sleep 5
+        end
+      end
+    end
+
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'can stop with soft timeout for blocks never stops, even with Timecop' do |args|
+      Timecop.freeze(Time.parse("2016-11-04 18:49:00"))
+      begin
+        driver_class, plugin_class = args
+        d = driver_class.new(Class.new(plugin_class))
+        assert_nothing_raised do
+          before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          d.end_if{ false }
+          d.run(timeout: 1) do
+            sleep 0.1 until d.stop?
+          end
+          after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          assert{ after >= before + 1.0 }
+        end
+      ensure
+        Timecop.return
+      end
+    end
+
+    test 'raise errors raised in threads' do
+      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::Input) do
+        helpers :thread
+        def start
+          super
+          thread_create(:input_thread_for_test_driver_test) do
+            sleep 0.5
+            raise "yaaaaaaaaaay!"
+          end
+        end
+      end
+      assert_raise "yaaaaaaaaaay!" do
+        d.end_if{ false }
+        d.run(timeout: 3) do
+          sleep 0.1 until d.stop?
+        end
+      end
+    end
+  end
+
+  sub_test_case 'output plugin test driver' do
+    test 'returns the block value as the return value of #run' do
+      d = Fluent::Test::Driver::Output.new(Class.new(Fluent::Plugin::Output)) do
+        def process(tag, es)
+          # drop
+        end
+      end
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+  end
+
+  sub_test_case 'filter plugin test driver' do
+    test 'returns the block value as the return value of #run' do
+      d = Fluent::Test::Driver::Filter.new(Class.new(Fluent::Plugin::Filter)) do
+        def filter(tag, time, record)
+          record
+        end
+      end
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.8
+  version: 0.14.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2016-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -319,6 +319,7 @@ files:
 - example/in_udp.conf
 - example/multi_filters.conf
 - example/out_copy.conf
+- example/out_exec_filter.conf
 - example/out_file.conf
 - example/out_forward.conf
 - example/out_forward_buf_file.conf
@@ -402,6 +403,7 @@ files:
 - lib/fluent/plugin/file_wrapper.rb
 - lib/fluent/plugin/filter.rb
 - lib/fluent/plugin/filter_grep.rb
+- lib/fluent/plugin/filter_parser.rb
 - lib/fluent/plugin/filter_record_transformer.rb
 - lib/fluent/plugin/filter_stdout.rb
 - lib/fluent/plugin/formatter.rb
@@ -413,6 +415,7 @@ files:
 - lib/fluent/plugin/formatter_out_file.rb
 - lib/fluent/plugin/formatter_single_value.rb
 - lib/fluent/plugin/formatter_stdout.rb
+- lib/fluent/plugin/formatter_tsv.rb
 - lib/fluent/plugin/in_debug_agent.rb
 - lib/fluent/plugin/in_dummy.rb
 - lib/fluent/plugin/in_exec.rb
@@ -448,6 +451,7 @@ files:
 - lib/fluent/plugin/parser_csv.rb
 - lib/fluent/plugin/parser_json.rb
 - lib/fluent/plugin/parser_ltsv.rb
+- lib/fluent/plugin/parser_msgpack.rb
 - lib/fluent/plugin/parser_multiline.rb
 - lib/fluent/plugin/parser_nginx.rb
 - lib/fluent/plugin/parser_none.rb
@@ -536,6 +540,7 @@ files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_filter_record_transformer.rb
 - test/plugin/test_filter_stdout.rb
 - test/plugin/test_formatter_csv.rb
@@ -578,10 +583,10 @@ files:
 - test/plugin/test_output_as_buffered_secondary.rb
 - test/plugin/test_output_as_standard.rb
 - test/plugin/test_owned_by.rb
+- test/plugin/test_parser.rb
 - test/plugin/test_parser_apache.rb
 - test/plugin/test_parser_apache2.rb
 - test/plugin/test_parser_apache_error.rb
-- test/plugin/test_parser_base.rb
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
@@ -630,6 +635,7 @@ files:
 - test/test_process.rb
 - test/test_root_agent.rb
 - test/test_supervisor.rb
+- test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb
@@ -690,6 +696,7 @@ test_files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_filter_record_transformer.rb
 - test/plugin/test_filter_stdout.rb
 - test/plugin/test_formatter_csv.rb
@@ -732,10 +739,10 @@ test_files:
 - test/plugin/test_output_as_buffered_secondary.rb
 - test/plugin/test_output_as_standard.rb
 - test/plugin/test_owned_by.rb
+- test/plugin/test_parser.rb
 - test/plugin/test_parser_apache.rb
 - test/plugin/test_parser_apache2.rb
 - test/plugin/test_parser_apache_error.rb
-- test/plugin/test_parser_base.rb
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
@@ -784,6 +791,7 @@ test_files:
 - test/test_process.rb
 - test/test_root_agent.rb
 - test/test_supervisor.rb
+- test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb