fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

data/test/test_event_time.rb

@@ -51,7 +51,9 @@ class EventTimeTest < Test::Unit::TestCase
 
   test 'Oj.dump' do
     time = Fluent::EventTime.new(100)
-    assert_equal('{"time":100}', Oj.dump({'time' => time}, mode: :compat))
+    require 'fluent/env'
+    Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+    assert_equal('{"time":100}', Oj.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
   end
 

data/test/test_output.rb

@@ -178,12 +178,32 @@ module FluentOutputTest
 
     CONFIG = %[
       buffer_path #{TMP_DIR}/foo
-      time_slice_format %Y%m%d
+      time_slice_format %Y%m%d%H
     ]
 
     class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput
+      attr_reader :written_chunk_keys, :errors_in_write
+      def initialize
+        super
+        @written_chunk_keys = []
+        @errors_in_write = []
+      end
+
+      def configure(conf)
+        super
+
+        @formatter = Fluent::Plugin.new_formatter('out_file')
+        @formatter.configure(conf)
+      end
+
       def format(tag, time, record)
-        ''
+        @formatter.format(tag, time, record)
+      end
+      def write(chunk)
+        @written_chunk_keys << chunk.key
+        true
+      rescue => e
+        @errors_in_write << e
       end
     end
 
@@ -191,6 +211,16 @@ module FluentOutputTest
       Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)
     end
 
+    data(:none => '',
+         :utc => "utc",
+         :localtime => 'localtime',
+         :timezone => 'timezone +0000')
+    test 'configure with timezone related parameters' do |param|
+      assert_nothing_raised {
+        create_driver(CONFIG + param)
+      }
+    end
+
     sub_test_case "test emit" do
       setup do
         @time = Time.parse("2011-01-02 13:14:15 UTC")
@@ -209,6 +239,27 @@ module FluentOutputTest
           d.instance.emit_events('test', OneEventStream.new('string', 10))
         end
       end
+
+      test "plugin can get key of chunk in #write" do
+        d = create_driver
+        d.instance.start
+        d.instance.after_start
+        d.instance.emit_events('test', OneEventStream.new(event_time("2016-11-08 17:44:30 +0900"), {"message" => "yay"}))
+        d.instance.force_flush
+        assert_equal [], d.instance.errors_in_write
+        assert_equal ["2016110808"], d.instance.written_chunk_keys # default timezone is UTC
+      end
+
+      test "check formatted time compatibility with utc. Should Z, not +00:00" do
+        d = create_driver(CONFIG + %[
+          utc
+          include_time_key
+        ])
+        time = Time.parse("2016-11-08 12:00:00 UTC").to_i
+        d.emit({"a" => 1}, time)
+        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}\n]
+        d.run
+      end
     end
   end
 end

data/test/test_plugin_classes.rb

@@ -110,6 +110,26 @@ module FluentTest
     end
   end
 
+  class FluentTestBufferedOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_buffered', self)
+    def write(chunk)
+      # drop everything
+    end
+  end
+
+  class FluentTestEmitOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_emit', self)
+    helpers :event_emitter
+    def write(chunk)
+      tag = chunk.metadata.tag || 'test'
+      array = []
+      chunk.each do |time, record|
+        array << [time, record]
+      end
+      router.emit_array(tag, array)
+    end
+  end
+
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 

data/test/test_root_agent.rb

@@ -207,6 +207,145 @@ EOC
     end
   end
 
+  sub_test_case 'configured with label and secondary plugin' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_buffered
+    <secondary>
+      @type test_out_emit
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label which the plugin is in' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+
+      assert_equal 1, c1.outputs.size
+      assert !c1.outputs.first.has_router?
+
+      assert c1.outputs.first.secondary
+      assert c1.outputs.first.secondary.has_router?
+      assert_equal c1.event_router, c1.outputs.first.secondary.router
+    end
+  end
+
+  sub_test_case 'configured with label and secondary plugin with @label specifier' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_buffered
+    <secondary>
+      @type test_out_emit
+      @label @route_b
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label specified in secondary section' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+      c2 = @ra.labels['@route_b']
+
+      assert_equal 1, c1.outputs.size
+      assert !c1.outputs.first.has_router?
+
+      assert c1.outputs.first.secondary
+      assert c1.outputs.first.secondary.has_router?
+      assert_equal c2.event_router, c1.outputs.first.secondary.router
+    end
+  end
+
+  sub_test_case 'configured with label and secondary plugin with @label specifier in primary output' do
+    setup do
+      @ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent{ @ra }
+      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+<source>
+  @type test_in
+  @label @route_a
+</source>
+<label @route_a>
+  <match a.**>
+    @type test_out_emit
+    @label @route_b
+    <secondary>
+      @type test_out_emit
+    </secondary>
+  </match>
+</label>
+<label @route_b>
+  <match b.**>
+    @type test_out
+  </match>
+</label>
+EOC
+    end
+
+    test 'secondary plugin has an event router for the label specified in secondary section' do
+      assert_equal 1, @ra.inputs.size
+      assert_equal 2, @ra.labels.size
+      assert_equal ['@route_a', '@route_b'], @ra.labels.keys
+      assert_equal '@route_a', @ra.labels['@route_a'].context
+      assert_equal '@route_b', @ra.labels['@route_b'].context
+
+      c1 = @ra.labels['@route_a']
+      c2 = @ra.labels['@route_b']
+
+      assert_equal 1, c1.outputs.size
+      assert c1.outputs.first.secondary
+
+      p1 = c1.outputs.first
+      assert p1.has_router?
+      assert_equal c1.event_router, p1.context_router
+      assert_equal c2.event_router, p1.router
+
+      s1 = p1.secondary
+      assert s1.has_router?
+      assert_equal c1.event_router, s1.context_router
+      assert_equal c2.event_router, s1.router
+    end
+  end
+
   sub_test_case 'configured with MultiOutput plugins' do
     setup do
       @ra = RootAgent.new(log: $log)

data/test/test_test_drivers.rb

@@ -0,0 +1,135 @@
+require_relative 'helper'
+require 'fluent/plugin/input'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/output'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/filter'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/multi_output'
+require 'fluent/test/driver/multi_output'
+require 'fluent/plugin/parser'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/formatter'
+require 'fluent/test/driver/formatter'
+
+require 'timecop'
+
+class TestDriverTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'plugin test driver' do
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'returns the block value as the return value of #run' do |args|
+      driver_class, plugin_class = args
+      d = driver_class.new(Class.new(plugin_class))
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'raises error for hard timeout' do |args|
+      driver_class, plugin_class = args
+      d = driver_class.new(Class.new(plugin_class))
+      assert_raise Fluent::Test::Driver::TestTimedOut do
+        d.run(timeout: 1) do
+          sleep 5
+        end
+      end
+    end
+
+    data(
+      'input plugin test driver'  => [Fluent::Test::Driver::Input, Fluent::Plugin::Input],
+      'multi_output plugin test driver' => [Fluent::Test::Driver::MultiOutput, Fluent::Plugin::MultiOutput],
+      'parser plugin test driver'       => [Fluent::Test::Driver::Parser, Fluent::Plugin::Parser],
+      'formatter plugin test driver'    => [Fluent::Test::Driver::Formatter, Fluent::Plugin::Formatter],
+    )
+    test 'can stop with soft timeout for blocks never stops, even with Timecop' do |args|
+      Timecop.freeze(Time.parse("2016-11-04 18:49:00"))
+      begin
+        driver_class, plugin_class = args
+        d = driver_class.new(Class.new(plugin_class))
+        assert_nothing_raised do
+          before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          d.end_if{ false }
+          d.run(timeout: 1) do
+            sleep 0.1 until d.stop?
+          end
+          after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          assert{ after >= before + 1.0 }
+        end
+      ensure
+        Timecop.return
+      end
+    end
+
+    test 'raise errors raised in threads' do
+      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::Input) do
+        helpers :thread
+        def start
+          super
+          thread_create(:input_thread_for_test_driver_test) do
+            sleep 0.5
+            raise "yaaaaaaaaaay!"
+          end
+        end
+      end
+      assert_raise "yaaaaaaaaaay!" do
+        d.end_if{ false }
+        d.run(timeout: 3) do
+          sleep 0.1 until d.stop?
+        end
+      end
+    end
+  end
+
+  sub_test_case 'output plugin test driver' do
+    test 'returns the block value as the return value of #run' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::Output) do
+        def prefer_buffered_processing
+          false
+        end
+        def process(tag, es)
+          # drop
+        end
+      end
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+  end
+
+  sub_test_case 'filter plugin test driver' do
+    test 'returns the block value as the return value of #run' do
+      d = Fluent::Test::Driver::Filter.new(Fluent::Plugin::Filter) do
+        def filter(tag, time, record)
+          record
+        end
+      end
+      v = d.run do
+        x = 1 + 2
+        y = 2 + 4
+        3 || x || y
+      end
+      assert_equal 3, v
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.7
+  version: 0.14.10
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-07 00:00:00.000000000 Z
+date: 2016-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -62,16 +62,22 @@ dependencies:
   name: serverengine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
   requirement: !ruby/object:Gem::Requirement
@@ -360,6 +366,7 @@ files:
 - bin/fluent-debug
 - bin/fluent-gem
 - bin/fluentd
+- code-of-conduct.md
 - example/copy_roundrobin.conf
 - example/filter_stdout.conf
 - example/in_dummy_with_compression.conf
@@ -375,6 +382,7 @@ files:
 - example/in_udp.conf
 - example/multi_filters.conf
 - example/out_copy.conf
+- example/out_exec_filter.conf
 - example/out_file.conf
 - example/out_forward.conf
 - example/out_forward_buf_file.conf
@@ -458,6 +466,7 @@ files:
 - lib/fluent/plugin/file_wrapper.rb
 - lib/fluent/plugin/filter.rb
 - lib/fluent/plugin/filter_grep.rb
+- lib/fluent/plugin/filter_parser.rb
 - lib/fluent/plugin/filter_record_transformer.rb
 - lib/fluent/plugin/filter_stdout.rb
 - lib/fluent/plugin/formatter.rb
@@ -469,6 +478,7 @@ files:
 - lib/fluent/plugin/formatter_out_file.rb
 - lib/fluent/plugin/formatter_single_value.rb
 - lib/fluent/plugin/formatter_stdout.rb
+- lib/fluent/plugin/formatter_tsv.rb
 - lib/fluent/plugin/in_debug_agent.rb
 - lib/fluent/plugin/in_dummy.rb
 - lib/fluent/plugin/in_exec.rb
@@ -504,6 +514,7 @@ files:
 - lib/fluent/plugin/parser_csv.rb
 - lib/fluent/plugin/parser_json.rb
 - lib/fluent/plugin/parser_ltsv.rb
+- lib/fluent/plugin/parser_msgpack.rb
 - lib/fluent/plugin/parser_multiline.rb
 - lib/fluent/plugin/parser_nginx.rb
 - lib/fluent/plugin/parser_none.rb
@@ -524,6 +535,9 @@ files:
 - lib/fluent/plugin_helper/inject.rb
 - lib/fluent/plugin_helper/parser.rb
 - lib/fluent/plugin_helper/retry_state.rb
+- lib/fluent/plugin_helper/server.rb
+- lib/fluent/plugin_helper/socket.rb
+- lib/fluent/plugin_helper/socket_option.rb
 - lib/fluent/plugin_helper/storage.rb
 - lib/fluent/plugin_helper/thread.rb
 - lib/fluent/plugin_helper/timer.rb
@@ -592,6 +606,7 @@ files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_filter_record_transformer.rb
 - test/plugin/test_filter_stdout.rb
 - test/plugin/test_formatter_csv.rb
@@ -634,10 +649,10 @@ files:
 - test/plugin/test_output_as_buffered_secondary.rb
 - test/plugin/test_output_as_standard.rb
 - test/plugin/test_owned_by.rb
+- test/plugin/test_parser.rb
 - test/plugin/test_parser_apache.rb
 - test/plugin/test_parser_apache2.rb
 - test/plugin/test_parser_apache_error.rb
-- test/plugin/test_parser_base.rb
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
@@ -659,6 +674,7 @@ files:
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_retry_state.rb
+- test/plugin_helper/test_server.rb
 - test/plugin_helper/test_storage.rb
 - test/plugin_helper/test_thread.rb
 - test/plugin_helper/test_timer.rb
@@ -686,6 +702,7 @@ files:
 - test/test_process.rb
 - test/test_root_agent.rb
 - test/test_supervisor.rb
+- test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb
@@ -746,6 +763,7 @@ test_files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_filter_record_transformer.rb
 - test/plugin/test_filter_stdout.rb
 - test/plugin/test_formatter_csv.rb
@@ -788,10 +806,10 @@ test_files:
 - test/plugin/test_output_as_buffered_secondary.rb
 - test/plugin/test_output_as_standard.rb
 - test/plugin/test_owned_by.rb
+- test/plugin/test_parser.rb
 - test/plugin/test_parser_apache.rb
 - test/plugin/test_parser_apache2.rb
 - test/plugin/test_parser_apache_error.rb
-- test/plugin/test_parser_base.rb
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
@@ -813,6 +831,7 @@ test_files:
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_retry_state.rb
+- test/plugin_helper/test_server.rb
 - test/plugin_helper/test_storage.rb
 - test/plugin_helper/test_thread.rb
 - test/plugin_helper/test_timer.rb
@@ -840,6 +859,7 @@ test_files:
 - test/test_process.rb
 - test/test_root_agent.rb
 - test/test_supervisor.rb
+- test/test_test_drivers.rb
 - test/test_time_formatter.rb
 - test/test_time_parser.rb
 - test/test_unique_id.rb