fluentd 0.14.14 → 0.14.15

data/test/plugin/test_in_http.rb

@@ -215,14 +215,14 @@ class HttpInputTest < Test::Unit::TestCase
   def test_multi_json_with_add_remote_addr_given_x_forwarded_for
     d = create_driver(CONFIG + "add_remote_addr true")
 
+    tag = "tag1"
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
     records = [{"a"=>1},{"a"=>2}]
     events = [
-      ["tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}],
-      ["tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}],
+      [tag, time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}],
+      [tag, time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}],
     ]
-    tag = "tag1"
     res_codes = []
 
     d.run(expect_records: 2, timeout: 5) do
@@ -235,6 +235,37 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_add_remote_addr_given_multi_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+
+    tag = "tag1"
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    record = {"a" => 1}
+    event = ["tag1", time, {"REMOTE_ADDR" => "129.78.138.66", "a" => 1}]
+    res_code = nil
+
+    d.run(expect_records: 1, timeout: 5) do
+      res = post("/#{tag}", {"json" => record.to_json, "time" => time_i.to_s}) { |http, req|
+        # net/http can't send multiple headers so overwrite it.
+        def req.each_capitalized
+          block_given? or return enum_for(__method__) { @header.size }
+          @header.each do |k, vs|
+            vs.each { |v|
+              yield capitalize(k), v
+            }
+          end
+        end
+        req.add_field("X-Forwarded-For", "129.78.138.66, 127.0.0.1")
+        req.add_field("X-Forwarded-For", "8.8.8.8")
+      }
+      res_code = res.code
+    end
+    assert_equal "200", res_code
+    assert_equal event, d.events.first
+    assert_equal_event_time time, d.events.first[1]
+  end
+
   def test_multi_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -565,9 +596,10 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
   end
 
-  def post(path, params, header = {})
+  def post(path, params, header = {}, &block)
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Post.new(path, header)
+    block.call(http, req) if block
     if params.is_a?(String)
       unless header.has_key?('Content-Type')
         header['Content-Type'] = 'application/octet-stream'

data/test/plugin/test_out_file.rb

@@ -20,6 +20,9 @@ class FileOutputTest < Test::Unit::TestCase
     path #{TMP_DIR}/out_file_test
     compress gz
     utc
+    <buffer>
+      timekey_use_utc true
+    </buffer>
   ]
 
   def create_driver(conf = CONFIG, opts = {})
@@ -405,6 +408,9 @@ class FileOutputTest < Test::Unit::TestCase
       path #{TMP_DIR_WITH_SYSTEM}/out_file_test
       compress gz
       utc
+      <buffer>
+        timekey_use_utc true
+      </buffer>
       <system>
         file_permission #{OVERRIDE_FILE_PERMISSION}
         dir_permission #{OVERRIDE_DIR_PERMISSION}
@@ -527,6 +533,9 @@ class FileOutputTest < Test::Unit::TestCase
         compress gz
         utc
         append true
+        <buffer>
+          timekey_use_utc true
+        </buffer>
       ]
       d.run(default_tag: 'test'){
         d.feed(time, {"a"=>1})

data/test/plugin/test_output_as_buffered.rb

@@ -75,6 +75,31 @@ module FluentPluginOutputAsBufferedTest
     end
   end
   class DummyCustomFormatBufferedOutput < DummyBareOutput
+    def initialize
+      super
+      @format_type_is_msgpack = nil
+      @prefer_delayed_commit = nil
+      @write = nil
+      @try_write = nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def formatted_to_msgpack_binary?
+      @format_type_is_msgpack ? @format_type_is_msgpack.call : false
+    end
+    def prefer_delayed_commit
+      @prefer_delayed_commit ? @prefer_delayed_commit.call : false
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  # check for formatted_to_msgpack_binary compatibility
+  class DummyOldCustomFormatBufferedOutput < DummyBareOutput
     def initialize
       super
       @format_type_is_msgpack = nil
@@ -163,6 +188,7 @@ class BufferedOutputTest < Test::Unit::TestCase
     when :full     then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new
     when :old_buf  then FluentPluginOutputAsBufferedTest::DummyOldBufferedOutput.new
     when :old_obj  then FluentPluginOutputAsBufferedTest::DummyOldObjectBufferedOutput.new
+    when :old_custom then FluentPluginOutputAsBufferedTest::DummyOldCustomFormatBufferedOutput.new
     else
       raise ArgumentError, "unknown type: #{type}"
     end
@@ -306,9 +332,11 @@ class BufferedOutputTest < Test::Unit::TestCase
       assert_equal 0, events_from_chunk.size
     end
 
-    test 'plugin using custom format can iterate chunk in #write if #format returns msgpack' do
+    data('formatted_to_msgpack_binary?' => :custom,
+         'formatted_to_msgpack_binary' => :old_custom)
+    test 'plugin using custom format can iterate chunk in #write if #format returns msgpack' do |out_type|
       events_from_chunk = []
-      @i = create_output(:custom)
+      @i = create_output(out_type)
       @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
       @i.register(:prefer_delayed_commit){ false }
       @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }

data/test/test_plugin_classes.rb

@@ -235,6 +235,21 @@ module FluentTest
     end
   end
 
+  class FluentTestBuffer < Fluent::Plugin::Buffer
+    ::Fluent::Plugin.register_buffer('test_buffer', self)
+
+    def resume
+      return {}, []
+    end
+
+    def generate_chunk(metadata)
+    end
+
+    def multi_workers_ready?
+      false
+    end
+  end
+
   class TestEmitErrorHandler
     def initialize
       @events = Hash.new { |h, k| h[k] = [] }

data/test/test_root_agent.rb

@@ -608,4 +608,208 @@ EOC
       assert_equal [true, true], dyn_out.child.outputs.map{|i| i.terminated? }
     end
   end
+
+  sub_test_case 'configured at worker2 with 4 workers environment' do
+    setup do
+      ENV['SERVERENGINE_WORKER_ID'] = '2'
+      @ra = RootAgent.new(log: $log)
+      system_config = SystemConfig.new
+      system_config.workers = 4
+      stub(Engine).worker_id { 2 }
+      stub(Engine).root_agent { @ra }
+      stub(Engine).system_config { system_config }
+      @ra
+    end
+
+    teardown '' do
+      ENV.delete('SERVERENGINE_WORKER_ID')
+    end
+
+    def configure_ra(conf_str)
+      conf = Config.parse(conf_str, "(test)", "(test_dir)", true)
+      @ra.configure(conf)
+      @ra
+    end
+
+    test 'raises configuration error for missing worker id' do
+      errmsg = 'Missing worker id on <worker> directive'
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for too big worker id' do
+      errmsg = "worker id 4 specified by <worker> directive is not allowed. Available worker id is between 0 and 3"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 4>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for invalid elements as a child of worker section' do
+      errmsg = '<worker> section cannot have <system> directive'
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 2>
+<system>
+</system>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error when configured plugins do not have support multi worker configuration' do
+      errmsg = "Plugin 'test_out' does not support multi workers configuration (FluentTest::FluentTestOutput)"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<match **>
+@type test_out
+</match>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'does not raise configuration error when configured plugins in worker section do not have support multi worker configuration' do
+      assert_nothing_raised do
+        conf = <<-EOC
+<worker 2>
+<match **>
+  @type test_out
+</match>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'does not raise configuration error when configured plugins as a children of MultiOutput in worker section do not have support multi worker configuration' do
+      assert_nothing_raised do
+        conf = <<-EOC
+<worker 2>
+<match **>
+  @type copy
+  <store>
+    @type test_out
+  </store>
+  <store>
+    @type test_out
+  </store>
+</match>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'does not raise configuration error when configured plugins owned by plugin do not have support multi worker configuration' do
+      assert_nothing_raised do
+        conf = <<-EOC
+<worker 2>
+<match **>
+  @type test_out_buffered
+  <buffer>
+    @type test_buffer
+  </buffer>
+</match>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'with plugins' do
+      conf = <<-EOC
+<worker 2>
+<source>
+  @type test_in
+  @id test_in
+</source>
+<filter>
+  type test_filter
+  id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    type test_out
+    id test_out
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+  </match>
+</label>
+</worker>
+EOC
+      ra = configure_ra(conf)
+      assert_kind_of FluentTestInput, ra.inputs.first
+      assert_kind_of Plugin::RelabelOutput, ra.outputs.first
+      assert_kind_of FluentTestFilter, ra.filters.first
+      assert ra.error_collector
+
+      %W(@test @ERROR).each { |label_symbol|
+        assert_include ra.labels, label_symbol
+        assert_kind_of Label, ra.labels[label_symbol]
+      }
+
+      test_label = ra.labels['@test']
+      assert_kind_of FluentTestOutput, test_label.outputs.first
+      assert_equal ra, test_label.root_agent
+
+      error_label = ra.labels['@ERROR']
+      assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first
+      assert_kind_of RootAgent::RootAgentProxyWithoutErrorCollector, error_label.root_agent
+    end
+
+    test 'with plugins but for another worker' do
+      conf = <<-EOC
+<worker 0>
+<source>
+  @type test_in
+  @id test_in
+</source>
+<filter>
+  type test_filter
+  id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    type test_out
+    id test_out
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+  </match>
+</label>
+</worker>
+EOC
+      ra = configure_ra(conf)
+      assert_equal 0, ra.inputs.size
+      assert_equal 0, ra.outputs.size
+      assert_equal 0, ra.filters.size
+      assert_equal 0, ra.labels.size
+      refute ra.error_collector
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.14
+  version: 0.14.15
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-24 00:00:00.000000000 Z
+date: 2017-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -348,6 +348,7 @@ files:
 - example/suppress_config_dump.conf
 - example/v0_12_filter.conf
 - example/v1_literal_example.conf
+- example/worker_section.conf
 - fluent.conf
 - fluentd.gemspec
 - lib/fluent/agent.rb
@@ -713,7 +714,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector