fluentd 1.7.2 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ce449e77af62c25fcef41773c3717738ec443550becdde0ca666710739f7eae
-  data.tar.gz: 00d125b1c53ac99c8d2edf8b915909b91e07642e7fdad35154147a3b5a472416
+  metadata.gz: b09f1cbc0b3f4be0bc2dedfc142a1f6f0ed195df071ec809c019987de83c0993
+  data.tar.gz: bfdc00fe223cbbef68da3472d1d99518c3a52562dc10e2f720912f93d40cd8d3
 SHA512:
-  metadata.gz: d78a18ab98f55b3568b0e401f87d80444d14b09b544fc4962a14bf821180c5f97a155332888961ce096adfa9ae6d01def4a154dc4a769939b6256846cf0b14b3
-  data.tar.gz: cc36a5137a9753d9888f6669d04dbfe943d35d3cb030f5718fa439f957e8658632e96246b219151b73f46a2ac74191a0f6e87f049aae1fe446b2e62fc3c4a274
+  metadata.gz: 1ac20a296903da6869d0cd28baafa4ec2e4ac001b96820f03417f7ec2913a5bf52641db630f9814f02b10ddafa993c5157e461dffae09688fba9cdcd704922e4
+  data.tar.gz: 4bc9cce3c2083b0de6d256cd6ffaa07c57e9390de886491d5f0cbb35b181b9af2b3c3a48ab619b6cecc4f9355a7fb0b71ae87f403db0a49ba13198149770f92b

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 # v1.7
 
+## Release v1.7.4 - 2019/10/24
+
+### Enhancement
+
+* in_http: Add `use_204_response` parameter to return proper 204 response instead of 200.
+  fluentd v2 will change this parameter to `true`.
+  https://github.com/fluent/fluentd/pull/2640
+
+### Bug fixes
+
+* child_process helper: fix stderr blocking for discard case
+  https://github.com/fluent/fluentd/pull/2649
+* log: Fix log rotation handling on Windows
+  https://github.com/fluent/fluentd/pull/2663
+
+## Release v1.7.3 - 2019/10/01
+
+### Enhancement
+
+* in_syslog: Replace priority_key with severity_key
+  https://github.com/fluent/fluentd/pull/2636
+
+### Bug fixes
+
+* out_forward: Fix nil error after purge obsoleted sockets in socket cache
+  https://github.com/fluent/fluentd/pull/2635
+* fix typo in ChangeLog
+  https://github.com/fluent/fluentd/pull/2633
+
 ## Release v1.7.2 - 2019/09/19
 
 ### Enhancement
@@ -36,7 +65,7 @@
   https://github.com/fluent/fluentd/pull/2598
 * buf_file_single: fix duplicated path setting check
   https://github.com/fluent/fluentd/pull/2600
-* fix msgpack-ruby depedency to use recent feature
+* fix msgpack-ruby dependency to use recent feature
   https://github.com/fluent/fluentd/pull/2606
 
 
@@ -640,7 +669,7 @@
 
 ## Release v1.0.0 - 2017/12/6
 
-See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
+See [CNCF announcement](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
 
 ### New features / Enhancements
 
@@ -910,7 +939,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1492
 * parser: Allow escape sequence in Apache access log
   https://github.com/fluent/fluentd/pull/1479
-* config: Add actual value in the placholder error message
+* config: Add actual value in the placeholder error message
   https://github.com/fluent/fluentd/pull/1497
 * log: Add Fluent::Log#<< to support some SDKs
   https://github.com/fluent/fluentd/pull/1478
@@ -1227,7 +1256,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1067
 * filter_record_transformer: Fix to prevent overwriting reserved placeholder keys
   https://github.com/fluent/fluentd/pull/1176
-* Migrate some build-in plugins into v0.14 API
+* Migrate some built-in plugins into v0.14 API
   https://github.com/fluent/fluentd/pull/1149
   https://github.com/fluent/fluentd/pull/1151
 * Update dependencies

data/lib/fluent/command/fluentd.rb

@@ -311,6 +311,16 @@ exit 0 if early_exit
 
 require 'fluent/supervisor'
 if opts[:supervise]
+  if Fluent.windows?
+    if opts[:log_path] && opts[:log_path] != "-"
+      if opts[:log_rotate_age] || opts[:log_rotate_size]
+        require 'pathname'
+
+        log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
+        opts[:log_path] = log_path
+      end
+    end
+  end
   Fluent::Supervisor.new(opts).run_supervisor
 else
   if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1

data/lib/fluent/plugin/in_http.rb

@@ -58,6 +58,8 @@ module Fluent::Plugin
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
+    desc 'Respond status code with 204.'
+    config_param :use_204_response, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -152,7 +154,11 @@ module Fluent::Plugin
           if @respond_with_empty_img
             return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
           else
-            return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+            if @use_204_response
+              return  ["204 No Content", {}]
+            else
+              return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+            end
           end
         end
 
@@ -219,7 +225,11 @@ module Fluent::Plugin
       if @respond_with_empty_img
         return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
       else
-        return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+        if @use_204_response
+          return  ["204 No Content", {}]
+        else
+          return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+        end
       end
     end
 

data/lib/fluent/plugin/in_syslog.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       23  => 'local7'
     }
 
-    PRIORITY_MAP = {
+    SEVERITY_MAP = {
       0  => 'emerg',
       1  => 'alert',
       2  => 'crit',
@@ -91,8 +91,8 @@ module Fluent::Plugin
     config_param :resolve_hostname, :bool, default: nil
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
-    desc 'The field name of the priority.'
-    config_param :priority_key, :string, default: nil
+    desc 'The field name of the severity.'
+    config_param :severity_key, :string, default: nil, alias: :priority_key
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
 
@@ -119,6 +119,10 @@ module Fluent::Plugin
 
       super
 
+      if conf.has_key?('priority_key')
+        log.warn "priority_key is deprecated. Use severity_key instead"
+      end
+
       @use_default = false
 
       @parser = parser_create
@@ -233,14 +237,14 @@ module Fluent::Plugin
 
         pri ||= record.delete('pri')
         facility = FACILITY_MAP[pri >> 3]
-        priority = PRIORITY_MAP[pri & 0b111]
+        severity = SEVERITY_MAP[pri & 0b111]
 
-        record[@priority_key] = priority if @priority_key
+        record[@severity_key] = severity if @severity_key
         record[@facility_key] = facility if @facility_key
         record[@source_address_key] = sock.remote_addr if @source_address_key
         record[@source_hostname_key] = sock.remote_host if @source_hostname_key
 
-        tag = "#{@tag}.#{facility}.#{priority}"
+        tag = "#{@tag}.#{facility}.#{severity}"
         emit(tag, time, record)
       end
     rescue => e

data/lib/fluent/plugin/out_forward/socket_cache.rb

@@ -81,7 +81,9 @@ module Fluent::Plugin
               end
             end
           end
-          @available_sockets = @available_sockets.select { |_, v| !v.empty? }
+
+          # reuse same object (@available_sockets)
+          @available_sockets.reject! { |_, v| v.empty? }
 
           sockets += @inactive_sockets
           @inactive_sockets.clear

data/lib/fluent/plugin_helper/child_process.rb

@@ -257,7 +257,8 @@ module Fluent
         readio = writeio = stderrio = wait_thread = nil
         readio_in_use = writeio_in_use = stderrio_in_use = false
 
-        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr) && stderr != :discard # connect
+        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
+          spawn_opts[:err] = IO::NULL if stderr == :discard
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -281,7 +282,7 @@ module Fluent
           stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
           stderrio_in_use = true
         else
-          stderrio.reopen(IO::NULL) if stderrio && stderrio == :discard
+          stderrio.reopen(IO::NULL) if stderrio && stderr == :discard
         end
 
         pid = wait_thread.pid # wait_thread => Process::Waiter

data/lib/fluent/supervisor.rb

@@ -353,13 +353,21 @@ module Fluent
         @log_rotate_size = log_rotate_size
       end
 
+      def worker_id_suffixed_path(worker_id, path)
+        require 'pathname'
+
+        Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
+      end
+
       def init(process_type, worker_id)
         @opts[:process_type] = process_type
         @opts[:worker_id] = worker_id
 
         if @path && @path != "-"
           @logdev = if @log_rotate_age || @log_rotate_size
-                     Fluent::LogDeviceIO.new(@path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
+                     Fluent::LogDeviceIO.new(Fluent.windows? ?
+                                               worker_id_suffixed_path(worker_id, @path) : @path,
+                                             shift_age: @log_rotate_age, shift_size: @log_rotate_size)
                    else
                      File.open(@path, "a")
                    end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.2'
+  VERSION = '1.7.4'
 
 end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -5,7 +5,7 @@ require 'timecop'
 
 class SocketCacheTest < Test::Unit::TestCase
   sub_test_case 'checkout_or' do
-    test 'when gived key does not exist' do
+    test 'when given key does not exist' do
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.open { 'socket' }.subject
       assert_equal('socket', c.checkout_or('key') { sock.open })
@@ -38,6 +38,16 @@ class SocketCacheTest < Test::Unit::TestCase
       sock = mock!.open { new_sock }.subject
       assert_equal(new_sock, c.checkout_or('key') { sock.open })
     end
+
+    test 'reuse same hash object after calling purge_obsolete_socks' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      c.checkout_or('key') { 'socket' }
+      c.purge_obsolete_socks
+
+      assert_nothing_raised(NoMethodError) do
+        c.checkout_or('key') { 'new socket' }
+      end
+    end
   end
 
   sub_test_case 'checkin' do

data/test/plugin/test_buffer_file_chunk.rb

@@ -495,7 +495,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
     end
   end
 
-  test 'ensure to remove metadata file if #write_metadata raise an error becuase of disk full' do
+  test 'ensure to remove metadata file if #write_metadata raise an error because of disk full' do
     chunk_path = File.join(@chunkdir, 'test.*.log')
     stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
 

data/test/plugin/test_in_http.rb

@@ -32,6 +32,7 @@ class HttpInputTest < Test::Unit::TestCase
     body_size_limit 10m
     keepalive_timeout 5
     respond_with_empty_img true
+    use_204_response false
   ]
 
   def create_driver(conf=CONFIG)
@@ -549,8 +550,8 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
-  def test_resonse_with_empty_img
-    d = create_driver(CONFIG + "respond_with_empty_img true")
+  def test_response_with_empty_img
+    d = create_driver(CONFIG)
     assert_equal true, d.instance.respond_with_empty_img
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -577,6 +578,61 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_response_without_empty_img
+    d = create_driver(CONFIG + "respond_with_empty_img false")
+    assert_equal false, d.instance.respond_with_empty_img
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
+  def test_response_use_204_response
+    d = create_driver(CONFIG + %[
+          respond_with_empty_img false
+          use_204_response true
+        ])
+    assert_equal true, d.instance.use_204_response
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["204", "204"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_cors_allowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins

data/test/plugin/test_in_syslog.rb

@@ -217,11 +217,15 @@ EOS
     compare_test_result(d.events, tests, {host: host})
   end
 
-  def test_msg_size_with_priority_key
-    d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
+  data(
+    severity_key: 'severity_key',
+    priority_key: 'priority_key',
+  )
+  def test_msg_size_with_severity_key(param_name)
+    d = create_driver([CONFIG, "#{param_name} severity"].join("\n"))
     tests = create_test_case
 
-    priority = 'info'
+    severity = 'info'
     d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
@@ -231,7 +235,7 @@ EOS
     end
 
     assert(d.events.size > 0)
-    compare_test_result(d.events, tests, {priority: priority})
+    compare_test_result(d.events, tests, {severity: severity})
   end
 
   def test_msg_size_with_facility_key
@@ -311,7 +315,7 @@ EOS
       assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
       assert_equal(options[:address], events[i][2]['source_address']) if options[:address]
       assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]
-      assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
+      assert_equal(options[:severity], events[i][2]['severity']) if options[:severity]
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end

data/test/plugin/test_output_as_buffered.rb

@@ -1112,7 +1112,7 @@ class BufferedOutputTest < Test::Unit::TestCase
         assert_equal(0, @i.write_count)
 
         Timecop.freeze(Time.parse('2019-02-09 00:00:12 +0900'))
-        # wirte should be called in few seconds since
+        # write should be called in few seconds since
         # running interval of enque thread is timekey_wait / 11.0.
         waiting(5){ sleep 0.1 until @i.write_count == 1 }
       end

data/test/plugin_helper/test_child_process.rb

@@ -124,6 +124,43 @@ class ChildProcessTest < Test::Unit::TestCase
     assert_equal expected, ary
   end
 
+  test 'can execute external command at just once, which can handle both of read and write. Ignore stderr message/no block' do
+    m = Mutex.new
+    ary = []
+    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+      ran = false
+      # lots of stderr message should not be blocked and message should not be printed in test.
+      cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; STDERR.puts line.chomp * 1000; STDOUT.puts line.chomp; STDOUT.flush rescue nil; end'"
+      @d.child_process_execute(:t2_and_ignore_stderr, cmd, mode: [:write, :read]) do |writeio, readio|
+        m.lock
+        ran = true
+
+        [[1,2],[3,4],[5,6]].each do |i,j|
+          writeio.write "my data#{i}\n"
+          writeio.write "my data#{j}\n"
+          writeio.flush
+        end
+        writeio.close
+
+        while line = readio.readline
+          ary << line
+        end
+        m.unlock
+      end
+      begin
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
+        m.lock
+      rescue
+      ensure
+        m.unlock
+      end
+    end
+
+    assert_equal [], @d.log.out.logs
+    expected = (1..6).map{|i| "my data#{i}\n" }
+    assert_equal expected, ary
+  end
+
   test 'can execute external command at just once, which can handle all of read, write and stderr' do
     m = Mutex.new
     ary1 = []

data/test/test_root_agent.rb

@@ -710,7 +710,7 @@ EOC
       end
     end
 
-    test 'raises configuration error for worker id collisions on multi workers syntax when multi avaliable worker_ids are left' do
+    test 'raises configuration error for worker id collisions on multi workers syntax when multi available worker_ids are left' do
       errmsg = "specified worker_id<1> collisions is detected on <worker> directive. Available worker id(s): [2, 3]"
       assert_raise Fluent::ConfigError.new(errmsg) do
         conf = <<-EOC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.7.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-20 00:00:00.000000000 Z
+date: 2019-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack