fluentd 1.7.3 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 027c04b3446f864debe4b57849a0c30a2f3c24aac96f69e629d437a265ffafde
-  data.tar.gz: '0996f7d53774c1fc151b589b12570baf39f59b9edb39e761a40ea40f37908189'
+  metadata.gz: b09f1cbc0b3f4be0bc2dedfc142a1f6f0ed195df071ec809c019987de83c0993
+  data.tar.gz: bfdc00fe223cbbef68da3472d1d99518c3a52562dc10e2f720912f93d40cd8d3
 SHA512:
-  metadata.gz: bf68902c8b6947a65956451d3601856d95f147428f3e3fc9b5f6a746ac15c02eaecb79a191c7811b605ca462310fac5849ce80167f37d1148c86f4f7e5598de4
-  data.tar.gz: 2c35f820be92a1e3741472c227c5fc8532aa06ef208001db981dd527d2bfa45c30d53888a4326e175e7db460980ade85a2f6b16bde6511940887d0786fdd9ad1
+  metadata.gz: 1ac20a296903da6869d0cd28baafa4ec2e4ac001b96820f03417f7ec2913a5bf52641db630f9814f02b10ddafa993c5157e461dffae09688fba9cdcd704922e4
+  data.tar.gz: 4bc9cce3c2083b0de6d256cd6ffaa07c57e9390de886491d5f0cbb35b181b9af2b3c3a48ab619b6cecc4f9355a7fb0b71ae87f403db0a49ba13198149770f92b

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # v1.7
 
+## Release v1.7.4 - 2019/10/24
+
+### Enhancement
+
+* in_http: Add `use_204_response` parameter to return proper 204 response instead of 200.
+  fluentd v2 will change this parameter to `true`.
+  https://github.com/fluent/fluentd/pull/2640
+
+### Bug fixes
+
+* child_process helper: fix stderr blocking for discard case
+  https://github.com/fluent/fluentd/pull/2649
+* log: Fix log rotation handling on Windows
+  https://github.com/fluent/fluentd/pull/2663
+
 ## Release v1.7.3 - 2019/10/01
 
 ### Enhancement

data/lib/fluent/command/fluentd.rb

@@ -311,6 +311,16 @@ exit 0 if early_exit
 
 require 'fluent/supervisor'
 if opts[:supervise]
+  if Fluent.windows?
+    if opts[:log_path] && opts[:log_path] != "-"
+      if opts[:log_rotate_age] || opts[:log_rotate_size]
+        require 'pathname'
+
+        log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
+        opts[:log_path] = log_path
+      end
+    end
+  end
   Fluent::Supervisor.new(opts).run_supervisor
 else
   if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1

data/lib/fluent/plugin/in_http.rb

@@ -58,6 +58,8 @@ module Fluent::Plugin
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
+    desc 'Respond status code with 204.'
+    config_param :use_204_response, :bool, default: false
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -152,7 +154,11 @@ module Fluent::Plugin
           if @respond_with_empty_img
             return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
           else
-            return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+            if @use_204_response
+              return  ["204 No Content", {}]
+            else
+              return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+            end
           end
         end
 
@@ -219,7 +225,11 @@ module Fluent::Plugin
       if @respond_with_empty_img
         return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
       else
-        return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+        if @use_204_response
+          return  ["204 No Content", {}]
+        else
+          return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+        end
       end
     end
 

data/lib/fluent/plugin_helper/child_process.rb

@@ -257,7 +257,8 @@ module Fluent
         readio = writeio = stderrio = wait_thread = nil
         readio_in_use = writeio_in_use = stderrio_in_use = false
 
-        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr) && stderr != :discard # connect
+        if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
+          spawn_opts[:err] = IO::NULL if stderr == :discard
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -281,7 +282,7 @@ module Fluent
           stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
           stderrio_in_use = true
         else
-          stderrio.reopen(IO::NULL) if stderrio && stderrio == :discard
+          stderrio.reopen(IO::NULL) if stderrio && stderr == :discard
         end
 
         pid = wait_thread.pid # wait_thread => Process::Waiter

data/lib/fluent/supervisor.rb

@@ -353,13 +353,21 @@ module Fluent
         @log_rotate_size = log_rotate_size
       end
 
+      def worker_id_suffixed_path(worker_id, path)
+        require 'pathname'
+
+        Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
+      end
+
       def init(process_type, worker_id)
         @opts[:process_type] = process_type
         @opts[:worker_id] = worker_id
 
         if @path && @path != "-"
           @logdev = if @log_rotate_age || @log_rotate_size
-                     Fluent::LogDeviceIO.new(@path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
+                     Fluent::LogDeviceIO.new(Fluent.windows? ?
+                                               worker_id_suffixed_path(worker_id, @path) : @path,
+                                             shift_age: @log_rotate_age, shift_size: @log_rotate_size)
                    else
                      File.open(@path, "a")
                    end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.3'
+  VERSION = '1.7.4'
 
 end

data/test/plugin/test_in_http.rb

@@ -32,6 +32,7 @@ class HttpInputTest < Test::Unit::TestCase
     body_size_limit 10m
     keepalive_timeout 5
     respond_with_empty_img true
+    use_204_response false
   ]
 
   def create_driver(conf=CONFIG)
@@ -549,8 +550,8 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
-  def test_resonse_with_empty_img
-    d = create_driver(CONFIG + "respond_with_empty_img true")
+  def test_response_with_empty_img
+    d = create_driver(CONFIG)
     assert_equal true, d.instance.respond_with_empty_img
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -577,6 +578,61 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_response_without_empty_img
+    d = create_driver(CONFIG + "respond_with_empty_img false")
+    assert_equal false, d.instance.respond_with_empty_img
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
+  def test_response_use_204_response
+    d = create_driver(CONFIG + %[
+          respond_with_empty_img false
+          use_204_response true
+        ])
+    assert_equal true, d.instance.use_204_response
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["204", "204"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_cors_allowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins

data/test/plugin_helper/test_child_process.rb

@@ -124,6 +124,43 @@ class ChildProcessTest < Test::Unit::TestCase
     assert_equal expected, ary
   end
 
+  test 'can execute external command at just once, which can handle both of read and write. Ignore stderr message/no block' do
+    m = Mutex.new
+    ary = []
+    Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+      ran = false
+      # lots of stderr message should not be blocked and message should not be printed in test.
+      cmd = "ruby -e 'while !STDIN.eof? && line = STDIN.readline; STDERR.puts line.chomp * 1000; STDOUT.puts line.chomp; STDOUT.flush rescue nil; end'"
+      @d.child_process_execute(:t2_and_ignore_stderr, cmd, mode: [:write, :read]) do |writeio, readio|
+        m.lock
+        ran = true
+
+        [[1,2],[3,4],[5,6]].each do |i,j|
+          writeio.write "my data#{i}\n"
+          writeio.write "my data#{j}\n"
+          writeio.flush
+        end
+        writeio.close
+
+        while line = readio.readline
+          ary << line
+        end
+        m.unlock
+      end
+      begin
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
+        m.lock
+      rescue
+      ensure
+        m.unlock
+      end
+    end
+
+    assert_equal [], @d.log.out.logs
+    expected = (1..6).map{|i| "my data#{i}\n" }
+    assert_equal expected, ary
+  end
+
   test 'can execute external command at just once, which can handle all of read, write and stderr' do
     m = Mutex.new
     ary1 = []

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.7.3
+  version: 1.7.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-01 00:00:00.000000000 Z
+date: 2019-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack