fluentd 1.10.3 → 1.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a83d0f39bb55eb7343c7a19e126bd98ebf9ab907439ce95ef39546993eb9188
-  data.tar.gz: dac26a7c7a3db09b59ae5b96cf1e76f7dd7aa975a3e08474aa636f1950dfa4e8
+  metadata.gz: 578798388cf4aa5c2628456764505f249ee7c5c50b6cf3c45cfd48bfccc5a62e
+  data.tar.gz: 172bf6a9caf7b38ae9274c3a1ada15b828655d79c4186a27c90c5de6882707f6
 SHA512:
-  metadata.gz: b39ee59f0649ce9f0558d335b8a0acbdfc371601bb1af9978cc97eb955fe4e3dfdb6ac7839583a146116479a61ff7d2afd3ce4eeccad5327273ea930aa9a73ff
-  data.tar.gz: ca26fc34d1327d39e97cbbaa3c782720971ae56273f9393656abbcc601584117b11d35c8db2d60e548ab6292e309cb27a056c3ad76fc644579f6ac5d4c2f46f0
+  metadata.gz: 8056b113c52d327fc314166e0cc4fda3765afe0dd41614e644e423a5504ff164297c01ab03f90c605f43fce84e902b226de7847c9b7f41d40ad8b462b7f86d43
+  data.tar.gz: 4ac3f7fb124ab42e21df67901b7435d4040cea3ad20e58b60515aa3b198755bb763c3415e48760aab7ebe12c895ff827e47487209de26ad06bccd46594742907

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.10
 
+## Release v1.10.4 - 2020/05/12
+
+### Enhancement
+
+* out_http: Support single json array payload
+  https://github.com/fluent/fluentd/pull/2973
+* Refactoring
+  https://github.com/fluent/fluentd/pull/2988
+
+### Bug fix
+
+* supervisor: Call `File.umask(0)` for standalone worker
+  https://github.com/fluent/fluentd/pull/2987
+* out_forward: Fix ZeroDivisionError issue with `weight 0`
+  https://github.com/fluent/fluentd/pull/2989
+
 ## Release v1.10.3 - 2020/05/01
 
 ### Enhancement

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
+* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/lib/fluent/plugin/in_dummy.rb

@@ -105,10 +105,10 @@ module Fluent::Plugin
       begin
         if @size > 1
           num.times do
-            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
+            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
           end
         else
-          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
+          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
         end
       rescue => _
         # ignore all errors not to stop emits by emit errors

data/lib/fluent/plugin/in_forward.rb

@@ -327,7 +327,7 @@ module Fluent::Plugin
                  record = e[1]
                  next if record.nil?
                  time = e[0]
-                 time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
+                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
                  es.add(time, record)
                }
                es
@@ -347,7 +347,7 @@ module Fluent::Plugin
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Fluent::Engine.now if time.to_i == 0
+        time = Fluent::EventTime.now if time.to_i == 0
         if @enable_field_injection
           record[@source_address_key] = conn.remote_addr if @source_address_key
           record[@source_hostname_key] = conn.remote_host if @source_hostname_key

data/lib/fluent/plugin/in_http.rb

@@ -176,9 +176,9 @@ module Fluent::Plugin
         end
         time = if param_time = params['time']
                  param_time = param_time.to_f
-                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
+                 param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                else
-                 record_time.nil? ? Fluent::Engine.now : record_time
+                 record_time.nil? ? Fluent::EventTime.now : record_time
                end
       rescue
         return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -223,7 +223,7 @@ module Fluent::Plugin
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }

data/lib/fluent/plugin/in_unix.rb

@@ -95,7 +95,7 @@ module Fluent
           record = e[1]
           next if record.nil?
           time = e[0]
-          time = (now ||= Engine.now) if time.to_i == 0
+          time = (now ||= EventTime.now) if time.to_i == 0
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -106,7 +106,7 @@ module Fluent
         return if record.nil?
 
         time = msg[1]
-        time = Engine.now if time.to_i == 0
+        time = EventTime.now if time.to_i == 0
         router.emit(tag, time, record)
       end
     end

data/lib/fluent/plugin/out_file.rb

@@ -155,7 +155,7 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
         test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"

data/lib/fluent/plugin/out_forward.rb

@@ -709,7 +709,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       end
 
       def rebuild_weight_array(nodes)
-        standby_nodes, regular_nodes = nodes.partition {|n|
+        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
           n.standby?
         }
 

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
     config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
     desc 'Content-Type for HTTP request'
     config_param :content_type, :string, default: nil
+    desc 'JSON array data format for HTTP request body'
+    config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
 
@@ -100,6 +102,13 @@ module Fluent::Plugin
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
       @content_type = setup_content_type unless @content_type
+
+      if @json_array
+        if @formatter_configs.first[:@type] != "json"
+          raise Fluent::ConfigError, "json_array option could be used with json formatter only"
+        end
+        define_singleton_method(:format, method(:format_json_array))
+      end
     end
 
     def multi_workers_ready?
@@ -114,6 +123,10 @@ module Fluent::Plugin
       @formatter.format(tag, time, record)
     end
 
+    def format_json_array(tag, time, record)
+      @formatter.format(tag, time, record) << ","
+    end
+
     def write(chunk)
       uri = parse_endpoint(chunk)
       req = create_request(chunk, uri)
@@ -128,7 +141,7 @@ module Fluent::Plugin
     def setup_content_type
       case @formatter_configs.first[:@type]
       when 'json'
-        'application/x-ndjson'
+        @json_array ? 'application/json' : 'application/x-ndjson'
       when 'csv'
         'text/csv'
       when 'tsv', 'ltsv'
@@ -202,7 +215,7 @@ module Fluent::Plugin
         req.basic_auth(@auth.username, @auth.password)
       end
       set_headers(req)
-      req.body = chunk.read
+      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
       req
     end
 

data/lib/fluent/supervisor.rb

@@ -593,7 +593,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.3'
+  VERSION = '1.10.4'
 
 end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end

data/test/plugin/test_in_unix.rb

@@ -11,14 +11,13 @@ module StreamInputTest
     d = create_driver
 
     time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
       d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
+        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
       }
     end
   end

data/test/plugin/test_out_http.rb

@@ -80,6 +80,8 @@ class HTTPOutputTest < Test::Unit::TestCase
         req.body.each_line { |l|
           data << JSON.parse(l)
         }
+      when 'application/json'
+        data = JSON.parse(req.body)
       when 'text/plain'
         # Use single_value in this test
         req.body.each_line { |line|
@@ -181,6 +183,19 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
   end
 
+  # Check if an exception is raised on not JSON format use
+  data('not_json' => 'msgpack')
+  def test_configure_with_json_array_err(format_type)
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        json_array true
+        <format>
+          @type #{format_type}
+        </format>
+      ])
+    end
+  end
+
   data('json' => ['json', 'application/x-ndjson'],
        'ltsv' => ['ltsv', 'text/tab-separated-values'],
        'msgpack' => ['msgpack', 'application/x-msgpack'],
@@ -195,6 +210,14 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_equal content_type, d.instance.content_type
   end
 
+  # Check that json_array setting sets content_type = application/json
+  data('json' => 'application/json')
+  def test_configure_content_type_json_array(content_type)
+    d = create_driver(config + "json_array true")
+
+    assert_equal content_type, d.instance.content_type
+  end
+
   data('PUT' => 'put', 'POST' => 'post')
   def test_write_with_method(method)
     d = create_driver(config + "http_method #{method}")
@@ -211,6 +234,21 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_empty result.headers
   end
 
+  # Check that JSON at HTTP request body is valid
+  def test_write_with_json_array_setting
+    d = create_driver(config + "json_array true")
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal 'application/json', result.content_type
+    assert_equal test_events, result.data
+    assert_not_empty result.headers
+  end
+
   def test_write_with_single_value_format
     d = create_driver(config + %[
       <format>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.10.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-01 00:00:00.000000000 Z
+date: 2020-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack