fluentd 1.10.3-x86-mingw32 → 1.10.4-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e50b4e23679ea753e495fb94888322669b6b0e58dd903d86f72ef65115300c3
-  data.tar.gz: 03de1387897feaf543fc41600b9b3a59beb59caff9cc5576b5d714a295198523
+  metadata.gz: ec7203169c5cba3e274753b28122e96f72bb34451b2dc66b124c7cdfb7f5523b
+  data.tar.gz: f9167fc13a96f5651cbb8e662812e7a4baf020f66d258895c9aa51840eccd143
 SHA512:
-  metadata.gz: 131672fdf195a8c556a7476ce4fd6622adef9179595404138a206b7a0ce0dd9907d23c3926b56a92deac9522476f2de09d7ab38db5de8284ec4b6ae33a639092
-  data.tar.gz: 462342bad8e0a4800efe3fd25888415ac3e612924c902b2f622572972356403dac4cee18daf7b859da5b6ec3d5abaae1f3129d60899b3a16cf8b40381dc738d7
+  metadata.gz: 8452c882cdb9caad2bcc3b24866ea63ae6d093d2fc54231b477d3e14115d38cd12b7feeeb7a4f4f621c2bec3034926833e9e9e025ce25a7cf362679524839141
+  data.tar.gz: 8c969acdbd28dd6d131d8b2921fbee7e5a940073efe8a46bfb84d874a74ec9520db1ee3afc79c31b47fcbdb2f18c5f32533ccca2445eee60bc6b5bc300d2ec86

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.10
 
+## Release v1.10.4 - 2020/05/12
+
+### Enhancement
+
+* out_http: Support single json array payload
+  https://github.com/fluent/fluentd/pull/2973
+* Refactoring
+  https://github.com/fluent/fluentd/pull/2988
+
+### Bug fix
+
+* supervisor: Call `File.umask(0)` for standalone worker
+  https://github.com/fluent/fluentd/pull/2987
+* out_forward: Fix ZeroDivisionError issue with `weight 0`
+  https://github.com/fluent/fluentd/pull/2989
+
 ## Release v1.10.3 - 2020/05/01
 
 ### Enhancement

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
+* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/lib/fluent/plugin/in_dummy.rb

@@ -105,10 +105,10 @@ module Fluent::Plugin
       begin
         if @size > 1
           num.times do
-            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
+            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
           end
         else
-          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
+          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
         end
       rescue => _
         # ignore all errors not to stop emits by emit errors

data/lib/fluent/plugin/in_forward.rb

@@ -327,7 +327,7 @@ module Fluent::Plugin
                  record = e[1]
                  next if record.nil?
                  time = e[0]
-                 time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
+                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
                  es.add(time, record)
                }
                es
@@ -347,7 +347,7 @@ module Fluent::Plugin
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Fluent::Engine.now if time.to_i == 0
+        time = Fluent::EventTime.now if time.to_i == 0
         if @enable_field_injection
           record[@source_address_key] = conn.remote_addr if @source_address_key
           record[@source_hostname_key] = conn.remote_host if @source_hostname_key

data/lib/fluent/plugin/in_http.rb

@@ -176,9 +176,9 @@ module Fluent::Plugin
         end
         time = if param_time = params['time']
                  param_time = param_time.to_f
-                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
+                 param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                else
-                 record_time.nil? ? Fluent::Engine.now : record_time
+                 record_time.nil? ? Fluent::EventTime.now : record_time
                end
       rescue
         return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -223,7 +223,7 @@ module Fluent::Plugin
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }

data/lib/fluent/plugin/in_unix.rb

@@ -95,7 +95,7 @@ module Fluent
           record = e[1]
           next if record.nil?
           time = e[0]
-          time = (now ||= Engine.now) if time.to_i == 0
+          time = (now ||= EventTime.now) if time.to_i == 0
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -106,7 +106,7 @@ module Fluent
         return if record.nil?
 
         time = msg[1]
-        time = Engine.now if time.to_i == 0
+        time = EventTime.now if time.to_i == 0
         router.emit(tag, time, record)
       end
     end

data/lib/fluent/plugin/out_file.rb

@@ -155,7 +155,7 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
         test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"

data/lib/fluent/plugin/out_forward.rb

@@ -709,7 +709,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       end
 
       def rebuild_weight_array(nodes)
-        standby_nodes, regular_nodes = nodes.partition {|n|
+        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
           n.standby?
         }
 

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
     config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
     desc 'Content-Type for HTTP request'
     config_param :content_type, :string, default: nil
+    desc 'JSON array data format for HTTP request body'
+    config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
 
@@ -100,6 +102,13 @@ module Fluent::Plugin
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
       @content_type = setup_content_type unless @content_type
+
+      if @json_array
+        if @formatter_configs.first[:@type] != "json"
+          raise Fluent::ConfigError, "json_array option could be used with json formatter only"
+        end
+        define_singleton_method(:format, method(:format_json_array))
+      end
     end
 
     def multi_workers_ready?
@@ -114,6 +123,10 @@ module Fluent::Plugin
       @formatter.format(tag, time, record)
     end
 
+    def format_json_array(tag, time, record)
+      @formatter.format(tag, time, record) << ","
+    end
+
     def write(chunk)
       uri = parse_endpoint(chunk)
       req = create_request(chunk, uri)
@@ -128,7 +141,7 @@ module Fluent::Plugin
     def setup_content_type
       case @formatter_configs.first[:@type]
       when 'json'
-        'application/x-ndjson'
+        @json_array ? 'application/json' : 'application/x-ndjson'
       when 'csv'
         'text/csv'
       when 'tsv', 'ltsv'
@@ -202,7 +215,7 @@ module Fluent::Plugin
         req.basic_auth(@auth.username, @auth.password)
       end
       set_headers(req)
-      req.body = chunk.read
+      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
       req
     end
 

data/lib/fluent/supervisor.rb

@@ -593,7 +593,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.3'
+  VERSION = '1.10.4'
 
 end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end

data/test/plugin/test_in_unix.rb

@@ -11,14 +11,13 @@ module StreamInputTest
     d = create_driver
 
     time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
       d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
+        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
       }
     end
   end

data/test/plugin/test_out_http.rb

@@ -80,6 +80,8 @@ class HTTPOutputTest < Test::Unit::TestCase
         req.body.each_line { |l|
           data << JSON.parse(l)
         }
+      when 'application/json'
+        data = JSON.parse(req.body)
       when 'text/plain'
         # Use single_value in this test
         req.body.each_line { |line|
@@ -181,6 +183,19 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
   end
 
+  # Check if an exception is raised on not JSON format use
+  data('not_json' => 'msgpack')
+  def test_configure_with_json_array_err(format_type)
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        json_array true
+        <format>
+          @type #{format_type}
+        </format>
+      ])
+    end
+  end
+
   data('json' => ['json', 'application/x-ndjson'],
        'ltsv' => ['ltsv', 'text/tab-separated-values'],
        'msgpack' => ['msgpack', 'application/x-msgpack'],
@@ -195,6 +210,14 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_equal content_type, d.instance.content_type
   end
 
+  # Check that json_array setting sets content_type = application/json
+  data('json' => 'application/json')
+  def test_configure_content_type_json_array(content_type)
+    d = create_driver(config + "json_array true")
+
+    assert_equal content_type, d.instance.content_type
+  end
+
   data('PUT' => 'put', 'POST' => 'post')
   def test_write_with_method(method)
     d = create_driver(config + "http_method #{method}")
@@ -211,6 +234,21 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_empty result.headers
   end
 
+  # Check that JSON at HTTP request body is valid
+  def test_write_with_json_array_setting
+    d = create_driver(config + "json_array true")
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal 'application/json', result.content_type
+    assert_equal test_events, result.data
+    assert_not_empty result.headers
+  end
+
   def test_write_with_single_value_format
     d = create_driver(config + %[
       <format>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.10.4
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-01 00:00:00.000000000 Z
+date: 2020-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack