fluentd 1.14.4 → 1.14.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: deabdaa85e1adbc3d54e342e2faefaa2c952b492de3167e68fc4be021c605acb
-  data.tar.gz: f1e08a07176309930295b14c45bebaca86e7a7fd8de1191440d987170c752b96
+  metadata.gz: 98b1a3f0960184a4611f6c5139a0e4cb24cec72ec28581ad29d0012d32e1b9aa
+  data.tar.gz: 69cd5c67d4171fb9afb6a5c9d2c656b87921cc6e7a96aeed3017a62011010d63
 SHA512:
-  metadata.gz: 2f8d7f216549cfd57266bec6a06a42e2def597073aa103bde30bb2bad816f3a25b6b5b942fb7320d69f15164b887ab951e15039f62e1808162e554d11f1c7db6
-  data.tar.gz: f6a31183c931411e1cd4fae6d3dd8691f06ddfdfed4597bba001415052de1bf0081f1da6c75cf52ff4fd7479bf27894306d6336eb0f320c7adf1451dbcdf579a
+  metadata.gz: 32de1e2b6f891d041bac810aba2b1b9614b6cc72d2ffa22bb58cc4c9b61db3c54bac2d1bbfed56bfeae9147bacd840c37f1f43b8e841f15fcb7b07572301db83
+  data.tar.gz: 847ad4f2ff4615ee3fed3fe287d58a8734da7476a8ec8f444dcbf7ccec21b16a90e6c34351fc14737fa3ae0a9380e9cb53cbceda2873082cbaaa3e419b56c60a

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+# v1.14.5
+
+## Release v1.14.5 - 2022/02/09
+
+### Enhancement
+
+* Add support for "application/x-ndjson" to `in_http`
+  https://github.com/fluent/fluentd/pull/3616
+* Add support for ucrt binary for Windows
+  https://github.com/fluent/fluentd/pull/3613
+
+### Bug fixes
+
+* Don't retry when `retry_max_times == 0`
+  https://github.com/fluent/fluentd/pull/3608
+* Fix hang-up issue during TLS handshake in `out_forward`
+  https://github.com/fluent/fluentd/pull/3601
+* Bump up required ServerEngine to v2.2.5
+  https://github.com/fluent/fluentd/pull/3599
+* Fix "invalid byte sequence is replaced" warning on Kubernetes
+  https://github.com/fluent/fluentd/pull/3596
+* Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
+  https://github.com/fluent/fluentd/pull/3592
+
 # v1.14.4
 
 ## Release v1.14.4 - 2022/01/06

data/README.md

@@ -1,17 +1,12 @@
 Fluentd: Open-Source Log Collector
 ===================================
-GitHub Actions:
 
 [![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
 [![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
 [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
+[![Testing on Arm64](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
 [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
-
-Drone CI for Arm64:
-
-[![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
 
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
@@ -19,16 +14,6 @@ Drone CI for Arm64:
 <img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
 </p>
 
-An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
-
-## Example Use Cases
-
-Use Case | Description | Diagram
--------- | ------------|:---------:
-Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
-Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
-Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
-
 ## Quick Start
 
     $ gem install fluentd
@@ -66,12 +51,6 @@ You can run specified test via `TEST` environment variable:
     $ bundle exec rake test TEST=test/test_specified_path.rb
     $ bundle exec rake test TEST=test/test_*.rb
 
-## Fluentd UI: Admin GUI
-
-[Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
-
-<p align="center"><img width="500" src="https://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
-
 ## More Information
 
 - Website: https://www.fluentd.org/

data/Rakefile

@@ -14,7 +14,7 @@ namespace :build do
   desc 'Build gems for all platforms'
   task :all do
     Bundler.with_clean_env do
-      %w[ruby x86-mingw32 x64-mingw32].each do |name|
+      %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute
       end

data/fluentd.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -51,5 +51,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
+  gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
 end

data/lib/fluent/config/error.rb

@@ -29,4 +29,16 @@ module Fluent
 
   class SetDefault < Exception
   end
+
+  class NotFoundPluginError < ConfigError
+    attr_reader :type, :kind
+
+    def initialize(msg, type: nil, kind: nil)
+      @msg = msg
+      @type = type
+      @kind = kind
+
+      super(msg)
+    end
+  end
 end

data/lib/fluent/plugin/base.rb

@@ -72,8 +72,8 @@ module Fluent
 
       def string_safe_encoding(str)
         unless str.valid_encoding?
-          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
           str = str.scrub('?')
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
         end
         yield str
       end

data/lib/fluent/plugin/in_http.rb

@@ -314,8 +314,16 @@ module Fluent::Plugin
         @parser_json.parse(js) do |_time, record|
           return nil, record
         end
+      elsif ndjson = params['ndjson']
+        events = []
+        ndjson.split(/\r?\n/).each do |js|
+          @parser_json.parse(js) do |_time, record|
+            events.push(record)
+          end
+        end
+        return nil, events
       else
-        raise "'json' or 'msgpack' parameter is required"
+        raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
     end
 
@@ -567,6 +575,8 @@ module Fluent::Plugin
           params['json'] = @body
         elsif @content_type =~ /^application\/msgpack/
           params['msgpack'] = @body
+        elsif @content_type =~ /^application\/x-ndjson/
+          params['ndjson'] = @body
         end
         path_info = uri.path
 

data/lib/fluent/plugin/output.rb

@@ -1275,11 +1275,15 @@ module Fluent
 
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
-            if error
-              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
-              log.warn_backtrace error.backtrace
+            if @retry.limit?
+              # @retry_max_times == 0, fail imediately by the following block
+            else
+              if error
+                log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
+                log.warn_backtrace error.backtrace
+              end
+              return
             end
-            return
           end
 
           # @retry exists

data/lib/fluent/plugin_helper/socket.rb

@@ -96,6 +96,7 @@ module Fluent
           enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
           cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
           cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
+          connect_timeout: nil,
           **kwargs, &block)
 
         host_is_ipaddress = IPAddr.new(host) rescue false
@@ -158,13 +159,23 @@ module Fluent
         end
         Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
 
-        tcpsock = socket_create_tcp(host, port, **kwargs)
+        tcpsock = socket_create_tcp(host, port, connect_timeout: connect_timeout, **kwargs)
         sock = WrappedSocket::TLS.new(tcpsock, context)
         sock.sync_close = true
         sock.hostname = fqdn if verify_fqdn && fqdn && sock.respond_to?(:hostname=)
 
         log.trace "entering TLS handshake"
-        sock.connect
+        if connect_timeout
+          begin
+            Timeout.timeout(connect_timeout) { sock.connect }
+          rescue Timeout::Error
+            log.warn "timeout while connecting tls session", host: host
+            sock.close rescue nil
+            raise
+          end
+        else
+          sock.connect
+        end
 
         begin
           if verify_fqdn

data/lib/fluent/registry.rb

@@ -45,7 +45,8 @@ module Fluent
       if value = @map[type]
         return value
       end
-      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
+      raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
+                                    kind: @kind, type: type)
     end
 
     def reverse_lookup(value)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.4'
+  VERSION = '1.14.5'
 
 end

data/test/compat/test_parser.rb

@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
   Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
 
   def test_lookup_unknown_format
-    assert_raise Fluent::ConfigError do
+    assert_raise Fluent::NotFoundPluginError do
       Fluent::Plugin.new_parser('unknown')
     end
   end

data/test/plugin/test_filter_parser.rb

@@ -46,7 +46,7 @@ class ParserFilterTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       create_driver('')
     }
-    assert_raise(Fluent::ConfigError) {
+    assert_raise(Fluent::NotFoundPluginError) {
       create_driver %[
         key_name foo
         <parse>

data/test/plugin/test_filter_stdout.rb

@@ -63,7 +63,7 @@ class StdoutFilterTest < Test::Unit::TestCase
       end
 
       def test_invalid_output_type
-        assert_raise(Fluent::ConfigError) do
+        assert_raise(Fluent::NotFoundPluginError) do
           d = create_driver(CONFIG + config_element("", "", { "output_type" => "foo" }))
           d.run {}
         end
@@ -139,7 +139,7 @@ class StdoutFilterTest < Test::Unit::TestCase
       def test_invalid_output_type
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "foo" })
-        assert_raise(Fluent::ConfigError) do
+        assert_raise(Fluent::NotFoundPluginError) do
           d = create_driver(conf)
           d.run {}
         end

data/test/plugin/test_in_http.rb

@@ -540,6 +540,29 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_application_ndjson
+    d = create_driver
+    events = [
+        ["tag1", 1643935663, "{\"a\":1}\n{\"b\":2}"],
+        ["tag2", 1643935664, "{\"a\":3}\r\n{\"b\":4}"]
+    ]
+
+    expected = [
+        ["tag1", 1643935663, {"a"=>1}],
+        ["tag1", 1643935663, {"b"=>2}],
+        ["tag2", 1643935664, {"a"=>3}],
+        ["tag2", 1643935664, {"b"=>4}]
+    ]
+
+    d.run(expect_records: 1) do
+      events.each do |tag, time, record|
+        res = post("/#{tag}?time=#{time}", record, {"Content-Type"=>"application/x-ndjson"})
+        assert_equal("200", res.code)
+      end
+    end
+    assert_equal(expected, d.events)
+  end
+
   def test_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")

data/test/plugin/test_out_stdout.rb

@@ -32,7 +32,7 @@ class StdoutOutputTest < Test::Unit::TestCase
       assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
       assert_equal 'hash', d.instance.formatter.output_type
 
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         d = create_driver(CONFIG + "\noutput_type foo")
       end
     end
@@ -126,7 +126,7 @@ class StdoutOutputTest < Test::Unit::TestCase
       assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
       assert_equal 'hash', d.instance.formatter.output_type
 
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
       end
     end

data/test/plugin/test_output_as_buffered_retries.rb

@@ -701,6 +701,53 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       assert{ @i.buffer.stage.size == 1 }
       assert{ chunks.all?{|c| c.empty? } }
     end
+
+    test 'Do not retry when retry_max_times is 0' do
+      written_tags = []
+
+      chunk_key = 'tag'
+      hash = {
+        'flush_interval' => 1,
+        'flush_thread_burst_interval' => 0.1,
+        'retry_type' => :periodic,
+        'retry_wait' => 1,
+        'retry_randomize' => false,
+        'retry_max_times' => 0,
+        'queued_chunks_limit_size' => 100
+      }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
+      @i.register(:prefer_buffered_processing){ true }
+      @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
+      @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze( now )
+
+      @i.emit_events("test.tag.1", dummy_event_stream())
+
+      now = Time.parse('2016-04-13 18:33:31 -0700')
+      Timecop.freeze( now )
+
+      @i.emit_events("test.tag.2", dummy_event_stream())
+
+      assert_equal(0, @i.write_count)
+      assert_equal(0, @i.num_errors)
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }
+
+      assert(@i.write_count == 1)
+      assert(@i.num_errors == 1)
+      assert(@i.log.out.logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
+      assert(@i.buffer.queue.size == 0)
+      assert(@i.buffer.stage.size == 1)
+      assert(@i.buffer.queue.all?{|c| c.empty? })
+    end
   end
 
   sub_test_case 'buffered output configured as retry_forever' do

data/test/test_formatter.rb

@@ -349,7 +349,7 @@ module FormatterTest
     include FormatterTest
 
     def test_unknown_format
-      assert_raise ConfigError do
+      assert_raise NotFoundPluginError do
         Fluent::Plugin.new_formatter('unknown')
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.4
+  version: 1.14.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-06 00:00:00.000000000 Z
+date: 2022-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,7 +84,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.5
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -94,7 +94,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.5
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -338,6 +338,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.23'
 - !ruby/object:Gem::Dependency
   name: async-http
   requirement: !ruby/object:Gem::Requirement
@@ -945,7 +959,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector