fluentd 1.13.0 → 1.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20ece94d6aedf52ed5f7bdefedfda93056ef92bbc44a7b7a27c4b4e4660dc5e0
-  data.tar.gz: 76166c17b6018360db1d192c2a9aa8094c1fea780bc74f715264035f0485d601
+  metadata.gz: 2974ccbbcf3e5273d3561322c9786ca7ad4cdda66817f13e4ed73962f4762b7b
+  data.tar.gz: 5bd0cd30a89e325fd1cbb3868a3f13f4685c00ca557cd38dedf04c478c6dc249
 SHA512:
-  metadata.gz: 214cbbef726516095e961786462e33b355fd77ecdcd278f5ada3a6f8d02489620ddbf84b9a7a4ba1de27b925f5455dda3c2b90c024bfa573afda05cc453d8e3a
-  data.tar.gz: 75de8aad92959bcb746be2c225c627e3f6590e2ffba38a91d0d0c540c7752bb2ebacc0623132a8ee613775ce63530a8083070fd008bb54646ecc37e4e2f6ca31
+  metadata.gz: 4fba66b7426b7818c7e60c6120ce46de4396ce903474b7e3d4e5db0db3f52ed5b324c0d54d312267abcc110ce5010108cac5ceaddb503a2234a973c9f99f879b
+  data.tar.gz: d6d3771be3d7850e4bf265836cc2663c81f97f0c602e2e95457fc2d30ea8c8c7b67e9340ef5db276253c5d15f5538f2712614b9c8b9bfaee4b1e302761ca2052

data/.github/workflows/linux-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.0', '2.7', '2.6', '2.5']
+        ruby-version: ['3.0', '2.7', '2.6']
         os: [ubuntu-latest]
         experimental: [false]
         include:

data/.github/workflows/windows-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7', '2.6', '2.5']
+        ruby-version: ['2.7', '2.6']
         os:
           - windows-latest
         experimental: [false]

data/.gitlab-ci.yml

@@ -13,16 +13,6 @@ stages:
   - build
   - tests
 
-2-5-8:
-  image: "ruby:2.5.8"
-  stage: build
-  extends: .install-template
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-5-8"
-    paths:
-
 2-6-6:
   image: "ruby:2.6.6"
   stage: build
@@ -64,18 +54,6 @@ rubyhead:
     paths:
       - ./*
 
-2-5-8-test:
-  image: "ruby:2.5.8"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-5-8"
-    paths:
-      - ./*
-
 2-6-6-test:
   image: "ruby:2.6.6"
   stage: tests

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# v1.13.1
+
+## Release v1.13.1 - 2021/06/25
+
+### Bug fix
+
+* out_forward: Fixed a race condition on handshake
+  It's caused by using a same unpacker from multiple threads.
+  https://github.com/fluent/fluentd/pull/3405
+  https://github.com/fluent/fluentd/pull/3406
+* in_tail: Fixed to remove too much verbose debugging logs
+  It was unexpectedly introduced by #3185 log throttling feature.
+  https://github.com/fluent/fluentd/pull/3418
+* Fixed not to echo back the provides path as is on a 404 error
+  There was a potential cross-site scripting vector even though
+  it is quite difficult to exploit.
+  https://github.com/fluent/fluentd/pull/3427
+
+### Misc
+
+* Pretty print for Fluent::Config::Section has been supported
+  for debugging
+  https://github.com/fluent/fluentd/pull/3398
+* CI: Dropped to run CI for Ruby 2.5
+  https://github.com/fluent/fluentd/pull/3412
+
 # v1.13
 
 ## Release v1.13.0 - 2021/05/29

data/lib/fluent/config/section.rb

@@ -50,6 +50,11 @@ module Fluent
         "<Fluent::Config::Section #{@params.to_json}>"
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       def nil?
         false
       end

data/lib/fluent/plugin/in_tail.rb

@@ -993,7 +993,6 @@ module Fluent::Plugin
                     @fifo << data
                     @fifo.read_lines(@lines)
 
-                    @log.debug("reading file: #{@path}")
                     if limit_bytes_per_second_reached? || should_shutdown_now?
                       # Just get out from tailing loop.
                       read_more = false

data/lib/fluent/plugin/out_forward.rb

@@ -565,8 +565,6 @@ module Fluent::Plugin
           username: server.username || '',
         )
 
-        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
-
         @resolved_host = nil
         @resolved_time = 0
         @resolved_once = false
@@ -613,7 +611,7 @@ module Fluent::Plugin
               sleep @sender.read_interval
               next
             end
-            @unpacker.feed_each(buf) do |data|
+            Fluent::MessagePackFactory.msgpack_unpacker.feed_each(buf) do |data|
               if @handshake.invoke(sock, ri, data) == :established
                 @log.debug "connection established", host: @host, port: @port
               end

data/lib/fluent/plugin/service_discovery.rb

@@ -13,21 +13,6 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
 require 'fluent/plugin/base'
 

data/lib/fluent/plugin_helper/http_server/router.rb

@@ -22,7 +22,7 @@ module Fluent
       class Router
         class NotFoundApp
           def self.call(req)
-            [404, { 'Content-Type' => 'text/plain' }, "404 Not Found: #{req.path}\n"]
+            [404, { 'Content-Type' => 'text/plain' }, "404 Not Found\n"]
           end
         end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.13.0'
+  VERSION = '1.13.1'
 
 end

data/test/command/test_cat.rb

@@ -18,20 +18,23 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
+    @port = unused_port
   end
 
   def teardown
     FileUtils.rm_rf(TMP_DIR)
+    @port = nil
   end
 
   TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}")
   FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + "/../../bin/fluent-cat")
 
-  PORT = unused_port
-  CONFIG = %[
-    port #{PORT}
-    bind 127.0.0.1
-  ]
+  def config
+    %[
+      port #{@port}
+      bind 127.0.0.1
+    ]
+  end
 
   SECONDARY_CONFIG = %[
     directory #{TMP_DIR}
@@ -41,7 +44,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     def write(chunk); end
   end
 
-  def create_driver(conf=CONFIG)
+  def create_driver(conf=config)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
   end
 
@@ -66,7 +69,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     def test_cat_json
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} json") do |stdin|
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
           stdin.puts('{"key":"value"}')
           stdin.close
         end
@@ -83,7 +86,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       path = d.instance.write(@chunk)
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} --format msgpack secondary") do |stdin|
+        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
           stdin.write(File.read(path))
           stdin.close
         end

data/test/config/test_section.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/config/section'
+require 'pp'
 
 module Fluent::Config
   class TestSection < ::Test::Unit::TestCase
@@ -176,6 +177,14 @@ module Fluent::Config
           s1 = Fluent::Config::Section.new(h1)
           assert_equal(expected, s1.respond_to?(method))
         end
+
+        test '#pretty_print' do
+          q = PP.new
+          h1 = {name: "s1", klass: "A"}
+          s1 = Fluent::Config::Section.new(h1)
+          s1.pretty_print(q)
+          assert_equal s1.inspect, q.output
+        end
       end
     end
   end

data/test/plugin/test_in_forward.rb

@@ -18,52 +18,55 @@ class ForwardInputTest < Test::Unit::TestCase
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
     @d = nil
+    @port = unused_port
   end
 
   def teardown
     @d.instance_shutdown if @d
+    @port = nil
   end
 
-  PORT = unused_port
-
   SHARED_KEY = 'foobar1'
   USER_NAME = 'tagomoris'
   USER_PASSWORD = 'fluentd'
 
-  CONFIG = %[
-    port #{PORT}
-    bind 127.0.0.1
-  ]
+  def base_config
+    %[
+      port #{@port}
+      bind 127.0.0.1
+    ]
+  end
   LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
   LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
   DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
-  CONFIG_AUTH = %[
-    port #{PORT}
-    bind 127.0.0.1
-    <security>
-      self_hostname localhost
-      shared_key foobar1
-      user_auth true
-      <user>
-        username #{USER_NAME}
-        password #{USER_PASSWORD}
-      </user>
-      <client>
-        network 127.0.0.0/8
-        shared_key #{SHARED_KEY}
-        users ["#{USER_NAME}"]
-      </client>
-    </security>
-  ]
-
-  def create_driver(conf=CONFIG)
+
+  def config_auth
+    base_config + %[
+      <security>
+        self_hostname localhost
+        shared_key foobar1
+        user_auth true
+        <user>
+          username #{USER_NAME}
+          password #{USER_PASSWORD}
+        </user>
+        <client>
+          network 127.0.0.0/8
+          shared_key #{SHARED_KEY}
+          users ["#{USER_NAME}"]
+        </client>
+      </security>
+    ]
+  end
+
+  def create_driver(conf=base_config)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
   end
 
   sub_test_case '#configure' do
     test 'simple' do
       @d = d = create_driver
-      assert_equal PORT, d.instance.port
+      assert_equal @port, d.instance.port
       assert_equal '127.0.0.1', d.instance.bind
       assert_equal 0, d.instance.linger_timeout
       assert_equal 0.5, d.instance.blocking_timeout
@@ -71,8 +74,8 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     test 'auth' do
-      @d = d = create_driver(CONFIG_AUTH)
-      assert_equal PORT, d.instance.port
+      @d = d = create_driver(config_auth)
+      assert_equal @port, d.instance.port
       assert_equal '127.0.0.1', d.instance.bind
       assert_equal 0, d.instance.linger_timeout
       assert !d.instance.backlog
@@ -86,17 +89,17 @@ class ForwardInputTest < Test::Unit::TestCase
          add_tag_prefix: "add_tag_prefix")
     test 'tag parameters' do |data|
       assert_raise(Fluent::ConfigError.new("'#{data}' parameter must not be empty")) {
-        create_driver(CONFIG + "#{data} ''")
+        create_driver(base_config + "#{data} ''")
       }
     end
 
     test 'send_keepalive_packet is disabled by default' do
-      @d = d = create_driver(CONFIG_AUTH)
+      @d = d = create_driver(config_auth)
       assert_false d.instance.send_keepalive_packet
     end
 
     test 'send_keepalive_packet can be enabled' do
-      @d = d = create_driver(CONFIG_AUTH + %[
+      @d = d = create_driver(config_auth + %[
         send_keepalive_packet true
       ])
       assert_true d.instance.send_keepalive_packet
@@ -104,7 +107,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     test 'both send_keepalive_packet and deny_keepalive cannot be enabled' do
       assert_raise(Fluent::ConfigError.new("both 'send_keepalive_packet' and 'deny_keepalive' cannot be set to true")) do
-        create_driver(CONFIG_AUTH + %[
+        create_driver(config_auth + %[
           send_keepalive_packet true
           deny_keepalive true
         ])
@@ -173,7 +176,7 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     test 'skip_invalid_event' do
-      @d = d = create_driver(CONFIG + "skip_invalid_event true")
+      @d = d = create_driver(base_config + "skip_invalid_event true")
 
       time = event_time("2011-01-02 13:14:15 UTC")
 
@@ -243,20 +246,18 @@ class ForwardInputTest < Test::Unit::TestCase
 
   sub_test_case 'forward' do
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'plain' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -285,7 +286,7 @@ class ForwardInputTest < Test::Unit::TestCase
            result: "new_prefix.tag1"
          })
     test 'tag parameters' do |data|
-      @d = create_driver(CONFIG + data[:param])
+      @d = create_driver(base_config + data[:param])
       time = event_time("2011-01-02 13:14:15 UTC")
       options = {auth: false}
 
@@ -305,20 +306,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'time_as_integer' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time_i = event_time("2011-01-02 13:14:15 UTC")
@@ -340,20 +339,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'skip_invalid_event' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config + "skip_invalid_event true")
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -380,20 +377,18 @@ class ForwardInputTest < Test::Unit::TestCase
 
   sub_test_case 'packed forward' do
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'plain' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -422,7 +417,7 @@ class ForwardInputTest < Test::Unit::TestCase
            result: "new_prefix.tag1"
          })
     test 'tag parameters' do |data|
-      @d = create_driver(CONFIG + data[:param])
+      @d = create_driver(base_config + data[:param])
       time = event_time("2011-01-02 13:14:15 UTC")
       options = {auth: false}
 
@@ -442,20 +437,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'time_as_integer' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time_i = event_time("2011-01-02 13:14:15 UTC").to_i
@@ -476,20 +469,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'skip_invalid_event' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config + "skip_invalid_event true")
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -577,7 +568,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
   sub_test_case 'warning' do
     test 'send_large_chunk_warning' do
-      @d = d = create_driver(CONFIG + %[
+      @d = d = create_driver(base_config + %[
         chunk_size_warn_limit 16M
         chunk_size_limit 32M
       ])
@@ -613,7 +604,7 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     test 'send_large_chunk_only_warning' do
-      @d = d = create_driver(CONFIG + %[
+      @d = d = create_driver(base_config + %[
         chunk_size_warn_limit 16M
       ])
       time = event_time("2014-04-25 13:14:15 UTC")
@@ -639,7 +630,7 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     test 'send_large_chunk_limit' do
-      @d = d = create_driver(CONFIG + %[
+      @d = d = create_driver(base_config + %[
         chunk_size_warn_limit 16M
         chunk_size_limit 32M
       ])
@@ -697,20 +688,18 @@ class ForwardInputTest < Test::Unit::TestCase
 
   sub_test_case 'respond to required ack' do
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'message' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -736,20 +725,18 @@ class ForwardInputTest < Test::Unit::TestCase
 
     # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'forward' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -776,20 +763,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'packed_forward' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -817,22 +802,20 @@ class ForwardInputTest < Test::Unit::TestCase
 
     data(
       tcp: {
-        config: CONFIG,
         options: {
           auth: false
         }
       },
       ### Auth is not supported with json
       # auth: {
-      #   config: CONFIG_AUTH,
       #   options: {
       #     auth: true
       #   }
       # },
     )
     test 'message_json' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time_i = event_time("2011-01-02 13:14:15 UTC")
@@ -859,20 +842,18 @@ class ForwardInputTest < Test::Unit::TestCase
 
   sub_test_case 'not respond without required ack' do
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'message' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -893,20 +874,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'forward' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -929,20 +908,18 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     data(tcp: {
-           config: CONFIG,
            options: {
              auth: false
            }
          },
          auth: {
-           config: CONFIG_AUTH,
            options: {
              auth: true
            }
          })
     test 'packed_forward' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time = event_time("2011-01-02 13:14:15 UTC")
@@ -966,22 +943,21 @@ class ForwardInputTest < Test::Unit::TestCase
 
     data(
       tcp: {
-        config: CONFIG,
         options: {
           auth: false
         }
       },
       ### Auth is not supported with json
       # auth: {
-      #   config: CONFIG_AUTH,
+      #   config: config_auth,
       #   options: {
       #     auth: true
       #   }
       # },
     )
     test 'message_json' do |data|
-      config = data[:config]
       options = data[:options]
+      config = options[:auth] ? config_auth : base_config
       @d = d = create_driver(config)
 
       time_i = event_time("2011-01-02 13:14:15 UTC").to_i
@@ -1096,7 +1072,7 @@ class ForwardInputTest < Test::Unit::TestCase
   end
 
   def connect
-    TCPSocket.new('127.0.0.1', PORT)
+    TCPSocket.new('127.0.0.1', @port)
   end
 
   # Data ordering is not assured:
@@ -1165,7 +1141,7 @@ class ForwardInputTest < Test::Unit::TestCase
   end
 
   def execute_test_with_source_hostname_key(*keys, &block)
-    conf = CONFIG.dup
+    conf = base_config.dup
     if keys.include?(:hostname)
       conf << <<EOL
 source_hostname_key source_hostname