fluentd 1.11.4 → 1.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46c5740a224a7e5d8ecacf44bb857570817d1d45a71e62f20079a2017bab8494
-  data.tar.gz: f68bb5bfbdc31f86f70a6f1028ecfe824e4c90cd1de86392f21507d0ca15e636
+  metadata.gz: 04be4b818522c222348ba309bccd34f03d0aa79bfc2fbd21f2ce738677d7a5e5
+  data.tar.gz: '078ce4c3003b74be7356367ff81d96d910a5df322ebe2319d96120cb1e2898b3'
 SHA512:
-  metadata.gz: d62f1f3eed61c490d67c1eabd9ab18e3c8c15781524414b7b42f7b4594c25c015dd2daef69b2ed6505ee278d01bc7d2c6614036108ba616fbce2e53fd8ecb3a7
-  data.tar.gz: cb31f1cd2355020ad662afe508136eda92efd0846802a8097a97db24e792e5272817ba83338256480f879afd943226b1ca76030475e5081b87b29dd13e9acf34
+  metadata.gz: 77298246f6064ddf342765c37efb269236025566b33828cac9c88d69b6dabbbd2881c28ddae699d4620817a44a77310e15c4ac191a0ad51b6aeab51eb7442961
+  data.tar.gz: 318668fa9fde451292fd3b4284fc513124e6aa29218388a8a7cdfaaeb3943b3d3c4ffa45041cb0b2487da6573a9818f225c81b8769b7c025342bc5a6b5b05b53

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,6 @@
 ---
 name:  Bug Report
-about: Create a report to help us improve. If you have questions about Fluentd and plugins, please direct these to https://groups.google.com/forum/#!forum/fluentd
+about: Create a report with a procedure for reproducing the bug
 
 ---
 

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://groups.google.com/forum/#!forum/fluentd
+    about: I have questions about Fluentd and plugins. Please ask and answer questions here

data/.github/workflows/build.yaml

@@ -0,0 +1,29 @@
+name: build
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: [2.7, 2.6, 2.5]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test TESTOPTS=-v

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"
+        exempt-pr-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"

data/.travis.yml

@@ -9,44 +9,63 @@ matrix:
   include:
     - rvm: 2.4.9
       os: linux
+      env: USE_CAPNG=false
     - rvm: 2.4.9
       os: linux-ppc64le
+      env: USE_CAPNG=false
     - rvm: 2.5.7
       os: linux
+      env: USE_CAPNG=false
     - rvm: 2.5.7
       os: linux
       arch: s390x
       dist: xenial
+      env: USE_CAPNG=false
     - rvm: 2.6.5
       os: linux
+      env: USE_CAPNG=false
+    - rvm: 2.6.6
+      os: linux
+      env: USE_CAPNG=true
     - rvm: 2.7.0
       os: linux
+      env: USE_CAPNG=false
     - rvm: ruby-head
       os: linux
+      env: USE_CAPNG=false
     - rvm: ruby-head
       os: linux-ppc64le
+      env: USE_CAPNG=false
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3 # OSX 10.12
+      env: USE_CAPNG=false
     - rvm: ruby-head
       os: osx
       osx_image: xcode8.3 # OSX 10.12
+      env: USE_CAPNG=false
   allow_failures:
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
+      env: USE_CAPNG=false
     - rvm: 2.5.7
       os: linux
       arch: s390x
       dist: xenial
+      env: USE_CAPNG=false
     - rvm: ruby-head
+      env: USE_CAPNG=false
 
 branches:
   only:
     - master
 
-before_install:
-  - gem update --system=3.1.2
+before_install: |
+  gem update --system=3.1.2
+  if [[ x"${USE_CAPNG}" == "xtrue" ]]; then
+    echo 'gem "capng_c"' >> Gemfile.local
+  fi
 
 sudo: false
 dist: trusty # for TLSv1.2 support
@@ -55,3 +74,4 @@ addons:
   apt:
     packages:
       - libgmp3-dev
+      - libcap-ng-dev

data/CHANGELOG.md

@@ -1,5 +1,82 @@
+# v1.12
+
+## Release v1.12.1 - 2021/02/18
+
+### Enhancement
+
+* out_http: Add `headers_from_placeholders` parameter
+  https://github.com/fluent/fluentd/pull/3241
+* fluent-plugin-config-format: Add `--table` option to use markdown table
+  https://github.com/fluent/fluentd/pull/3240
+* Add `--disable-shared-socket`/`disable_shared_socket` to disable ServerEngine's shared socket setup
+  https://github.com/fluent/fluentd/pull/3250
+
+### Bug fix
+
+* ca_generate: Fix creating TLS certification files which include broken extensions
+  https://github.com/fluent/fluentd/pull/3246
+* test: Drop TLS 1.1 tests
+  https://github.com/fluent/fluentd/pull/3256
+* Remove old gem constraints to support Ruby 3
+
+### Misc
+
+* Use GitHub Actions
+  https://github.com/fluent/fluentd/pull/3233
+  https://github.com/fluent/fluentd/pull/3255
+
+## Release v1.12.0 - 2021/01/05
+
+### New feature
+
+* in_tail: Add `follow_inode` to support log rotation with wild card
+  https://github.com/fluent/fluentd/pull/2992
+* in_tail: Handle linux capability
+  https://github.com/fluent/fluentd/pull/3155
+  https://github.com/fluent/fluentd/pull/3162
+* windows: Add win32 events alternative to unix signals
+  https://github.com/fluent/fluentd/pull/3131
+
+### Enhancement
+
+* buffer: Enable metadata comparison optimization on all platforms
+  https://github.com/fluent/fluentd/pull/3095
+* fluent-plugin-config-formatter: Handle `service_discovery` type
+  https://github.com/fluent/fluentd/pull/3178
+* in_http: Add `add_query_params` parameter to add query params to event record
+  https://github.com/fluent/fluentd/pull/3197
+* inject: Support `unixtime_micros` and `unixtime_nanos` in `time_type`
+  https://github.com/fluent/fluentd/pull/3220
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3167
+  https://github.com/fluent/fluentd/pull/3170
+  https://github.com/fluent/fluentd/pull/3180
+  https://github.com/fluent/fluentd/pull/3196
+  https://github.com/fluent/fluentd/pull/3213
+  https://github.com/fluent/fluentd/pull/3222
+
+### Bug fix
+
+* output: Prevent retry.step from being called too many times in a short time
+  https://github.com/fluent/fluentd/pull/3203
+
 # v1.11
 
+## Release v1.11.5 - 2020/11/06
+
+### Enhancement
+
+* formatter: Provide `newline` parameter to support `CRLF`
+  https://github.com/fluent/fluentd/pull/3152
+* out_http: adding support for intermediate certificates
+  https://github.com/fluent/fluentd/pull/3146
+* Update serverengine dependency to 2.2.2 or later
+
+### Bug fix
+
+* Fix a bug that windows service isn't stopped gracefuly
+  https://github.com/fluent/fluentd/pull/3156
+
 ## Release v1.11.4 - 2020/10/13
 
 ### Enhancement

data/README.md

@@ -13,7 +13,7 @@ Drone CI for Arm64:
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
-<img src="https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/1.0/images/fluentd-architecture.png" width="500px"/>
+<img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
 </p>
 
 An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
@@ -78,7 +78,7 @@ You can run specified test via `TEST` environment variable:
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
-- Copyright: 2011-2019 Fluentd Authors
+- Copyright: 2011-2021 Fluentd Authors
 - License: Apache License, Version 2.0
 
 ## Security

data/appveyor.yml

@@ -7,6 +7,9 @@ install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
   - ruby --version
   - gem --version
+  # stay 0.14.0 for Windows CI until https://github.com/socketry/protocol-http2/issues/6 will be fixed
+  - ps: Write-Output "gem 'protocol-http2', ['<= 0.14.0']" | Out-File -FilePath Gemfile.local -Encoding default
+  - type Gemfile.local
   - ridk.cmd exec bundle install
 build: off
 test_script:

data/bin/fluent-cap-ctl

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/cap_ctl'
+
+Fluent::CapCtl.new.call

data/bin/fluent-ctl

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/ctl'
+
+Fluent::Ctl.new.call

data/fluentd.gemspec

@@ -18,10 +18,11 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.4'
 
+  gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -48,6 +49,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
-  gem.add_development_dependency("ext_monitor", [">= 0.1.2", "< 0.2"])
-  gem.add_development_dependency("async-http")
+  gem.add_development_dependency("async-http", ">= 0.50.0")
 end

data/lib/fluent/capability.rb

@@ -0,0 +1,87 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "fluent/env"
+
+if Fluent.linux?
+  begin
+    require 'capng'
+  rescue LoadError
+  end
+end
+
+module Fluent
+  if defined?(CapNG)
+    class Capability
+      def initialize(target = nil, pid = nil)
+        @capng = CapNG.new(target, pid)
+      end
+
+      def usable?
+        true
+      end
+
+      def apply(select_set)
+        @capng.apply(select_set)
+      end
+
+      def clear(select_set)
+        @capng.clear(select_set)
+      end
+
+      def have_capability?(type, capability)
+        @capng.have_capability?(type, capability)
+      end
+
+      def update(action, type, capability_or_capability_array)
+        @capng.update(action, type, capability_or_capability_array)
+      end
+
+      def have_capabilities?(select_set)
+        @capng.have_capabilities?(select_set)
+      end
+    end
+  else
+    class Capability
+      def initialize(target = nil, pid = nil)
+      end
+
+      def usable?
+        false
+      end
+
+      def apply(select_set)
+        false
+      end
+
+      def clear(select_set)
+        false
+      end
+
+      def have_capability?(type, capability)
+        false
+      end
+
+      def update(action, type, capability_or_capability_array)
+        false
+      end
+
+      def have_capabilities?(select_set)
+        false
+      end
+    end
+  end
+end

data/lib/fluent/command/ca_generate.rb

@@ -75,6 +75,8 @@ HELP
 
       digest = OpenSSL::Digest::SHA256.new
 
+      factory = OpenSSL::X509::ExtensionFactory.new
+
       cert = OpenSSL::X509::Certificate.new
       cert.not_before = Time.at(0)
       cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
@@ -82,7 +84,7 @@ HELP
       cert.serial = 1
       cert.issuer = issuer
       cert.subject = subject
-      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+      cert.add_extension(factory.create_extension('basicConstraints', 'CA:TRUE'))
       cert.sign(key, digest)
 
       return cert, key
@@ -111,8 +113,9 @@ HELP
       cert.issuer = issuer
       cert.subject = subject
 
-      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-      cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+      factory = OpenSSL::X509::ExtensionFactory.new
+      server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))
+      server_cert.add_extension(factory.create_extension('nsCertType', 'server'))
 
       cert.sign ca_key, digest
 

data/lib/fluent/command/cap_ctl.rb

@@ -0,0 +1,174 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/capability'
+
+module Fluent
+  class CapCtl
+    def prepare_option_parser
+      @op = OptionParser.new
+
+      @op.on('--clear', "Clear Fluentd Ruby capability") {|s|
+        @opts[:clear_capabilities] = true
+      }
+
+      @op.on('--add [CAPABILITITY1,CAPABILITY2, ...]', "Add capabilities into Fluentd Ruby") {|s|
+        @opts[:add_capabilities] = s
+      }
+
+      @op.on('--drop [CAPABILITITY1,CAPABILITY2, ...]', "Drop capabilities from Fluentd Ruby") {|s|
+        @opts[:drop_capabilities] = s
+      }
+
+      @op.on('--get', "Get capabilities for Fluentd Ruby") {|s|
+        @opts[:get_capabilities] = true
+      }
+
+      @op.on('-f', '--file FILE', "Specify target file to add Linux capabilities") {|s|
+        @opts[:target_file] = s
+      }
+    end
+
+    def usage(msg)
+      puts @op.to_s
+      puts "error: #{msg}" if msg
+      exit 1
+    end
+
+    def initialize(argv = ARGV)
+      @opts = {}
+      @argv = argv
+
+      if Fluent.linux?
+        begin
+          require 'capng'
+
+          @capng = CapNG.new
+        rescue LoadError
+          puts "Error: capng_c is not loaded. Please install it first."
+          exit 1
+        end
+      else
+        puts "Error: This environment is not supported."
+        exit 2
+      end
+
+      prepare_option_parser
+    end
+
+    def call
+      parse_options!(@argv)
+
+      target_file = if !!@opts[:target_file]
+                       @opts[:target_file]
+                     else
+                       File.readlink("/proc/self/exe")
+                     end
+
+      if @opts[:clear_capabilities]
+        clear_capabilities(@opts, target_file)
+      elsif @opts[:add_capabilities]
+        add_capabilities(@opts, target_file)
+      elsif @opts[:drop_capabilities]
+        drop_capabilities(@opts, target_file)
+      end
+      if @opts[:get_capabilities]
+        get_capabilities(@opts, target_file)
+      end
+    end
+
+    def clear_capabilities(opts, target_file)
+      if !!opts[:clear_capabilities]
+        @capng.clear(:caps)
+        ret = @capng.apply_caps_file(target_file)
+        puts "Clear capabilities #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def add_capabilities(opts, target_file)
+      if add_caps = opts[:add_capabilities]
+        @capng.clear(:caps)
+        @capng.caps_file(target_file)
+        capabilities = add_caps.split(/\s*,\s*/)
+        check_capabilities(capabilities, get_valid_capabilities)
+        ret = @capng.update(:add,
+                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,
+                            capabilities)
+        puts "Updating #{add_caps} #{ret ? 'done' : 'fail'}."
+        ret = @capng.apply_caps_file(target_file)
+        puts "Adding #{add_caps} #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def drop_capabilities(opts, target_file)
+      if drop_caps = opts[:drop_capabilities]
+        @capng.clear(:caps)
+        @capng.caps_file(target_file)
+        capabilities = drop_caps.split(/\s*,\s*/)
+        check_capabilities(capabilities, get_valid_capabilities)
+        ret = @capng.update(:drop,
+                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,
+                            capabilities)
+        puts "Updating #{drop_caps} #{ret ? 'done' : 'fail'}."
+        @capng.apply_caps_file(target_file)
+        puts "Dropping #{drop_caps} #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def get_capabilities(opts, target_file)
+      if opts[:get_capabilities]
+        @capng.caps_file(target_file)
+        print = CapNG::Print.new
+        puts "Capabilities in '#{target_file}',"
+        puts "Effective:   #{print.caps_text(:buffer, :effective)}"
+        puts "Inheritable: #{print.caps_text(:buffer, :inheritable)}"
+        puts "Permitted:   #{print.caps_text(:buffer, :permitted)}"
+      end
+    end
+
+    def get_valid_capabilities
+      capabilities = []
+      cap = CapNG::Capability.new
+      cap.each do |_code, capability|
+        capabilities << capability
+      end
+      capabilities
+    end
+
+    def check_capabilities(capabilities, valid_capabilities)
+      capabilities.each do |capability|
+        unless valid_capabilities.include?(capability)
+          raise ArgumentError, "'#{capability}' is not valid capability. Valid Capabilities are:  #{valid_capabilities.join(", ")}"
+        end
+      end
+    end
+
+    def parse_options!(argv)
+      begin
+        rest = @op.parse(argv)
+
+        if rest.length != 0
+          usage nil
+        end
+      rescue
+        usage $!.to_s
+      end
+    end
+  end
+end