fluentd 1.8.1-x86-mingw32 → 1.9.0-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b3b39be7796dd7e8e65acd5f43d4e39d90e2be293dd4d6d358b99f9323d161c
-  data.tar.gz: 3d6bec6f6e93525601ef8ded63f8ac424cfe20f2da9a38a2c77c8cdae0cebac5
+  metadata.gz: c012a28dff140583d26c6cdf490dd3efdeedb5e502f3f6764796ff7415298385
+  data.tar.gz: e1bcdebb3998dfc297e60cddf9fa1c0db4079595169610b3088927a61c1d6d32
 SHA512:
-  metadata.gz: 45a176a60cff687302b6b5fdaeaf24f161ea9bad69142323e29855fae12ff89f3a853fd4ddc506582507357cae7ebd1fb8be9981409c36f1e514211182061416
-  data.tar.gz: 242b685fad3135b3ca6fc4fde37766db8cd688dafec82249f723fc40b8fa2925d3bfd3762402a34aa01255460108cfd785d076d7de0f5a4952fd180b497503bf
+  metadata.gz: dc4fb419d9a30830881e7dc5a563f7d19030008f5bac3452cb1549da6860aa58b28bec8b9bcb2e2c8b41a744ff1de1c713ff8235343e2154531ecbed09344bdd
+  data.tar.gz: 49553765cf43ab3094962306f4abfdf2c23b8da8695756702f2a1855b834130bf195a2ceaa39e48bb030680c480133129ff99125280840de3896208362765ef3

data/.gitlab-ci.yml

@@ -1,38 +1,22 @@
-include:
-  - local: '/.gitlab/cicd-template.yaml'
+.install-template:
+  before_script:
+    - gem update --system=2.7.8
+    - ruby -v
+
+.test-template:
+  before_script:
+    - gem install rake
+    - bundle install
+    - chmod 777 -R ./test
 
 stages:
   - build
   - tests
 
-2-1-10:
-  image: "ruby:2.1.10"
-  stage: build
-  allow_failure: true
-  extends: .install-template 
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-1-10"
-    paths:
-      - ./*
-
-2-2-10:
-  image: "ruby:2.2.10"
-  stage: build
-  allow_failure: true
-  extends: .install-template 
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-2-10"
-    paths:
-
 2-4-5:
   image: "ruby:2.4.5"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -42,8 +26,7 @@ stages:
 2-5-5:
   image: "ruby:2.5.5"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -53,8 +36,7 @@ stages:
 2-6-3:
   image: "ruby:2.6.3"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -64,8 +46,7 @@ stages:
 rubyhead:
   image: "ruby:latest"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -73,30 +54,6 @@ rubyhead:
     paths:
       - ./*
 
-2-1-10-test:
-  image: "ruby:2.1.10"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-1-10"
-    paths:
-      - ./*
-
-2-2-10-test:
-  image: "ruby:2.2.10"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-2-10"
-    paths:
-      - ./*
-
 2-4-5-test:
   image: "ruby:2.4.5"
   stage: tests
@@ -144,4 +101,3 @@ rubyhead-test:
     key: "$CI_COMMIT_REF_SLUG latest"
     paths:
       - ./*
-    

data/.travis.yml

@@ -7,12 +7,6 @@ cache: bundler
 # See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c
 matrix:
   include:
-    - rvm: 2.1.10
-      os: linux
-    - rvm: 2.2.10
-      os: linux
-    - rvm: 2.2.10
-      os: linux-ppc64le
     - rvm: 2.4.9
       os: linux
     - rvm: 2.4.9
@@ -25,13 +19,12 @@ matrix:
       dist: xenial
     - rvm: 2.6.5
       os: linux
+    - rvm: 2.7.0
+      os: linux
     - rvm: ruby-head
       os: linux
     - rvm: ruby-head
       os: linux-ppc64le
-    - rvm: 2.1.10
-      os: osx
-      osx_image: xcode8.3 # OSX 10.12
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3 # OSX 10.12
@@ -39,9 +32,6 @@ matrix:
       os: osx
       osx_image: xcode8.3 # OSX 10.12
   allow_failures:
-    - rvm: 2.1.10
-      os: osx
-      osx_image: xcode8.3
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
@@ -54,11 +44,6 @@ matrix:
 branches:
   only:
     - master
-    - v0.12
-    - v0.14
-
-before_install:
-  - gem update --system=2.7.8
 
 sudo: false
 dist: trusty # for TLSv1.2 support

data/CHANGELOG.md

@@ -1,3 +1,42 @@
+# v1.9
+
+## Release v1.9.0 - 2020/01/22
+
+### New feature
+
+* New light-weight config reload mechanizm
+  https://github.com/fluent/fluentd/pull/2716
+* Drop ruby 2.1/2.2/2.3 support
+  https://github.com/fluent/fluentd/pull/2750
+
+### Enhancement
+
+* output: Show better message for secondary warning
+  https://github.com/fluent/fluentd/pull/2751
+* Use `ext_monitor` gem if it is installed. For ruby 2.6 or earlier
+  https://github.com/fluent/fluentd/pull/2670
+* Support Ruby's Time class in msgpack serde
+  https://github.com/fluent/fluentd/pull/2775
+* Clean up code/test
+  https://github.com/fluent/fluentd/pull/2753
+  https://github.com/fluent/fluentd/pull/2763
+  https://github.com/fluent/fluentd/pull/2764
+  https://github.com/fluent/fluentd/pull/2780
+
+### Bug fix
+
+* buffer: Disable the optimization of Metadata instance comparison on Windows
+  https://github.com/fluent/fluentd/pull/2778
+* outut/buffer: Fix stage size computation
+  https://github.com/fluent/fluentd/pull/2734
+* server: Ignore Errno::EHOSTUNREACH in TLS accept to avoid fluentd restart
+  https://github.com/fluent/fluentd/pull/2773
+* server: Fix IPv6 dual stack mode issue for udp socket
+  https://github.com/fluent/fluentd/pull/2781
+* config: Support @include/include directive for spaces included path
+  https://github.com/fluent/fluentd/pull/2780
+
+
 # v1.8
 
 ## Release v1.8.1 - 2019/12/26

data/Gemfile

@@ -2,10 +2,7 @@ source 'https://rubygems.org/'
 
 gemspec
 
-# https://github.com/socketry/async-io/blob/v1.23.1/async-io.gemspec#L21
-if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.3.0')
-  gem 'async-http', '~> 0.42'
-end
+gem 'async-http', '~> 0.42'
 
 local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
 if File.exist?(local_gemfile)

data/README.md

@@ -6,10 +6,6 @@ Travis CI:
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
-GitLab CI:
-
-[![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
-
 Drone CI for Arm64:
 
 [![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
@@ -42,11 +38,11 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
 ### Branch
 
 - master: For v1 development.
-- v0.12: For v0.12. This is security maintenance mode. Only security fix is accepted.
+- v0.12: For v0.12. This is deprecated version. we already stopped supporting (See https://www.fluentd.org/blog/drop-schedule-announcement-in-2019).
 
 ### Prerequisites
 
-- Ruby 2.1 or later
+- Ruby 2.4 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/fluentd.gemspec

@@ -16,9 +16,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.1'
+  gem.required_ruby_version = '>= 2.4'
 
-  gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
+  gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
@@ -27,7 +27,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
-  gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -40,13 +39,14 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
   end
 
-  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("rake", ["~> 13.0"])
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
   gem.add_development_dependency("rr", ["~> 1.0"])
-  gem.add_development_dependency("timecop", ["~> 0.3"])
-  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("timecop", ["~> 0.9"])
+  gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
+  gem.add_development_dependency("ext_monitor", [">= 0.1.1", "< 0.2"])
 end

data/lib/fluent/command/cat.rb

@@ -101,13 +101,11 @@ rescue
   usage $!.to_s
 end
 
-
 require 'thread'
-require 'monitor'
 require 'socket'
 require 'yajl'
 require 'msgpack'
-
+require 'fluent/ext_monitor_require'
 
 class Writer
   include MonitorMixin

data/lib/fluent/command/plugin_generator.rb

@@ -281,7 +281,8 @@ HELP
       @text = ""
       @preamble_source = ""
       @preamble = nil
-      open(LICENSE_URL) do |io|
+      uri = URI.parse(LICENSE_URL)
+      uri.open do |io|
         @text = io.read
       end
       @preamble_source = @text[/^(\s*Copyright.+)/m, 1]

data/lib/fluent/config.rb

@@ -20,6 +20,25 @@ require 'fluent/configurable'
 
 module Fluent
   module Config
+    # @param config_path [String] config file path
+    # @param encoding [String] encoding of config file
+    # @param additional_config [String] config which is added to last of config body
+    # @param use_v1_config [Bool] config is formatted with v1 or not
+    # @return [Fluent::Config]
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+      config_fname = File.basename(config_path)
+      config_basedir = File.dirname(config_path)
+      config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
+        s = f.read
+        if additional_config
+          c = additional_config.gsub("\\n", "\n")
+          s += "\n#{c}"
+        end
+        s
+      end
+      Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
+    end
+
     def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
       parser = if fname =~ /\.rb$/ || syntax == :ruby
                  :ruby

data/lib/fluent/config/literal_parser.rb

@@ -19,7 +19,7 @@ require 'stringio'
 require 'json'
 require 'yajl'
 require 'socket'
-require 'irb/ruby-lex'  # RubyLex
+require 'ripper'
 
 require 'fluent/config/basic_parser'
 
@@ -155,15 +155,20 @@ module Fluent
       end
 
       def scan_embedded_code
-        rlex = RubyLex.new
-        src = '"#{'+@ss.rest+"\n=end\n}"
+        src = '"#{'+@ss.rest+"\n=begin\n=end\n}"
 
-        input = StringIO.new(src)
-        input.define_singleton_method(:encoding) { external_encoding }
-        rlex.set_input(input)
+        seek = -1
+        while (seek = src.index('}', seek + 1))
+          unless Ripper.sexp(src[0..seek] + '"').nil? # eager parsing until valid expression
+            break
+          end
+        end
+
+        unless seek
+          raise Fluent::ConfigParseError, @ss.rest
+        end
 
-        tk = rlex.token
-        code = src[3,tk.seek-3]
+        code = src[3, seek-3]
 
         if @ss.rest.length < code.length
           @ss.pos += @ss.rest.length

data/lib/fluent/config/v1_parser.rb

@@ -147,11 +147,13 @@ module Fluent
       end
 
       def eval_include(attrs, elems, uri)
-        u = URI.parse(uri)
-        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
+        # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
+        # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
+        u = URI.parse(uri.gsub(/ /, '+'))
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
           # When the Windows absolute path then u.scheme.length == 1
           # e.g. C:
-          path = u.path
+          path = URI.decode_www_form_component(u.path)
           if path[0] != ?/
             pattern = File.expand_path("#{@include_basepath}/#{path}")
           else

data/lib/fluent/engine.rb

@@ -23,6 +23,7 @@ require 'fluent/time'
 require 'fluent/system_config'
 require 'fluent/plugin'
 require 'fluent/fluent_log_event_router'
+require 'fluent/static_config_analysis'
 
 module Fluent
   class EngineClass
@@ -157,14 +158,47 @@ module Fluent
         raise
       end
 
-      unless @log_event_verbose
-        $log.enable_event(false)
-        @fluent_log_event_router.graceful_stop
+      stop_phase(@root_agent)
+    end
+
+    # @param conf [Fluent::Config]
+    # @param supervisor [Bool]
+    # @reutrn nil
+    def reload_config(conf, supervisor: false)
+      # configure first to reduce down time while restarting
+      new_agent = RootAgent.new(log: log, system_config: @system_config)
+      ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
+
+      ret.all_plugins.each do |plugin|
+        if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
+          raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+        end
       end
-      $log.info "shutting down fluentd worker", worker: worker_id
-      shutdown
 
-      @fluent_log_event_router.stop
+      # Assign @root_agent to new root_agent
+      # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
+      old_agent, @root_agent = @root_agent, new_agent
+      begin
+        @root_agent.configure(conf)
+      rescue
+        @root_agent = old_agent
+        raise
+      end
+
+      unless @suppress_config_dump
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+      end
+
+      # supervisor doesn't handle actual data. so the following code is unnecessary.
+      if supervisor
+        old_agent.shutdown      # to close thread created in #configure
+        return
+      end
+
+      stop_phase(old_agent)
+
+      $log.info 'restart fluentd worker', worker: worker_id
+      start_phase(new_agent)
     end
 
     def stop
@@ -189,12 +223,29 @@ module Fluent
     end
 
     private
-    def start
+
+    def stop_phase(root_agent)
+      unless @log_event_verbose
+        $log.enable_event(false)
+        @fluent_log_event_router.graceful_stop
+      end
+      $log.info 'shutting down fluentd worker', worker: worker_id
+      root_agent.shutdown
+
+      @fluent_log_event_router.stop
+    end
+
+    def start_phase(root_agent)
+      @fluent_log_event_router = FluentLogEventRouter.build(root_agent)
+      if @fluent_log_event_router.emittable?
+        $log.enable_event(true)
+      end
+
       @root_agent.start
     end
 
-    def shutdown
-      @root_agent.shutdown
+    def start
+      @root_agent.start
     end
   end