fluentd 1.9.0.rc1 → 1.9.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05d0842de47c79ef5a898ea8777454fe9c9863b4d5b6e5161f05c9d5e30bf1c7
-  data.tar.gz: 59fc0341af8411eb6e5830005f097c3f03851df9117d64ea838c91d36e9a7ff6
+  metadata.gz: d435419feb5d6622dfcfcaa8e464234a0ba8aa73abd501d61521568982c33736
+  data.tar.gz: 0a153fdfa008135008379514314c42575495004ba9f7fb49ce6d39c8fa647713
 SHA512:
-  metadata.gz: aee7aaba37e954564671aa0790f6af069d583670bd02819578342d35b94061a225e5714fc221b1631d099000d36eb3df04e014edad39a5c7ac3156733cbf172b
-  data.tar.gz: 1400809d32318726727359177ecf25e6393154dc0a9e5659516b6f9a73c6f87665c138f7c9f71359047818efa5dcf19f1c620faf998bead64edea5d6f17af3db
+  metadata.gz: fd0d19edc1817113632a530c931d155a46dce8af3bbc9c2b511c835dcb4ee69397a58e1af297ada745d4882008e58fbddba8beba9eb3f01c88120ffed716059b
+  data.tar.gz: b57a69591accb5eb39620afe1aeb0fbf39c171a431979349b110b15ad2892424bee5a28c4dbd7aad24ad1d19d3881759b86ca3d3595423e5c8b9004ec0d5d215

data/.gitlab-ci.yml

@@ -1,38 +1,22 @@
-include:
-  - local: '/.gitlab/cicd-template.yaml'
+.install-template:
+  before_script:
+    - gem update --system=2.7.8
+    - ruby -v
+
+.test-template:
+  before_script:
+    - gem install rake
+    - bundle install
+    - chmod 777 -R ./test
 
 stages:
   - build
   - tests
 
-2-1-10:
-  image: "ruby:2.1.10"
-  stage: build
-  allow_failure: true
-  extends: .install-template 
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-1-10"
-    paths:
-      - ./*
-
-2-2-10:
-  image: "ruby:2.2.10"
-  stage: build
-  allow_failure: true
-  extends: .install-template 
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-2-10"
-    paths:
-
 2-4-5:
   image: "ruby:2.4.5"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -42,8 +26,7 @@ stages:
 2-5-5:
   image: "ruby:2.5.5"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -53,8 +36,7 @@ stages:
 2-6-3:
   image: "ruby:2.6.3"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -64,8 +46,7 @@ stages:
 rubyhead:
   image: "ruby:latest"
   stage: build
-  allow_failure: true
-  extends: .install-template 
+  extends: .install-template
   script:
     - bundle install --jobs=3 --retry=3
   cache:
@@ -73,30 +54,6 @@ rubyhead:
     paths:
       - ./*
 
-2-1-10-test:
-  image: "ruby:2.1.10"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-1-10"
-    paths:
-      - ./*
-
-2-2-10-test:
-  image: "ruby:2.2.10"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-2-10"
-    paths:
-      - ./*
-
 2-4-5-test:
   image: "ruby:2.4.5"
   stage: tests
@@ -144,4 +101,3 @@ rubyhead-test:
     key: "$CI_COMMIT_REF_SLUG latest"
     paths:
       - ./*
-    

data/CHANGELOG.md

@@ -13,13 +13,21 @@
 
 * output: Show better message for secondary warning
   https://github.com/fluent/fluentd/pull/2751
-* Clean up code for ruby 2.7
+* Use `ext_monitor` gem if it is installed. For ruby 2.6 or earlier
+  https://github.com/fluent/fluentd/pull/2670
+* Support Ruby's Time class in msgpack serde
+  https://github.com/fluent/fluentd/pull/2775
+* Clean up code
   https://github.com/fluent/fluentd/pull/2753
+  https://github.com/fluent/fluentd/pull/2763
+  https://github.com/fluent/fluentd/pull/2764
 
 ### Bug fixes
 
 * outut/buffer: Fix stage size computation
   https://github.com/fluent/fluentd/pull/2734
+* server: Ignore Errno::EHOSTUNREACH in TLS accept to avoid fluentd restart
+  https://github.com/fluent/fluentd/pull/2773
 
 # v1.8
 

data/README.md

@@ -6,10 +6,6 @@ Travis CI:
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
-GitLab CI:
-
-[![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
-
 Drone CI for Arm64:
 
 [![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)

data/fluentd.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.4'
 
-  gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
+  gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
@@ -39,13 +39,14 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
   end
 
-  gem.add_development_dependency("rake", ["~> 12.0"])
+  gem.add_development_dependency("rake", ["~> 13.0"])
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
   gem.add_development_dependency("rr", ["~> 1.0"])
-  gem.add_development_dependency("timecop", ["~> 0.3"])
-  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("timecop", ["~> 0.9"])
+  gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
+  gem.add_development_dependency("ext_monitor", [">= 0.1.1", "< 0.2"])
 end

data/lib/fluent/command/cat.rb

@@ -101,13 +101,11 @@ rescue
   usage $!.to_s
 end
 
-
 require 'thread'
-require 'monitor'
 require 'socket'
 require 'yajl'
 require 'msgpack'
-
+require 'fluent/ext_monitor_require'
 
 class Writer
   include MonitorMixin

data/lib/fluent/command/plugin_generator.rb

@@ -281,7 +281,8 @@ HELP
       @text = ""
       @preamble_source = ""
       @preamble = nil
-      URI.open(LICENSE_URL) do |io|
+      uri = URI.parse(LICENSE_URL)
+      uri.open do |io|
         @text = io.read
       end
       @preamble_source = @text[/^(\s*Copyright.+)/m, 1]

data/lib/fluent/ext_monitor_require.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# To avoid duplicated requirements, extract this logic as file.
+
+if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')
+  require 'monitor'
+else
+  begin
+    # monitor_ext is bundled since ruby 2.7.0
+    require 'ext_monitor'
+  rescue LoadError => _
+    require 'monitor'
+  end
+end

data/lib/fluent/load.rb

@@ -2,7 +2,6 @@ require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'
-require 'monitor'
 require 'stringio'
 require 'fileutils'
 require 'json'
@@ -33,3 +32,4 @@ require 'fluent/input'
 require 'fluent/output'
 require 'fluent/filter'
 require 'fluent/match'
+require 'fluent/ext_monitor_require'

data/lib/fluent/msgpack_factory.rb

@@ -44,8 +44,8 @@ module Fluent
       end
     end
 
-    def self.engine_factory
-      @@engine_factory || factory
+    def self.engine_factory(enable_time_support: false)
+      @@engine_factory || factory(enable_time_support: enable_time_support)
     end
 
     def self.msgpack_packer(*args)
@@ -56,9 +56,15 @@ module Fluent
       engine_factory.unpacker(*args)
     end
 
-    def self.factory
+    def self.factory(enable_time_support: false)
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
       factory
     end
 
@@ -70,9 +76,15 @@ module Fluent
       factory.unpacker(*args)
     end
 
-    def self.init
+    def self.init(enable_time_support: false)
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
       @@engine_factory = factory
     end
 

data/lib/fluent/plugin/buffer.rb

@@ -17,8 +17,7 @@
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
 require 'fluent/unique_id'
-
-require 'monitor'
+require 'fluent/ext_monitor_require'
 
 module Fluent
   module Plugin

data/lib/fluent/plugin/buffer/chunk.rb

@@ -18,8 +18,8 @@ require 'fluent/plugin/buffer'
 require 'fluent/plugin/compressable'
 require 'fluent/unique_id'
 require 'fluent/event'
+require 'fluent/ext_monitor_require'
 
-require 'monitor'
 require 'tempfile'
 require 'zlib'
 

data/lib/fluent/plugin/output.rb

@@ -25,9 +25,9 @@ require 'fluent/plugin_helper'
 require 'fluent/timezone'
 require 'fluent/unique_id'
 require 'fluent/clock'
+require 'fluent/ext_monitor_require'
 
 require 'time'
-require 'monitor'
 
 module Fluent
   module Plugin

data/lib/fluent/plugin_helper/server.rb

@@ -711,7 +711,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/supervisor.rb

@@ -536,7 +536,7 @@ module Fluent
 
       begin
         ServerEngine::Privilege.change(@chuser, @chgroup)
-        MessagePackFactory.init
+        MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config, supervisor_mode: true)
         Fluent::Engine.run_configure(@conf, dry_run: dry_run)
       rescue Fluent::ConfigError => e
@@ -584,7 +584,7 @@ module Fluent
       main_process do
         create_socket_manager if @standalone_worker
         ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
-        MessagePackFactory.init
+        MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)
         Fluent::Engine.run

data/lib/fluent/system_config.rb

@@ -27,7 +27,7 @@ module Fluent
       :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value
+      :strict_config_value, :enable_msgpack_time_support
     ]
 
     config_param :workers,   :integer, default: 1
@@ -42,6 +42,7 @@ module Fluent
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
+    config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.9.0.rc1'
+  VERSION = '1.9.0.rc2'
 
 end

data/test/config/test_system_config.rb

@@ -76,6 +76,7 @@ module Fluent::Config
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_nil(sc.enable_msgpack_time_support)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
     end
@@ -89,6 +90,7 @@ module Fluent::Config
       'suppress_config_dump' => ['suppress_config_dump', true],
       'without_source' => ['without_source', true],
       'strict_config_value' => ['strict_config_value', true],
+      'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
     )
     test "accepts parameters" do |(k, v)|
       conf = parse_text(<<-EOS)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.9.0.rc1
+  version: 1.9.0.rc2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-07 00:00:00.000000000 Z
+date: 2020-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
@@ -178,14 +178,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
@@ -248,28 +248,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
@@ -304,6 +304,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: ext_monitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 description: Fluentd is an open source data collector designed to scale and simplify
   log management. It can collect, process and ship many kinds of data in near real-time.
 email:
@@ -327,7 +347,6 @@ files:
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".gitignore"
 - ".gitlab-ci.yml"
-- ".gitlab/cicd-template.yaml"
 - ".travis.yml"
 - ADOPTERS.md
 - AUTHORS
@@ -340,7 +359,6 @@ files:
 - MAINTAINERS.md
 - README.md
 - Rakefile
-- Vagrantfile
 - appveyor.yml
 - bin/fluent-binlog-reader
 - bin/fluent-ca-generate
@@ -448,6 +466,7 @@ files:
 - lib/fluent/error.rb
 - lib/fluent/event.rb
 - lib/fluent/event_router.rb
+- lib/fluent/ext_monitor_require.rb
 - lib/fluent/filter.rb
 - lib/fluent/fluent_log_event_router.rb
 - lib/fluent/formatter.rb

data/.gitlab/cicd-template.yaml

@@ -1,10 +0,0 @@
-.install-template:
-  before_script:
-    - gem update --system=2.7.8
-    - ruby -v 
-
-.test-template:
-  before_script:
-    - gem install rake
-    - bundle install 
-    - chmod 777 -R ./test

data/Vagrantfile

@@ -1,17 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-Vagrant.configure(2) do |config|
-  config.vm.box = "ubuntu/trusty64"
-  config.vm.network "private_network", type: "dhcp"
-  config.vm.provision "shell", inline: <<-SHELL
-    sudo apt-get install -y software-properties-common
-    sudo apt-add-repository ppa:brightbox/ruby-ng
-    sudo apt-get update
-    sudo apt-get install -y build-essential git ruby2.3 ruby2.3-dev
-    sudo apt-get install -y ruby-switch
-    sudo ruby-switch --set ruby2.3
-    gem install bundler
-    (cd /vagrant; bundle install)
-  SHELL
-end