fluentd 1.5.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e334a69c853a549b872123dd1b6232f04dfba60aea75c197b51940ad051e422
-  data.tar.gz: af6be10d887398a5a8efb08915cd5fec348da6fffde98199df39f7e9a9f6e803
+  metadata.gz: cc00110d00a473f82be35d411b2efcad974613e6ac16db59538840bced53ce02
+  data.tar.gz: 4bcb347f931027fabff0516c589a6b439763a0f36e38c3707213955f8b2fce2b
 SHA512:
-  metadata.gz: 6aec4c3726486c0fe5fd9734a33e3813f222578211bf1e71ffa102aebcc14af4ed3a99b789a12f4f38e9a3d95379c90363e8921c9557be03bda756cbfafb76c3
-  data.tar.gz: b161392482f816ad8cc4eaa45a55529ec13107ed8a1020a4e7ef45a18f2949580eab5cf833c1e4462e98118d7fd1f0a41e20537c9a2c57b55f423bb4857fd147
+  metadata.gz: 9eaf4fab8332a2ba7d172b6d63ed08eecab23f5bdcdbc56fe557b3a888bf155e3eb404bfa0a583dc3483fdee5ab5a415f8e63157379073299815c194bf45f2b3
+  data.tar.gz: be56a1d495a143a136a716f0b9142dfcc22d0c4ed88bd188ad0bc90e8253444eb47e70f6f7d4274855feb47c45f051ebd01fd399fbc046ab2d18ca8a9e321140

data/.gitlab-ci.yml

@@ -0,0 +1,147 @@
+include:
+  - local: '/.gitlab/cicd-template.yaml'
+
+stages:
+  - build
+  - tests
+
+2-1-10:
+  image: "ruby:2.1.10"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-1-10"
+    paths:
+      - ./*
+
+2-2-10:
+  image: "ruby:2.2.10"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-2-10"
+    paths:
+
+2-4-5:
+  image: "ruby:2.4.5"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-4-5"
+    paths:
+
+2-5-5:
+  image: "ruby:2.5.5"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-5-5"
+    paths:
+
+2-6-3:
+  image: "ruby:2.6.3"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-6-3"
+    paths:
+
+rubyhead:
+  image: "ruby:latest"
+  stage: build
+  allow_failure: true
+  extends: .install-template 
+  script:
+    - bundle install --jobs=3 --retry=3
+  cache:
+    key: "$CI_COMMIT_REF_SLUG latest"
+    paths:
+      - ./*
+
+2-1-10-test:
+  image: "ruby:2.1.10"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-1-10"
+    paths:
+      - ./*
+
+2-2-10-test:
+  image: "ruby:2.2.10"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-2-10"
+    paths:
+      - ./*
+
+2-4-5-test:
+  image: "ruby:2.4.5"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-4-5"
+    paths:
+      - ./*
+
+2-5-5-test:
+  image: "ruby:2.5.5"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-5-5"
+    paths:
+      - ./*
+
+2-6-3-test:
+  image: "ruby:2.6.3"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG 2-6-3"
+    paths:
+      - ./*
+
+rubyhead-test:
+  image: "ruby:latest"
+  stage: tests
+  allow_failure: true
+  extends: .test-template
+  script:
+    - bundle exec rake test
+  cache:
+    key: "$CI_COMMIT_REF_SLUG latest"
+    paths:
+      - ./*
+    

data/.gitlab/cicd-template.yaml

@@ -0,0 +1,10 @@
+.install-template:
+  before_script:
+    - gem update --system=2.7.8
+    - ruby -v 
+
+.test-template:
+  before_script:
+    - gem install rake
+    - bundle install 
+    - chmod 777 -R ./test

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # v1.5
 
+## Release v1.5.1 - 2019/06/05
+
+### Enhancement
+
+* in_tail: Increase read block size to reduce IO call
+  https://github.com/fluent/fluentd/pull/2418
+* in_monitor_agent: Refactor code
+  https://github.com/fluent/fluentd/pull/2422
+
+### Bug fixes
+
+* out_forward: Fix socket handling of keepalive
+  https://github.com/fluent/fluentd/pull/2434
+* parser: Fix the use of name based timezone
+  https://github.com/fluent/fluentd/pull/2421
+* in_monitor_agent: Fix debug parameter handling
+  https://github.com/fluent/fluentd/pull/2423
+* command: Fix error handling of log rotation age option
+  https://github.com/fluent/fluentd/pull/2427
+* command: Fix ERB warning for ruby 2.6 or later
+  https://github.com/fluent/fluentd/pull/2430
+
 ## Release v1.5.0 - 2019/05/18
 
 ### New feature

data/README.md

@@ -1,10 +1,15 @@
 Fluentd: Open-Source Log Collector
 ===================================
+Travis CI:
 
 [<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
+GitLab CI:
+
+[![pipeline status](https://gitlab.com/fluent/fluentd/badges/master/pipeline.svg)](https://gitlab.com/fluent/fluentd/commits/master)
+
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">

data/lib/fluent/command/fluentd.rb

@@ -92,8 +92,8 @@ op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
   else
     begin
       opts[:log_rotate_age] = Integer(age)
-    rescue TypeError
-      usage "log-rotate-age should be #{rotate_ages.join(', ')} or a number"
+    rescue TypeError, ArgumentError
+      usage "log-rotate-age should be #{ROTATE_AGE.join(', ')} or a number"
     end
   end
 }

data/lib/fluent/command/plugin_config_formatter.rb

@@ -164,7 +164,13 @@ class FluentPluginConfigFormatter
     params.each do |name, config|
       next if name == :section
       template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
-      dumped << ERB.new(template_path(template_name).read, nil, "-").result(binding)
+      template = template_path(template_name).read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
     end
     dumped << "\n"
     sections.each do |section_name, sub_section|
@@ -172,7 +178,13 @@ class FluentPluginConfigFormatter
       multi = sub_section.delete(:multi)
       alias_name = sub_section.delete(:alias)
       sub_section.delete(:section)
-      dumped << ERB.new(template_path("section.md.erb").read, nil, "-").result(binding)
+      template = template_path("section.md.erb").read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
     end
     dumped
   end

data/lib/fluent/command/plugin_generator.rb

@@ -72,7 +72,12 @@ class FluentPluginGenerator
 
   def template(source, dest)
     dest.dirname.mkpath
-    contents = ERB.new(source.read, nil, "-").result(binding)
+    contents =
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(source.read, trim_mode: "-")
+      else
+        ERB.new(source.read, nil, "-")
+      end.result(binding)
     label = create_label(dest, contents)
     puts "\t#{label} #{dest}"
     if label == "conflict"

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -50,7 +50,7 @@ module Fluent::Plugin
 
       def do_GET(req, res)
         begin
-          code, header, body = process(req, res)
+          code, header, body = process(req)
         rescue
           code, header, body = render_json_error(500, {
               'message '=> 'Internal Server Error',
@@ -67,85 +67,35 @@ module Fluent::Plugin
         res.body = body
       end
 
-      def build_object(req, res)
+      def build_object(req, opts)
         unless req.path_info == ""
           return render_json_error(404, "Not found")
         end
 
-        # parse ?=query string
-        if req.query_string
-          begin
-            qs = CGI.parse(req.query_string)
-          rescue
-            return render_json_error(400, "Invalid query string")
-          end
-        else
-          qs = Hash.new {|h,k| [] }
-        end
-
-        # if ?debug=1 is set, set :with_debug_info for get_monitor_info
-        # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
-        if s = qs['debug'] and s[0]
-          opts[:with_debug_info] = true
-          opts[:pretty_json] = true
-        end
-
-        if ivars = (qs['with_ivars'] || []).first
-          opts[:ivars] = ivars.split(',')
-        end
-
-        if with_config = get_search_parameter(qs, 'with_config'.freeze)
-          opts[:with_config] = Fluent::Config.bool_value(with_config)
-        end
-
-        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
-          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
-        end
-
-        if tag = get_search_parameter(qs, 'tag'.freeze)
+        qs = opts[:query]
+        if tag = qs['tag'.freeze].first
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
             list = [obj]
           else
             list = []
           end
-
-        elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
+        elsif plugin_id = (qs['@id'.freeze].first || qs['id'.freeze].first)
           # ?@id= to search a plugin by 'id <plugin_id>' config param
           if obj = @agent.plugin_info_by_id(plugin_id, opts)
             list = [obj]
           else
             list = []
           end
-
-        elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
-          # Without @ version of ?@id= for backward compatibility
-          if obj = @agent.plugin_info_by_id(plugin_id, opts)
-            list = [obj]
-          else
-            list = []
-          end
-
-        elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
+        elsif plugin_type = (qs['@type'.freeze].first || qs['type'.freeze].first)
           # ?@type= to search plugins by 'type <type>' config param
           list = @agent.plugins_info_by_type(plugin_type, opts)
-
-        elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
-          # Without @ version of ?@type= for backward compatibility
-          list = @agent.plugins_info_by_type(plugin_type, opts)
-
         else
           # otherwise show all plugins
           list = @agent.plugins_info_all(opts)
         end
 
-        return list, opts
-      end
-
-      def get_search_parameter(qs, param_name)
-        return nil unless qs.has_key?(param_name)
-        qs[param_name].first
+        list
       end
 
       def render_json(obj, opts={})
@@ -160,11 +110,52 @@ module Fluent::Plugin
         end
         [code, {'Content-Type'=>'application/json'}, js]
       end
+
+      private
+
+      def build_option(req)
+        qs = Hash.new { |_, _| [] }
+        # parse ?=query string
+        if req.query_string
+          begin
+            qs.merge!(CGI.parse(req.query_string))
+          rescue
+            return render_json_error(400, "Invalid query string")
+          end
+        end
+
+        # if ?debug=1 is set, set :with_debug_info for get_monitor_info
+        # and :pretty_json for render_json_error
+        opts = { query: qs }
+        if qs['debug'.freeze].first
+          opts[:with_debug_info] = true
+          opts[:pretty_json] = true
+        end
+
+        if ivars = qs['with_ivars'.freeze].first
+          opts[:ivars] = ivars.split(',')
+        end
+
+        if with_config = qs['with_config'.freeze].first
+          opts[:with_config] = Fluent::Config.bool_value(with_config)
+        else
+          opts[:with_config] = @agent.include_config
+        end
+
+        if with_retry = qs['with_retry'.freeze].first
+          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
+        else
+          opts[:with_retry] = @agent.include_retry
+        end
+
+        opts
+      end
     end
 
     class LTSVMonitorServlet < MonitorServlet
-      def process(req, res)
-        list, _opts = build_object(req, res)
+      def process(req)
+        opts = build_option(req)
+        list = build_object(req, opts)
         return unless list
 
         normalized = JSON.parse(list.to_json)
@@ -186,8 +177,9 @@ module Fluent::Plugin
     end
 
     class JSONMonitorServlet < MonitorServlet
-      def process(req, res)
-        list, opts = build_object(req, res)
+      def process(req)
+        opts = build_option(req)
+        list = build_object(req, opts)
         return unless list
 
         render_json({
@@ -197,7 +189,7 @@ module Fluent::Plugin
     end
 
     class ConfigMonitorServlet < MonitorServlet
-      def build_object(req, res)
+      def build_object(req, _opt)
         {
           'pid' => Process.pid,
           'ppid' => Process.ppid
@@ -206,8 +198,9 @@ module Fluent::Plugin
     end
 
     class LTSVConfigMonitorServlet < ConfigMonitorServlet
-      def process(req, res)
-        result = build_object(req, res)
+      def process(req)
+        opts = build_option(req)
+        result = build_object(req, opts)
 
         row = []
         JSON.parse(result.to_json).each_pair { |k, v|
@@ -220,9 +213,10 @@ module Fluent::Plugin
     end
 
     class JSONConfigMonitorServlet < ConfigMonitorServlet
-      def process(req, res)
-        result = build_object(req, res)
-        render_json(result)
+      def process(req)
+        opts = build_option(req)
+        result = build_object(req, opts)
+        render_json(result, opts)
       end
     end
 

data/lib/fluent/plugin/in_tail.rb

@@ -732,7 +732,7 @@ module Fluent::Plugin
               if !io.nil? && @lines.empty?
                 begin
                   while true
-                    @fifo << io.readpartial(2048, @iobuf)
+                    @fifo << io.readpartial(8192, @iobuf)
                     while (line = @fifo.next_line)
                       @lines << line
                     end

data/lib/fluent/plugin/out_forward.rb

@@ -638,7 +638,7 @@ module Fluent::Plugin
               @active_socks[key] = TimedSocket.new(timeout, yield, 0)
             end
 
-            @active_socks[key].ref += 1;
+            @active_socks[key].ref += 1
             @active_socks[key].sock
           end
         end
@@ -1084,10 +1084,10 @@ module Fluent::Plugin
           begin
             yield(sock)
           rescue
-            @socket_cache.revoke(sock) if @keepalive
+            @socket_cache.revoke if @keepalive
             raise
           else
-            @socket_cache.dec_ref(sock) if @keepalive
+            @socket_cache.dec_ref if @keepalive
           ensure
             sock.close unless @keepalive
           end

data/lib/fluent/time.rb

@@ -203,7 +203,13 @@ module Fluent
       # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
       offset_diff = case
                     when format_with_timezone then nil
-                    when timezone  then Time.now.localtime.utc_offset - Time.zone_offset(timezone)
+                    when timezone  then
+                      offset = Fluent::Timezone.utc_offset(timezone)
+                      if offset.respond_to?(:call)
+                        ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
+                      else
+                        Time.now.localtime.utc_offset - offset
+                      end
                     when localtime then 0
                     else Time.now.localtime.utc_offset # utc
                     end
@@ -214,8 +220,18 @@ module Fluent
                when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
                when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
                when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
-               when strptime then ->(v){ t = strptime.exec(v);         Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
-               when format   then ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+               when strptime then
+                 if offset_diff.respond_to?(:call)
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 else
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                 end
+               when format   then
+                 if offset_diff.respond_to?(:call)
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 else
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                 end
                else ->(v){ Fluent::EventTime.parse(v) }
                end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.5.0'
+  VERSION = '1.5.1'
 
 end

data/test/plugin/test_in_monitor_agent.rb

@@ -414,6 +414,19 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert_equal(["/etc/fluent/plugin"], res["plugin_dirs"])
       assert_nil(res["log_path"])
     end
+
+    test "/api/config.json?debug=1" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      # To check pretty print
+      assert_true !get("http://127.0.0.1:#{@port}/api/config.json").body.include?("\n")
+      assert_true get("http://127.0.0.1:#{@port}/api/config.json?debug=1").body.include?("\n")
+    end
   end
 
   sub_test_case "check retry of buffered plugins" do

data/test/plugin/test_in_tail.rb

@@ -196,7 +196,7 @@ class TailInputTest < Test::Unit::TestCase
         config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
       end
       d = create_driver(config)
-      msg = 'test' * 500 # in_tail reads 2048 bytes at once.
+      msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
 
       d.run(expect_emits: 1) do
         File.open("#{TMP_DIR}/tail.txt", "ab") {|f|

data/test/test_time_parser.rb

@@ -80,6 +80,14 @@ class TimeParserTest < ::Test::Unit::TestCase
     assert_equal_event_time(time, event_time("2016-09-02 18:42:31.123456789 -07:00", format: '%Y-%m-%d %H:%M:%S.%N %z'))
   end
 
+  def test_parse_time_with_expected_timezone_name
+    time = with_timezone("UTC-09") do
+      parser = Fluent::TimeParser.new("%Y-%m-%d %H:%M:%S.%N", nil, "Europe/Zurich")
+      parser.parse("2016-12-02 18:42:31.123456789")
+    end
+    assert_equal_event_time(time, event_time("2016-12-02 18:42:31.123456789 +01:00", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+  end
+
   sub_test_case 'TimeMixin::Parser' do
     class DummyForTimeParser
       include Fluent::Configurable

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-18 00:00:00.000000000 Z
+date: 2019-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -331,6 +331,8 @@ files:
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".gitignore"
+- ".gitlab-ci.yml"
+- ".gitlab/cicd-template.yaml"
 - ".travis.yml"
 - ADOPTERS.md
 - AUTHORS