fluentd 1.5.2 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e5e328bed0abe290a9ae9b9f4b84fc725064340f4b440e346eaa353dff653f8
-  data.tar.gz: b61c74d2c260da91d2141d261db4787448677e1926c63e3a5b08f5d5f98cfc82
+  metadata.gz: 8d97f061ed5f540026a4cae384b2508ee503be4dcb0ef836d37e118a541477df
+  data.tar.gz: 82edeaa9350871f8ba1b38d966cfc331f56c5ba97464c3dd14fb2bf7005a1bb5
 SHA512:
-  metadata.gz: af15e7b57e2bde74c846e71f768e7293901e68b4fc1d3a6d6a2620874d43df272611a44337b40437ec1307d2964cec5b7f057ac7c7451a2f89f1e74a8b40d679
-  data.tar.gz: b4a0e4c283faf5f8ff2144ad337a5993a99c0bfc2e4eae43dbad9b234d0f6a03420fcb4a149151effdc1976b507760eb453b754ced5ff4c6b8f323d4c9d3afad
+  metadata.gz: 7966763ba1215bde1a571f500dedaf5a100b6e72f14fb8b8121e09ca7d3fe697e4c6910b3dd7f844bd3d86484f0ea656997baff81fd9332aab94a898ff2e74d7
+  data.tar.gz: be65ac015fc43ac71d52ad3a2bb628e904a285918588b31102ecdf51e84af689ef77007028b30c72a9ecc25aa01e50a88091f136829396bf7cacaa608312b591

data/.github/ISSUE_TEMPLATE.md

@@ -1,8 +1,17 @@
 Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
 
-- fluentd or td-agent version.
-- Environment information:
-	- Operating system: `cat /etc/os-release`
-	- Kernel version: `uname -r`
-- Your configuration
-- Your problem explanation. If you have an error logs, write it together.
+**Got a question or problem?**
+
+RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
+
+If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).
+Don't use Github issue for asking questions. Here are examples:
+
+- I installed xxx plugin but it doesn't work. Why?
+- Fluentd starts but logs are not sent to xxx. Am I wrong?
+- I want to do xxx. How to realize it with plugins?
+
+We may close such questions to keep clear repository for developers and users.
+Github issue is mainly for submitting a bug report or feature request. See below.
+
+If you can't judge your case is a bug or not, use mailing list or slack first.

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,39 @@
+---
+name:  Bug Report
+about: Create a report to help us improve. If you have questions about Fluentd and plugins, please direct these to https://groups.google.com/forum/#!forum/fluentd
+labels: 'bug'
+
+---
+
+Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior: -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Your Environment**
+
+- Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+- Operating system: `cat /etc/os-release`
+- Kernel version: `uname -r`
+
+**Your Configuration**
+
+```
+<!-- Write your configuration here -->
+```
+
+**Your Error Log**
+
+```
+<!-- Write your **ALL** error log here -->
+```
+
+**Additional context**
+
+<!-- Add any other context about the problem here. -->

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+
+**Is your feature request related to a problem? Please describe.**
+
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+
+<!-- Add any other context or screenshots about the feature request here. -->

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+# v1.6
+
+## Release v1.6.0 - 2019/07/01
+
+### New feature
+
+* plugin: Add http_server helper and in_monitor_agent use it
+  https://github.com/fluent/fluentd/pull/2447
+
+### Enhancement
+
+* in_monitor_agent: Add more metrics for buffer/output
+  https://github.com/fluent/fluentd/pull/2450
+* time/plugin: Add `EventTime#to_time` method for fast conversion
+  https://github.com/fluent/fluentd/pull/2469
+* socket helper/out_forward: Add connect_timeout parameter
+  https://github.com/fluent/fluentd/pull/2467
+* command: Add `--conf-encoding` option
+  https://github.com/fluent/fluentd/pull/2453
+* parser_none: Small performance optimization
+  https://github.com/fluent/fluentd/pull/2455
+
+### Bug fixes
+
+* cert: Fix cert match pattern
+  https://github.com/fluent/fluentd/pull/2466
+* output: Fix forget to increment rollback count
+  https://github.com/fluent/fluentd/pull/2462
+
 # v1.5
 
 ## Release v1.5.2 - 2019/06/13

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs) repository.
+* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/Gemfile

@@ -2,6 +2,11 @@ source 'https://rubygems.org/'
 
 gemspec
 
+# https://github.com/socketry/async-io/blob/v1.23.1/async-io.gemspec#L21
+if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.3.0')
+  gem 'async-http', '~> 0.42'
+end
+
 local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
 if File.exist?(local_gemfile)
   puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`

data/Rakefile

@@ -34,7 +34,12 @@ Rake::TestTask.new(:base_test) do |t|
   t.test_files = if ENV["WIN_RAPID"]
                    ["test/test_event.rb", "test/test_supervisor.rb", "test/plugin_helper/test_event_loop.rb"]
                  else
-                   Dir["test/**/test_*.rb"].sort
+                   tests = Dir["test/**/test_*.rb"].sort
+                   if Process.uid.zero?
+                     puts "test_file_util.rb for non-root user env. Disable this test on root environment"
+                     tests.delete("test/plugin/test_file_util.rb")
+                   end
+                   tests
                  end
   t.verbose = true
   t.warning = true

data/appveyor.yml

@@ -5,10 +5,11 @@ version: '{build}'
 
 install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - "%devkit%\\devkitvars.bat"
+  - IF %ridk%==0 "%devkit%\\devkitvars.bat"
   - ruby --version
   - gem --version
-  - bundle install
+  - IF %ridk%==1 ridk.cmd exec bundle install
+  - IF %ridk%==0 bundle install
 build: off
 test_script:
   - bundle exec rake test
@@ -17,22 +18,20 @@ test_script:
 branches:
   only:
     - master
-    - v0.10
-    - v0.12
-    - v0.14
 
 # https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
     - ruby_version: "23-x64"
       devkit: C:\Ruby23-x64\DevKit
+      ridk: 0
     - ruby_version: "23"
       devkit: C:\Ruby23\DevKit
-    - ruby_version: "22-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "22"
-      devkit: C:\Ruby23\DevKit
-      WIN_RAPID: true
+      ridk: 0
+    - ruby_version: "24-x64"
+      ridk: 1
+    - ruby_version: "24"
+      ridk: 1
 matrix:
   allow_failures:
     - ruby_version: "21"

data/lib/fluent/command/fluentd.rb

@@ -155,6 +155,10 @@ op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dir
   opts[:gem_install_path] = s
 }
 
+op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
+  opts[:conf_encoding] = s
+}
+
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library

data/lib/fluent/config/literal_parser.rb

@@ -215,7 +215,7 @@ EOM
             parsed = nil
             begin
               parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
-            rescue JSON::ParserError => e
+            rescue JSON::ParserError
               # This '#' is in json string literals
             end
 
@@ -243,7 +243,7 @@ EOM
           line_buffer << char
           begin
             result = JSON.parse(buffer + line_buffer)
-          rescue JSON::ParserError => e
+          rescue JSON::ParserError
             # Incomplete json string yet
           end
         end

data/lib/fluent/plugin/base.rb

@@ -29,6 +29,7 @@ module Fluent
       attr_accessor :under_plugin_development
 
       def initialize
+        @log = nil
         super
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil

data/lib/fluent/plugin/buffer.rb

@@ -740,6 +740,28 @@ module Fluent
         enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
       end
+
+      def statistics
+        buffer_space = 1.0 - ((@stage_size + @queue_size * 1.0) / @total_limit_size).round
+        stats = {
+          'stage_length' => @stage.size,
+          'stage_byte_size' => @stage_size,
+          'queue_length' => @queue.size,
+          'queue_byte_size' => @queue_size,
+          'available_buffer_space_ratios' => buffer_space * 100,
+          'total_queued_size' => @stage_size + @queue_size,
+        }
+
+        if (m = timekeys.min)
+          stats['oldest_timekey'] = m
+        end
+
+        if (m = timekeys.max)
+          stats['newest_timekey'] = m
+        end
+
+        { 'buffer' => stats }
+      end
     end
   end
 end

data/lib/fluent/plugin/in_forward.rb

@@ -135,13 +135,13 @@ module Fluent::Plugin
           if client.host
             begin
               source = IPSocket.getaddress(client.host)
-            rescue SocketError => e
+            rescue SocketError
               raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
             end
           end
           source_addr = begin
                           IPAddr.new(source || client.network)
-                        rescue ArgumentError => e
+                        rescue ArgumentError
                           raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
                         end
           @nodes.push({

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -28,7 +28,7 @@ module Fluent::Plugin
   class MonitorAgentInput < Input
     Fluent::Plugin.register_input('monitor_agent', self)
 
-    helpers :timer, :thread
+    helpers :timer, :thread, :http_server
 
     desc 'The address to bind to.'
     config_param :bind, :string, default: '0.0.0.0'
@@ -43,35 +43,83 @@ module Fluent::Plugin
     desc 'Determine whether to include the retry information.'
     config_param :include_retry, :bool, default: true
 
-    class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
-      def initialize(server, agent)
+    class APIHandler
+      def initialize(agent)
         @agent = agent
       end
 
-      def do_GET(req, res)
-        begin
-          code, header, body = process(req)
-        rescue
-          code, header, body = render_json_error(500, {
-              'message '=> 'Internal Server Error',
-              'error' => "#{$!}",
-              'backgrace'=> $!.backtrace,
-            })
-        end
+      def plugins_ltsv(req)
+        list = build_object(build_option(req))
 
-        # set response code, header and body
-        res.status = code
-        header.each_pair {|k,v|
-          res[k] = v
-        }
-        res.body = body
+        render_ltsv(list)
+      end
+
+      def plugins_json(req)
+        opts = build_option(req)
+        obj = build_object(opts)
+
+        render_json({ 'plugins' => obj }, pretty_json: opts[:pretty_json])
+      end
+
+      def config_ltsv(_req)
+        obj = {
+          'pid' => Process.pid,
+          'ppid' => Process.ppid
+        }.merge(@agent.fluentd_opts)
+
+        render_ltsv([obj])
+      end
+
+      def config_json(req)
+        obj = {
+          'pid' => Process.pid,
+          'ppid' => Process.ppid
+        }.merge(@agent.fluentd_opts)
+        opts = build_option(req)
+
+        render_json(obj, pretty_json: opts[:pretty_json])
+      end
+
+      private
+
+      def render_error_json(code:, msg:, pretty_json: nil, **additional_params)
+        resp = additional_params.merge('message' => msg)
+        render_json(resp, code: code, pretty_json: pretty_json)
       end
 
-      def build_object(req, opts)
-        unless req.path_info == ""
-          return render_json_error(404, "Not found")
+      def render_json(obj, code: 200, pretty_json: nil)
+        body =
+          if pretty_json
+            JSON.pretty_generate(obj)
+          else
+            obj.to_json
+          end
+
+        [code, { 'Content-Type' => 'application/json' }, body]
+      end
+
+      def render_ltsv(obj, code: 200)
+        normalized = JSON.parse(obj.to_json)
+        text = ''
+        normalized.each do |hash|
+          row = []
+          hash.each do |k, v|
+            if v.is_a?(Array)
+              row << "#{k}:#{v.join(',')}"
+            elsif v.is_a?(Hash)
+              next
+            else
+              row << "#{k}:#{v}"
+            end
+          end
+
+          text << row.join("\t") << "\n"
         end
 
+        [code, { 'Content-Type' => 'text/plain' }, text]
+      end
+
+      def build_object(opts)
         qs = opts[:query]
         if tag = qs['tag'.freeze].first
           # ?tag= to search an output plugin by match pattern
@@ -98,30 +146,11 @@ module Fluent::Plugin
         list
       end
 
-      def render_json(obj, opts={})
-        render_json_error(200, obj, opts)
-      end
-
-      def render_json_error(code, obj, opts={})
-        if opts[:pretty_json]
-          js = JSON.pretty_generate(obj)
-        else
-          js = obj.to_json
-        end
-        [code, {'Content-Type'=>'application/json'}, js]
-      end
-
-      private
-
       def build_option(req)
         qs = Hash.new { |_, _| [] }
         # parse ?=query string
         if req.query_string
-          begin
-            qs.merge!(CGI.parse(req.query_string))
-          rescue
-            return render_json_error(400, "Invalid query string")
-          end
+          qs.merge!(CGI.parse(req.query_string))
         end
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
@@ -152,74 +181,6 @@ module Fluent::Plugin
       end
     end
 
-    class LTSVMonitorServlet < MonitorServlet
-      def process(req)
-        opts = build_option(req)
-        list = build_object(req, opts)
-        return unless list
-
-        normalized = JSON.parse(list.to_json)
-
-        text = ''
-
-        normalized.map {|hash|
-          row = []
-          hash.each_pair {|k,v|
-            unless v.is_a?(Hash) || v.is_a?(Array)
-              row << "#{k}:#{v}"
-            end
-          }
-          text << row.join("\t") << "\n"
-        }
-
-        [200, {'Content-Type'=>'text/plain'}, text]
-      end
-    end
-
-    class JSONMonitorServlet < MonitorServlet
-      def process(req)
-        opts = build_option(req)
-        list = build_object(req, opts)
-        return unless list
-
-        render_json({
-            'plugins' => list
-          }, opts)
-      end
-    end
-
-    class ConfigMonitorServlet < MonitorServlet
-      def build_object(req, _opt)
-        {
-          'pid' => Process.pid,
-          'ppid' => Process.ppid
-        }.merge(@agent.fluentd_opts)
-      end
-    end
-
-    class LTSVConfigMonitorServlet < ConfigMonitorServlet
-      def process(req)
-        opts = build_option(req)
-        result = build_object(req, opts)
-
-        row = []
-        JSON.parse(result.to_json).each_pair { |k, v|
-          row << "#{k}:#{v}"
-        }
-        text = row.join("\t")
-
-        [200, {'Content-Type'=>'text/plain'}, text]
-      end
-    end
-
-    class JSONConfigMonitorServlet < ConfigMonitorServlet
-      def process(req)
-        opts = build_option(req)
-        result = build_object(req, opts)
-        render_json(result, opts)
-      end
-    end
-
     def initialize
       super
 
@@ -235,23 +196,25 @@ module Fluent::Plugin
       true
     end
 
+    class NotFoundJson
+      BODY = { 'message' => 'Not found' }.to_json
+      def self.call(_req)
+        [404, { 'Content-Type' => 'application/json' }, BODY]
+      end
+    end
+
     def start
       super
 
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
-      @srv = WEBrick::HTTPServer.new({
-          BindAddress: @bind,
-          Port: @port,
-          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-          AccessLog: [],
-        })
-      @srv.mount('/api/plugins', LTSVMonitorServlet, self)
-      @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
-      @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
-      @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
-      thread_create :in_monitor_agent_servlet do
-        @srv.start
+      api_handler = APIHandler.new(self)
+      create_http_server(addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
+        serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
+        serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }
+        serv.get('/api/config') { |req| api_handler.config_ltsv(req) }
+        serv.get('/api/config.json') { |req| api_handler.config_json(req) }
       end
+
       if @tag
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
@@ -267,15 +230,7 @@ module Fluent::Plugin
       end
     end
 
-    def shutdown
-      if @srv
-        @srv.shutdown
-        @srv = nil
-      end
-
-      super
-    end
-
+    # They are deprecated but remain for compatibility
     MONITOR_INFO = {
       'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
       'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
@@ -377,6 +332,10 @@ module Fluent::Plugin
         end
       }
 
+      if pe.respond_to?(:statistics)
+        obj.merge!(pe.statistics['output'] || {})
+      end
+
       obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
 
       # include all instance variables if :with_debug_info is set