fluentd 1.5.2 → 1.6.0

data/lib/fluent/plugin/out_forward.rb

@@ -39,6 +39,8 @@ module Fluent::Plugin
 
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
+    desc 'The timeout time for socket connect'
+    config_param :connect_timeout, :time, default: nil
     # TODO: add linger_timeout, recv_timeout
 
     desc 'The protocol to use for heartbeats (default is the same with "transport").'
@@ -376,6 +378,7 @@ module Fluent::Plugin
           linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
+          connect_timeout: @connect_timeout,
           &block
         )
       when :tcp
@@ -384,6 +387,7 @@ module Fluent::Plugin
           linger_timeout: @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
+          connect_timeout: @connect_timeout,
           &block
         )
       else

data/lib/fluent/plugin/output.rb

@@ -184,6 +184,8 @@ module Fluent
         @emit_records = 0
         @write_count = 0
         @rollback_count = 0
+        @flush_time_count = 0
+        @slow_flush_count = 0
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
         if implement?(:synchronous)
@@ -1173,7 +1175,10 @@ module Fluent
               @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
             end
           end
-          @buffer.takeback_chunk(chunk.unique_id)
+
+          if @buffer.takeback_chunk(chunk.unique_id)
+            @counters_monitor.synchronize { @rollback_count += 1 }
+          end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
 
@@ -1202,7 +1207,10 @@ module Fluent
 
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
+        elapsed_millsec = (elapsed_time * 1000).to_i
+        @counters_monitor.synchronize { @flush_time_count += elapsed_millsec }
         if elapsed_time > @slow_flush_log_threshold
+          @counters_monitor.synchronize { @slow_flush_count += 1 }
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1457,6 +1465,28 @@ module Fluent
           state.mutex.unlock
         end
       end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records,
+          # Respect original name
+          # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
+          'retry_count' => @num_errors,
+          'emit_count' => @emit_count,
+          'write_count' => @write_count,
+          'rollback_count' => @rollback_count,
+          'slow_flush_count' => @slow_flush_count,
+          'flush_time_count' => @flush_time_count,
+        }
+
+        if @buffer && @buffer.respond_to?(:statistics)
+          (@buffer.statistics['buffer'] || {}).each do |k, v|
+            stats["buffer_#{k}"] = v
+          end
+        end
+
+        { 'output' => stats }
+      end
     end
   end
 end

data/lib/fluent/plugin/parser_none.rb

@@ -27,8 +27,7 @@ module Fluent
       config_param :message_key, :string, default: 'message'
 
       def parse(text)
-        record = {}
-        record[@message_key] = text
+        record = {@message_key => text}
         time = @estimate_current_event ? Fluent::EventTime.now : nil
         yield time, record
       end

data/lib/fluent/plugin_helper.rb

@@ -22,6 +22,7 @@ require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin_helper/storage'
 require 'fluent/plugin_helper/parser'
 require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin_helper/http_server'
 require 'fluent/plugin_helper/inject'
 require 'fluent/plugin_helper/extract'
 require 'fluent/plugin_helper/socket'

data/lib/fluent/plugin_helper/cert_option.rb

@@ -168,7 +168,7 @@ module Fluent
 
       def cert_option_certificates_from_file(path)
         data = File.read(path)
-        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n?', Regexp::MULTILINE)
         list = []
         data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
         list

data/lib/fluent/plugin_helper/http_server.rb

@@ -0,0 +1,75 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+begin
+  # raise if RUBY_VERSION < 2.3.x. see Gemfile
+  require 'async'
+  require 'fluent/plugin_helper/http_server/server'
+rescue LoadError => _
+  require 'fluent/plugin_helper/http_server/compat/server'
+  Fluent::PluginHelper::HttpServer::Server = Fluent::PluginHelper::HttpServer::Compat::Server
+end
+
+require 'fluent/plugin_helper/thread'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      include Fluent::PluginHelper::Thread
+      # stop     : stop http server and mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill thread
+
+      # @param addr [String] Listen address
+      # @param port [String] Listen port
+      # @param logger [Logger] logger used in this server
+      # @param default_app [Object] This method must have #call.
+      def create_http_server(addr:, port:, logger:, default_app: nil)
+        unless block_given?
+          raise ArgumentError, 'BUG: callback not specified'
+        end
+
+        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
+          yield(serv)
+        end
+
+        _block_until_http_server_start do |notify|
+          thread_create(:plugin_helper_http_server) do
+            @_http_server.start(notify)
+          end
+        end
+      end
+
+      def stop
+        if @_http_server
+          @_http_server.stop
+        end
+
+        super
+      end
+
+      private
+
+      # To block until server is ready to listen
+      def _block_until_http_server_start
+        que = Queue.new
+        yield(que)
+        que.pop
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/app.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/request'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class App
+        def initialize(router, logger)
+          @logger = logger
+          @router = router
+        end
+
+        # Required method by async-http
+        def call(request)
+          method = request.method
+          resp =
+            case method
+            when HttpServer::Methods::GET
+              get(request)
+            when HttpServer::Methods::HEAD
+              head(request)
+            when HttpServer::Methods::POST
+              post(request)
+            when HttpServer::Methods::PATCH
+              patch(request)
+            when HttpServer::Methods::PUT
+              put(request)
+            when HttpServer::Methods::DELETE
+              delete(request)
+            when HttpServer::Methods::OPTIONS
+              options(request)
+            when HttpServer::Methods::CONNECT
+              connect(request)
+            when HttpServer::Methods::TRACE
+              trace(request)
+            else
+              raise "Unknown method #{method}"
+            end
+          Protocol::HTTP::Response[*resp]
+        rescue => e
+          @logger.error(e)
+          Protocol::HTTP::Response[500, { 'Content-Type' => 'text/plain' }, 'Internal Server Error']
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |request|
+            req = Request.new(request)
+
+            path = req.path
+            canonical_path =
+              if path.size >= 2 && !path.end_with?('/')
+                "#{path}/"
+              else
+                path
+              end
+            @router.route!(name, canonical_path, req)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -0,0 +1,81 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/compat/webrick_handler'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class Server
+          # @param logger [Logger]
+          # @param default_app [Object] ignored option. only for compat
+          def initialize(addr:, port:, logger:, default_app: nil)
+            @addr = addr
+            @port = port
+            @logger = logger
+            @server = WEBrick::HTTPServer.new(
+              BindAddress: @addr,
+              Port: @port,
+              Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+              AccessLog: [],
+            )
+
+            # @example ["/example.json", :get, handler object]
+            @methods = []
+
+            if block_given?
+              yield(self)
+            end
+          end
+
+          def start(notify = nil)
+            build_handler
+            notify.push(:ready)
+            @logger.debug('Start webrick HTTP server listening')
+            @server.start
+          end
+
+          def stop
+            @server.shutdown
+            @server.stop
+          end
+
+          HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+            define_method(name) do |path, app = nil, &block|
+              if (block && app) || (!block && !app)
+                raise 'You must specify either app or block in the same time'
+              end
+
+              # Do not build a handler class here to able to handle multiple methods for single path.
+              @methods << [path, name, app || block]
+            end
+          end
+
+          private
+
+          def build_handler
+            @methods.group_by(&:first).each do |(path, rest)|
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
+              @server.mount(path, klass)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'webrick'
+require 'json'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class WebrickHandler
+          # **opt is enough. but I wrote a signature explicitly for readability
+          def self.build(get: nil, head: nil, post: nil, put: nil, patch: nil, delete: nil, connect: nil, options: nil, trace: nil)
+            opt = { get: get, head: head, post: post, put: put, patch: patch, delete: delete, connect: connect, options: options, trace: trace }
+
+            Class.new(WEBrick::HTTPServlet::AbstractServlet) do
+              HttpServer::Methods::ALL.each do |name|
+                define_method("do_#{name}") do |request, response|
+                  code, headers, body =
+                                 if request.path_info != ''
+                                   render_json(404, 'message' => 'Not found')
+                                 else
+                                   begin
+                                     opt[name.downcase.to_sym].call(request)
+                                   rescue => _
+                                     render_json(500, 'message' => 'Something went wrong')
+                                   end
+                                 end
+
+                  response.status = code
+                  headers.each { |k, v| response[k] = v }
+                  response.body = body
+                end
+
+                def render_json(code, obj)
+                  [code, { 'Content-Type' => 'application/json' }, obj.to_json]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/methods.rb

@@ -0,0 +1,35 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Methods
+        GET = 'GET'.freeze
+        HEAD = 'HEAD'.freeze
+        POST = 'POST'.freeze
+        PUT = 'PUT'.freeze
+        PATCH = 'PATCH'.freeze
+        DELETE = 'DELETE'.freeze
+        OPTIONS = 'OPTIONS'.freeze
+        CONNECT = 'CONNECT'.freeze
+        TRACE = 'TRACE'.freeze
+
+        ALL = [GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS, TRACE].freeze
+      end
+    end
+  end
+end