fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -0,0 +1,81 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/compat/webrick_handler'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class Server
+          # @param logger [Logger]
+          # @param default_app [Object] ignored option. only for compat
+          def initialize(addr:, port:, logger:, default_app: nil)
+            @addr = addr
+            @port = port
+            @logger = logger
+            @server = WEBrick::HTTPServer.new(
+              BindAddress: @addr,
+              Port: @port,
+              Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+              AccessLog: [],
+            )
+
+            # @example ["/example.json", :get, handler object]
+            @methods = []
+
+            if block_given?
+              yield(self)
+            end
+          end
+
+          def start(notify = nil)
+            build_handler
+            notify.push(:ready)
+            @logger.debug('Start webrick HTTP server listening')
+            @server.start
+          end
+
+          def stop
+            @server.shutdown
+            @server.stop
+          end
+
+          HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+            define_method(name) do |path, app = nil, &block|
+              if (block && app) || (!block && !app)
+                raise 'You must specify either app or block in the same time'
+              end
+
+              # Do not build a handler class here to able to handle multiple methods for single path.
+              @methods << [path, name, app || block]
+            end
+          end
+
+          private
+
+          def build_handler
+            @methods.group_by(&:first).each do |(path, rest)|
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
+              @server.mount(path, klass)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'webrick'
+require 'json'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class WebrickHandler
+          # **opt is enough. but I wrote a signature explicitly for readability
+          def self.build(get: nil, head: nil, post: nil, put: nil, patch: nil, delete: nil, connect: nil, options: nil, trace: nil)
+            opt = { get: get, head: head, post: post, put: put, patch: patch, delete: delete, connect: connect, options: options, trace: trace }
+
+            Class.new(WEBrick::HTTPServlet::AbstractServlet) do
+              HttpServer::Methods::ALL.each do |name|
+                define_method("do_#{name}") do |request, response|
+                  code, headers, body =
+                                 if request.path_info != ''
+                                   render_json(404, 'message' => 'Not found')
+                                 else
+                                   begin
+                                     opt[name.downcase.to_sym].call(request)
+                                   rescue => _
+                                     render_json(500, 'message' => 'Something went wrong')
+                                   end
+                                 end
+
+                  response.status = code
+                  headers.each { |k, v| response[k] = v }
+                  response.body = body
+                end
+
+                def render_json(code, obj)
+                  [code, { 'Content-Type' => 'application/json' }, obj.to_json]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/methods.rb

@@ -0,0 +1,35 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Methods
+        GET = 'GET'.freeze
+        HEAD = 'HEAD'.freeze
+        POST = 'POST'.freeze
+        PUT = 'PUT'.freeze
+        PATCH = 'PATCH'.freeze
+        DELETE = 'DELETE'.freeze
+        OPTIONS = 'OPTIONS'.freeze
+        CONNECT = 'CONNECT'.freeze
+        TRACE = 'TRACE'.freeze
+
+        ALL = [GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS, TRACE].freeze
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/request.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Request
+        attr_reader :path, :query_string
+
+        def initialize(request)
+          @request = request
+          path = request.path
+          @path, @query_string = path.split('?', 2)
+        end
+
+        def query
+          @query_string && CGI.parse(@query_string)
+        end
+
+        def body
+          @request.body && @request.body.read
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/router.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Router
+        class NotFoundApp
+          def self.call(req)
+            [404, { 'Content-Type' => 'text/plain' }, "404 Not Found: #{req.path}\n"]
+          end
+        end
+
+        def initialize(default_app = nil)
+          @router = { get: {}, head: {}, post: {}, put: {}, patch: {}, delete: {}, connect: {}, options: {}, trace: {} }
+          @default_app = default_app || NotFoundApp
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param app [Object]
+        def mount(method, path, app)
+          if @router[method].include?(path)
+            raise "#{path} is already mounted"
+          end
+
+          @router[method][path] = app
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param request [Fluent::PluginHelper::HttpServer::Request]
+        def route!(method, path, request)
+          @router.fetch(method).fetch(path, @default_app).call(request)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -0,0 +1,87 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async'
+require 'async/http'
+require 'async/http/endpoint'
+
+require 'fluent/plugin_helper/http_server/app'
+require 'fluent/plugin_helper/http_server/router'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Server
+        # @param default_app [Object] This method must have #call.
+        def initialize(addr:, port:, logger:, default_app: nil)
+          @addr = addr
+          @port = port
+          @logger = logger
+
+          # TODO: support https and http2
+          @uri = URI("http://#{@addr}:#{@port}").to_s
+          @router = Router.new(default_app)
+          @reactor = Async::Reactor.new
+          @server = Async::HTTP::Server.new(
+            App.new(@router, @logger),
+            Async::HTTP::Endpoint.parse(@uri)
+          )
+
+          if block_given?
+            yield(self)
+          end
+        end
+
+        def start(notify = nil)
+          @logger.debug("Start async HTTP server listening #{@uri}")
+          task = @reactor.run do
+            @server.run
+
+            if notify
+              notify.push(:ready)
+            end
+          end
+
+          task.stop
+          @logger.debug('Finished HTTP server')
+        end
+
+        def stop
+          @logger.debug('closing HTTP server')
+
+          if @reactor
+            @reactor.stop
+          end
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |path, app = nil, &block|
+            unless path.end_with?('/')
+              path << '/'
+            end
+
+            if (block && app) || (!block && !app)
+              raise 'You must specify either app or block in the same time'
+            end
+
+            @router.mount(name, path, app || block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server.rb

@@ -0,0 +1,76 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+begin
+  # raise if RUBY_VERSION < 2.3.x. see Gemfile
+  require 'async'
+  require 'fluent/plugin_helper/http_server/server'
+rescue LoadError => _
+  require 'fluent/plugin_helper/http_server/compat/server'
+  Fluent::PluginHelper::HttpServer::Server = Fluent::PluginHelper::HttpServer::Compat::Server
+end
+
+require 'fluent/plugin_helper/thread'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      include Fluent::PluginHelper::Thread
+      # stop     : stop http server and mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill thread
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param addr [String] Listen address
+      # @param port [String] Listen port
+      # @param logger [Logger] logger used in this server
+      # @param default_app [Object] This method must have #call.
+      def create_http_server(title, addr:, port:, logger:, default_app: nil)
+        unless block_given?
+          raise ArgumentError, 'BUG: callback not specified'
+        end
+
+        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
+          yield(serv)
+        end
+
+        _block_until_http_server_start do |notify|
+          thread_create(title) do
+            @_http_server.start(notify)
+          end
+        end
+      end
+
+      def stop
+        if @_http_server
+          @_http_server.stop
+        end
+
+        super
+      end
+
+      private
+
+      # To block until server is ready to listen
+      def _block_until_http_server_start
+        que = Queue.new
+        yield(que)
+        que.pop
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/inject.rb

@@ -0,0 +1,151 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+require 'socket'
+
+module Fluent
+  module PluginHelper
+    module Inject
+      def inject_values_to_record(tag, time, record)
+        return record unless @_inject_enabled
+
+        r = record.dup
+        if @_inject_hostname_key
+          r[@_inject_hostname_key] = @_inject_hostname
+        end
+        if @_inject_worker_id_key
+          r[@_inject_worker_id_key] = @_inject_worker_id
+        end
+        if @_inject_tag_key
+          r[@_inject_tag_key] = tag
+        end
+        if @_inject_time_key
+          r[@_inject_time_key] = @_inject_time_formatter.call(time)
+        end
+
+        r
+      end
+
+      def inject_values_to_event_stream(tag, es)
+        return es unless @_inject_enabled
+
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          r = record.dup
+          if @_inject_hostname_key
+            r[@_inject_hostname_key] = @_inject_hostname
+          end
+          if @_inject_worker_id_key
+            r[@_inject_worker_id_key] = @_inject_worker_id
+          end
+          if @_inject_tag_key
+            r[@_inject_tag_key] = tag
+          end
+          if @_inject_time_key
+            r[@_inject_time_key] = @_inject_time_formatter.call(time)
+          end
+          new_es.add(time, r)
+        end
+
+        new_es
+      end
+
+      module InjectParams
+        include Fluent::Configurable
+        config_section :inject, required: false, multi: false, param_name: :inject_config do
+          config_param :hostname_key, :string, default: nil
+          config_param :hostname, :string, default: nil
+          config_param :worker_id_key, :string, default: nil
+          config_param :tag_key, :string, default: nil
+          config_param :time_key, :string, default: nil
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param name, type, opts
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include InjectParams
+      end
+
+      def initialize
+        super
+        @_inject_enabled = false
+        @_inject_hostname_key = nil
+        @_inject_hostname = nil
+        @_inject_worker_id_key = nil
+        @_inject_worker_id = nil
+        @_inject_tag_key = nil
+        @_inject_time_key = nil
+        @_inject_time_formatter = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @inject_config
+          @_inject_hostname_key = @inject_config.hostname_key
+          if @_inject_hostname_key
+            if self.respond_to?(:buffer_config)
+              # Output plugin cannot use "hostname"(specified by @hostname_key),
+              # injected by this plugin helper, in chunk keys.
+              # This plugin helper works in `#format` (in many cases), but modified record
+              # don't have any side effect in chunking of output plugin.
+              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
+                log.error "Use filters to inject hostname to use it in buffer chunking."
+                raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
+              end
+            end
+
+            @_inject_hostname =  @inject_config.hostname
+            unless @_inject_hostname
+              @_inject_hostname = ::Socket.gethostname
+              log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
+            end
+          end
+          @_inject_worker_id_key = @inject_config.worker_id_key
+          if @_inject_worker_id_key
+            @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests
+          end
+          @_inject_tag_key = @inject_config.tag_key
+          @_inject_time_key = @inject_config.time_key
+          if @_inject_time_key
+            @_inject_time_formatter = case @inject_config.time_type
+                                      when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime then ->(time){ time.to_i }
+                                      else
+                                        localtime = @inject_config.localtime && !@inject_config.utc
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
+                                      end
+          else
+            if @inject_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
+          end
+
+          @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key
+        end
+      end
+    end
+  end
+end