dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin_helper/extract.rb

@@ -0,0 +1,104 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Extract
+      def extract_tag_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_tag_key && record.has_key?(@_extract_tag_key)
+          v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)
+          return v.to_s
+        end
+
+        nil
+      end
+
+      def extract_time_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_time_key && record.has_key?(@_extract_time_key)
+          v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)
+          return @_extract_time_parser.call(v)
+        end
+
+        nil
+      end
+
+      module ExtractParams
+        include Fluent::Configurable
+        config_section :extract, required: false, multi: false, param_name: :extract_config do
+          config_param :tag_key, :string, default: nil
+          config_param :keep_tag_key, :bool, default: false
+          config_param :time_key, :string, default: nil
+          config_param :keep_time_key, :bool, default: false
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param(name, type, **opts)
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include ExtractParams
+      end
+
+      def initialize
+        super
+        @_extract_enabled = false
+        @_extract_tag_key = nil
+        @_extract_keep_tag_key = nil
+        @_extract_time_key = nil
+        @_extract_keep_time_key = nil
+        @_extract_time_parser = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @extract_config
+          @_extract_tag_key = @extract_config.tag_key
+          @_extract_keep_tag_key = @extract_config.keep_tag_key
+          @_extract_time_key = @extract_config.time_key
+          if @_extract_time_key
+            @_extract_keep_time_key = @extract_config.keep_time_key
+            @_extract_time_parser = case @extract_config.time_type
+                                    when :float then Fluent::NumericTimeParser.new(:float)
+                                    when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
+                                    else
+                                      localtime = @extract_config.localtime && !@extract_config.utc
+                                      Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
+                                    end
+          else
+            if @extract_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
+          end
+
+          @_extract_enabled = @_extract_tag_key || @_extract_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/formatter.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/formatter'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Formatter
+      def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
+        formatter = @_formatters[usage]
+        return formatter if formatter && !type && !conf
+
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
+        formatter = Fluent::Plugin.new_formatter(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('format', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('format', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        formatter.configure(config)
+        if @_formatters_started
+          formatter.start
+        end
+
+        @_formatters[usage] = formatter
+        formatter
+      end
+
+      module FormatterParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate formatter plugin instances
+        config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string # config_set_default required for :@type
+        end
+      end
+
+      def self.included(mod)
+        mod.include FormatterParams
+      end
+
+      attr_reader :_formatters # for tests
+
+      def initialize
+        super
+        @_formatters_started = false
+        @_formatters = {} # usage => formatter
+      end
+
+      def configure(conf)
+        super
+
+        @formatter_configs.each do |section|
+          if @_formatters[section.usage]
+            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
+          end
+          formatter = Plugin.new_formatter(section[:@type], parent: self)
+          formatter.configure(section.corresponding_config_element)
+          @_formatters[section.usage] = formatter
+        end
+      end
+
+      def start
+        super
+        @_formatters_started = true
+        @_formatters.each_pair do |usage, formatter|
+          formatter.start
+        end
+      end
+
+      def formatter_operate(method_name, &block)
+        @_formatters.each_pair do |usage, formatter|
+          begin
+            formatter.__send__(method_name)
+            block.call(formatter) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        formatter_operate(:stop)
+      end
+
+      def before_shutdown
+        formatter_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        formatter_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        formatter_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        formatter_operate(:close)
+        super
+      end
+
+      def terminate
+        formatter_operate(:terminate)
+        @_formatters_started = false
+        @_formatters = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/app.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/request'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class App
+        def initialize(router, logger)
+          @logger = logger
+          @router = router
+        end
+
+        # Required method by async-http
+        def call(request)
+          method = request.method
+          resp =
+            case method
+            when HttpServer::Methods::GET
+              get(request)
+            when HttpServer::Methods::HEAD
+              head(request)
+            when HttpServer::Methods::POST
+              post(request)
+            when HttpServer::Methods::PATCH
+              patch(request)
+            when HttpServer::Methods::PUT
+              put(request)
+            when HttpServer::Methods::DELETE
+              delete(request)
+            when HttpServer::Methods::OPTIONS
+              options(request)
+            when HttpServer::Methods::CONNECT
+              connect(request)
+            when HttpServer::Methods::TRACE
+              trace(request)
+            else
+              raise "Unknown method #{method}"
+            end
+          Protocol::HTTP::Response[*resp]
+        rescue => e
+          @logger.error(e)
+          Protocol::HTTP::Response[500, { 'Content-Type' => 'text/plain' }, 'Internal Server Error']
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |request|
+            req = Request.new(request)
+
+            path = req.path
+            canonical_path =
+              if path.size >= 2 && !path.end_with?('/')
+                "#{path}/"
+              else
+                path
+              end
+            @router.route!(name, canonical_path, req)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/plugin_helper/http_server/compat/webrick_handler'
+require 'fluent/plugin_helper/http_server/compat/ssl_context_extractor'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class Server
+          # @param logger [Logger]
+          # @param default_app [Object] ignored option. only for compat
+          # @param tls_context [OpenSSL::SSL::SSLContext]
+          def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
+            @addr = addr
+            @port = port
+            @logger = logger
+
+            config = {
+              BindAddress: @addr,
+              Port: @port,
+              Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+              AccessLog: [],
+            }
+            if tls_context
+              require 'webrick/https'
+              @logger.warn('Webrick ignores given TLS version')
+              tls_opt = Fluent::PluginHelper::HttpServer::Compat::SSLContextExtractor.extract(tls_context)
+              config = tls_opt.merge(**config)
+            end
+
+            @server = WEBrick::HTTPServer.new(config)
+
+            # @example ["/example.json", :get, handler object]
+            @methods = []
+
+            if block_given?
+              yield(self)
+            end
+          end
+
+          def start(notify = nil)
+            build_handler
+            notify.push(:ready)
+            @logger.debug('Start webrick HTTP server listening')
+            @server.start
+          end
+
+          def stop
+            @server.shutdown
+            @server.stop
+          end
+
+          HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+            define_method(name) do |path, app = nil, &block|
+              if (block && app) || (!block && !app)
+                raise 'You must specify either app or block in the same time'
+              end
+
+              # Do not build a handler class here to able to handle multiple methods for single path.
+              @methods << [path, name, app || block]
+            end
+          end
+
+          private
+
+          def build_handler
+            @methods.group_by(&:first).each do |(path, rest)|
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
+              @server.mount(path, klass)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        # This class converts OpenSSL::SSL::SSLContext to Webrick SSL Config because webrick does not have interface to pass OpenSSL::SSL::SSLContext directory
+        # https://github.com/ruby/webrick/blob/v1.6.0/lib/webrick/ssl.rb#L67-L88
+        class SSLContextExtractor
+
+          #
+          # memo: https://github.com/ruby/webrick/blob/v1.6.0/lib/webrick/ssl.rb#L180-L205
+          # @param ctx [OpenSSL::SSL::SSLContext]
+          def self.extract(ctx)
+            {
+              SSLEnable: true,
+              SSLPrivateKey: ctx.key,
+              SSLCertificate: ctx.cert,
+              SSLClientCA: ctx.client_ca,
+              SSLExtraChainCert: ctx.extra_chain_cert,
+              SSLCACertificateFile: ctx.ca_file,
+              SSLCACertificatePath: ctx.ca_path,
+              SSLCertificateStore: ctx.cert_store,
+              SSLTmpDhCallback: ctx.tmp_dh_callback,
+              SSLVerifyClient: ctx.verify_mode,
+              SSLVerifyDepth: ctx.verify_depth,
+              SSLVerifyCallback: ctx.verify_callback,
+              SSLServerNameCallback: ctx.servername_cb,
+              SSLTimeout: ctx.timeout,
+              SSLOptions: ctx.options,
+              SSLCiphers: ctx.ciphers,
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'webrick'
+require 'json'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        class WebrickHandler
+          # **opt is enough. but I wrote a signature explicitly for readability
+          def self.build(get: nil, head: nil, post: nil, put: nil, patch: nil, delete: nil, connect: nil, options: nil, trace: nil)
+            opt = { get: get, head: head, post: post, put: put, patch: patch, delete: delete, connect: connect, options: options, trace: trace }
+
+            Class.new(WEBrick::HTTPServlet::AbstractServlet) do
+              HttpServer::Methods::ALL.each do |name|
+                define_method("do_#{name}") do |request, response|
+                  code, headers, body =
+                                 if request.path_info != ''
+                                   render_json(404, 'message' => 'Not found')
+                                 else
+                                   begin
+                                     opt[name.downcase.to_sym].call(request)
+                                   rescue => _
+                                     render_json(500, 'message' => 'Something went wrong')
+                                   end
+                                 end
+
+                  response.status = code
+                  headers.each { |k, v| response[k] = v }
+                  response.body = body
+                end
+              end
+
+              def render_json(code, obj)
+                [code, { 'Content-Type' => 'application/json' }, obj.to_json]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/methods.rb

@@ -0,0 +1,35 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Methods
+        GET = 'GET'.freeze
+        HEAD = 'HEAD'.freeze
+        POST = 'POST'.freeze
+        PUT = 'PUT'.freeze
+        PATCH = 'PATCH'.freeze
+        DELETE = 'DELETE'.freeze
+        OPTIONS = 'OPTIONS'.freeze
+        CONNECT = 'CONNECT'.freeze
+        TRACE = 'TRACE'.freeze
+
+        ALL = [GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS, TRACE].freeze
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/request.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Request
+        attr_reader :path, :query_string
+
+        def initialize(request)
+          @request = request
+          path = request.path
+          @path, @query_string = path.split('?', 2)
+        end
+
+        def query
+          @query_string && CGI.parse(@query_string)
+        end
+
+        def body
+          @request.body && @request.body.read
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/router.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async/http/protocol'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Router
+        class NotFoundApp
+          def self.call(req)
+            [404, { 'Content-Type' => 'text/plain' }, "404 Not Found\n"]
+          end
+        end
+
+        def initialize(default_app = nil)
+          @router = { get: {}, head: {}, post: {}, put: {}, patch: {}, delete: {}, connect: {}, options: {}, trace: {} }
+          @default_app = default_app || NotFoundApp
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param app [Object]
+        def mount(method, path, app)
+          if @router[method].include?(path)
+            raise "#{path} is already mounted"
+          end
+
+          @router[method][path] = app
+        end
+
+        # @param method [Symbol]
+        # @param path [String]
+        # @param request [Fluent::PluginHelper::HttpServer::Request]
+        def route!(method, path, request)
+          @router.fetch(method).fetch(path, @default_app).call(request)
+        end
+      end
+    end
+  end
+end