dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -0,0 +1,93 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'async'
+require 'async/http'
+require 'async/http/endpoint'
+
+require 'fluent/plugin_helper/http_server/app'
+require 'fluent/plugin_helper/http_server/router'
+require 'fluent/plugin_helper/http_server/methods'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      class Server
+        # @param logger [Logger]
+        # @param default_app [Object] This method must have #call.
+        # @param tls_context [OpenSSL::SSL::SSLContext]
+        def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
+          @addr = addr
+          @port = port
+          @logger = logger
+
+          # TODO: support http2
+          scheme = tls_context ? 'https' : 'http'
+          @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
+          @router = Router.new(default_app)
+          @reactor = Async::Reactor.new(nil, logger: @logger)
+
+          opts = if tls_context
+                   { ssl_context: tls_context }
+                 else
+                   {}
+                 end
+          @server = Async::HTTP::Server.new(App.new(@router, @logger), Async::HTTP::Endpoint.parse(@uri, **opts))
+
+          if block_given?
+            yield(self)
+          end
+        end
+
+        def start(notify = nil)
+          @logger.debug("Start async HTTP server listening #{@uri}")
+          task = @reactor.run do
+            @server.run
+
+            if notify
+              notify.push(:ready)
+            end
+          end
+
+          task.stop
+          @logger.debug('Finished HTTP server')
+        end
+
+        def stop
+          @logger.debug('closing HTTP server')
+
+          if @reactor
+            @reactor.stop
+          end
+        end
+
+        HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
+          define_method(name) do |path, app = nil, &block|
+            unless path.end_with?('/')
+              path << '/'
+            end
+
+            if (block && app) || (!block && !app)
+              raise 'You must specify either app or block in the same time'
+            end
+
+            @router.mount(name, path, app || block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb

@@ -0,0 +1,41 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/cert_option'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      # In order not to expose CertOption's methods unnecessary
+      class SSLContextBuilder
+        include Fluent::PluginHelper::CertOption
+
+        def initialize(log)
+          @log = log
+        end
+
+        # @param config [Fluent::Config::Section] @transport_config
+        def build(config)
+          cert_option_create_context(config.version, config.insecure, config.ciphers, config)
+        end
+
+        private
+
+        attr_reader :log
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+begin
+  require 'async'
+  require 'fluent/plugin_helper/http_server/server'
+rescue LoadError => _
+  require 'fluent/plugin_helper/http_server/compat/server'
+  Fluent::PluginHelper::HttpServer::Server = Fluent::PluginHelper::HttpServer::Compat::Server
+end
+
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/server' # For Server::ServerTransportParams
+require 'fluent/plugin_helper/http_server/ssl_context_builder'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      include Fluent::PluginHelper::Thread
+      include Fluent::Configurable
+
+      # stop     : stop http server and mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill thread
+
+      def self.included(mod)
+        mod.include Fluent::PluginHelper::Server::ServerTransportParams
+      end
+
+      def initialize(*)
+        super
+        @_http_server = nil
+      end
+
+      def create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)
+        logger.warn('this method is deprecated. Use #http_server_create_http_server instead')
+        http_server_create_http_server(title, addr: addr, port: port, logger: logger, default_app: default_app, proto: proto, tls_opts: tls_opts, &block)
+      end
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param addr [String] Listen address
+      # @param port [String] Listen port
+      # @param logger [Logger] logger used in this server
+      # @param default_app [Object] This method must have #call.
+      # @param proto [Symbol] :tls or :tcp
+      # @param tls_opts [Hash] options for TLS.
+      def http_server_create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)
+        unless block_given?
+          raise ArgumentError, 'BUG: callback not specified'
+        end
+
+        if proto == :tls || (@transport_config && @transport_config.protocol == :tls)
+          http_server_create_https_server(title, addr: addr, port: port, logger: logger, default_app: default_app, tls_opts: tls_opts, &block)
+        else
+          @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
+            yield(serv)
+          end
+
+          _block_until_http_server_start do |notify|
+            thread_create(title) do
+              @_http_server.start(notify)
+            end
+          end
+        end
+      end
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param addr [String] Listen address
+      # @param port [String] Listen port
+      # @param logger [Logger] logger used in this server
+      # @param default_app [Object] This method must have #call.
+      # @param tls_opts [Hash] options for TLS.
+      def http_server_create_https_server(title, addr:, port:, logger:, default_app: nil, tls_opts: nil)
+        topt =
+          if tls_opts
+            _http_server_overwrite_config(@transport_config, tls_opts)
+          else
+            @transport_config
+          end
+        ctx = Fluent::PluginHelper::HttpServer::SSLContextBuilder.new($log).build(topt)
+
+        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app, tls_context: ctx) do |serv|
+          yield(serv)
+        end
+
+        _block_until_http_server_start do |notify|
+          thread_create(title) do
+            @_http_server.start(notify)
+          end
+        end
+      end
+
+      def stop
+        if @_http_server
+          @_http_server.stop
+        end
+
+        super
+      end
+
+      private
+
+      def _http_server_overwrite_config(config, opts)
+        conf = config.dup
+        Fluent::PluginHelper::Server::SERVER_TRANSPORT_PARAMS.map(&:to_s).each do |param|
+          if opts.key?(param)
+            conf[param] = opts[param]
+          end
+        end
+        conf
+      end
+
+      # To block until server is ready to listen
+      def _block_until_http_server_start
+        que = Queue.new
+        yield(que)
+        que.pop
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/inject.rb

@@ -0,0 +1,154 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+require 'socket'
+
+module Fluent
+  module PluginHelper
+    module Inject
+      def inject_values_to_record(tag, time, record)
+        return record unless @_inject_enabled
+
+        r = record.dup
+        if @_inject_hostname_key
+          r[@_inject_hostname_key] = @_inject_hostname
+        end
+        if @_inject_worker_id_key
+          r[@_inject_worker_id_key] = @_inject_worker_id
+        end
+        if @_inject_tag_key
+          r[@_inject_tag_key] = tag
+        end
+        if @_inject_time_key
+          r[@_inject_time_key] = @_inject_time_formatter.call(time)
+        end
+
+        r
+      end
+
+      def inject_values_to_event_stream(tag, es)
+        return es unless @_inject_enabled
+
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          r = record.dup
+          if @_inject_hostname_key
+            r[@_inject_hostname_key] = @_inject_hostname
+          end
+          if @_inject_worker_id_key
+            r[@_inject_worker_id_key] = @_inject_worker_id
+          end
+          if @_inject_tag_key
+            r[@_inject_tag_key] = tag
+          end
+          if @_inject_time_key
+            r[@_inject_time_key] = @_inject_time_formatter.call(time)
+          end
+          new_es.add(time, r)
+        end
+
+        new_es
+      end
+
+      module InjectParams
+        include Fluent::Configurable
+        config_section :inject, required: false, multi: false, param_name: :inject_config do
+          config_param :hostname_key, :string, default: nil
+          config_param :hostname, :string, default: nil
+          config_param :worker_id_key, :string, default: nil
+          config_param :tag_key, :string, default: nil
+          config_param :time_key, :string, default: nil
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param(name, type, **opts)
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include InjectParams
+      end
+
+      def initialize
+        super
+        @_inject_enabled = false
+        @_inject_hostname_key = nil
+        @_inject_hostname = nil
+        @_inject_worker_id_key = nil
+        @_inject_worker_id = nil
+        @_inject_tag_key = nil
+        @_inject_time_key = nil
+        @_inject_time_formatter = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @inject_config
+          @_inject_hostname_key = @inject_config.hostname_key
+          if @_inject_hostname_key
+            if self.respond_to?(:buffer_config)
+              # Output plugin cannot use "hostname"(specified by @hostname_key),
+              # injected by this plugin helper, in chunk keys.
+              # This plugin helper works in `#format` (in many cases), but modified record
+              # don't have any side effect in chunking of output plugin.
+              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
+                log.error "Use filters to inject hostname to use it in buffer chunking."
+                raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
+              end
+            end
+
+            @_inject_hostname =  @inject_config.hostname
+            unless @_inject_hostname
+              @_inject_hostname = ::Socket.gethostname
+              log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
+            end
+          end
+          @_inject_worker_id_key = @inject_config.worker_id_key
+          if @_inject_worker_id_key
+            @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests
+          end
+          @_inject_tag_key = @inject_config.tag_key
+          @_inject_time_key = @inject_config.time_key
+          if @_inject_time_key
+            @_inject_time_formatter = case @inject_config.time_type
+                                      when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
+                                      when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
+                                      when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
+                                      when :unixtime then ->(time){ time.to_i }
+                                      else
+                                        localtime = @inject_config.localtime && !@inject_config.utc
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
+                                      end
+          else
+            if @inject_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
+          end
+
+          @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/metrics.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module PluginHelper
+    module Metrics
+      include Fluent::SystemConfig::Mixin
+
+      attr_reader :_metrics # For tests.
+
+      def initialize
+        super
+        @_metrics_started = false
+        @_metrics = {} # usage => metrics_state
+      end
+
+      def configure(conf)
+        super
+
+        @plugin_type_or_id = if self.plugin_id_configured?
+                               self.plugin_id
+                             else
+                               if type = (conf["@type"] || conf["type"])
+                                 "#{type}.#{self.plugin_id}"
+                               else
+                                 "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
+                               end
+                             end
+      end
+
+      def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
+        metrics = if system_config.metrics
+                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
+                  else
+                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
+                  end
+        config = if system_config.metrics
+                   system_config.metrics.corresponding_config_element
+                 else
+                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
+                 end
+        metrics.use_gauge_metric = prefer_gauge
+        metrics.configure(config)
+        # For multi workers environment, cmetrics should be distinguish with static labels.
+        if Fluent::Engine.system_config.workers > 1
+          labels.merge!(worker_id: fluentd_worker_id.to_s)
+        end
+        labels.merge!(plugin: @plugin_type_or_id)
+        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
+
+        @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
+
+        metrics
+      end
+
+      def metrics_operate(method_name, &block)
+        @_metrics.each_pair do |key, m|
+          begin
+            block.call(s) if block_given?
+            m.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
+          end
+        end
+      end
+
+      def start
+        super
+
+        metrics_operate(:start)
+        @_metrics_started = true
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        metrics_operate(:stop)
+      end
+
+      def before_shutdown
+        metrics_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        metrics_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        metrics_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        metrics_operate(:close)
+        super
+      end
+
+      def terminate
+        metrics_operate(:terminate)
+        @_metrics = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/parser.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Parser
+      def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
+        parser = @_parsers[usage]
+        return parser if parser && !type && !conf
+
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
+        parser = Fluent::Plugin.new_parser(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('parse', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('parse', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        parser.configure(config)
+        if @_parsers_started
+          parser.start
+        end
+
+        @_parsers[usage] = parser
+        parser
+      end
+
+      module ParserParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate parser plugin instances
+        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string # config_set_default required for :@type
+        end
+      end
+
+      def self.included(mod)
+        mod.include ParserParams
+      end
+
+      attr_reader :_parsers # for tests
+
+      def initialize
+        super
+        @_parsers_started = false
+        @_parsers = {} # usage => parser
+      end
+
+      def configure(conf)
+        super
+
+        @parser_configs.each do |section|
+          if @_parsers[section.usage]
+            raise Fluent::ConfigError, "duplicated parsers configured: #{section.usage}"
+          end
+          parser = Plugin.new_parser(section[:@type], parent: self)
+          parser.configure(section.corresponding_config_element)
+          @_parsers[section.usage] = parser
+        end
+      end
+
+      def start
+        super
+        @_parsers_started = true
+        @_parsers.each_pair do |usage, parser|
+          parser.start
+        end
+      end
+
+      def parser_operate(method_name, &block)
+        @_parsers.each_pair do |usage, parser|
+          begin
+            parser.__send__(method_name)
+            block.call(parser) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        parser_operate(:stop)
+      end
+
+      def before_shutdown
+        parser_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        parser_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        parser_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        parser_operate(:close)
+        super
+      end
+
+      def terminate
+        parser_operate(:terminate)
+        @_parsers_started = false
+        @_parsers = {}
+        super
+      end
+    end
+  end
+end