dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin.rb

@@ -0,0 +1,206 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/registry'
+require 'fluent/config/error'
+
+module Fluent
+  module Plugin
+    SEARCH_PATHS = []
+
+    # plugins for fluentd:         fluent/plugin/type_NAME.rb
+    # plugins for fluentd plugins: fluent/plugin/type/NAME.rb
+    #   ex: storage, buffer chunk, ...
+
+    # first class plugins (instantiated by Engine)
+    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_',         dir_search_prefix: 'in_')
+    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')
+    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')
+
+    # feature plugin: second class plugins (instantiated by Plugins or Helpers)
+    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')
+    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
+    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
+    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
+    SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
+    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')
+
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
+
+    def self.register_input(type, klass)
+      register_impl('input', INPUT_REGISTRY, type, klass)
+    end
+
+    def self.register_output(type, klass)
+      register_impl('output', OUTPUT_REGISTRY, type, klass)
+    end
+
+    def self.register_filter(type, klass)
+      register_impl('filter', FILTER_REGISTRY, type, klass)
+    end
+
+    def self.register_buffer(type, klass)
+      register_impl('buffer', BUFFER_REGISTRY, type, klass)
+    end
+
+    def self.register_sd(type, klass)
+      register_impl('sd', SD_REGISTRY, type, klass)
+    end
+
+    def self.register_metrics(type, klass)
+      register_impl('metrics', METRICS_REGISTRY, type, klass)
+    end
+
+    def self.register_parser(type, klass_or_proc)
+      if klass_or_proc.is_a?(Regexp)
+        # This usage is not recommended for new API
+        require 'fluent/parser'
+        register_impl('parser', PARSER_REGISTRY, type, Proc.new { Fluent::TextParser::RegexpParser.new(klass_or_proc) })
+      else
+        register_impl('parser', PARSER_REGISTRY, type, klass_or_proc)
+      end
+    end
+
+    def self.register_formatter(type, klass_or_proc)
+      if klass_or_proc.respond_to?(:call) && klass_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+        # This usage is not recommended for new API
+        require 'fluent/formatter'
+        register_impl('formatter', FORMATTER_REGISTRY, type, Proc.new { Fluent::TextFormatter::ProcWrappedFormatter.new(klass_or_proc) })
+      else
+        register_impl('formatter', FORMATTER_REGISTRY, type, klass_or_proc)
+      end
+    end
+
+    def self.register_storage(type, klass)
+      register_impl('storage', STORAGE_REGISTRY, type, klass)
+    end
+
+    def self.lookup_type_from_class(klass_or_its_name)
+      klass = if klass_or_its_name.is_a? Class
+                klass_or_its_name
+              elsif klass_or_its_name.is_a? String
+                eval(klass_or_its_name) # const_get can't handle qualified klass name (ex: A::B)
+              else
+                raise ArgumentError, "invalid argument type #{klass_or_its_name.class}: #{klass_or_its_name}"
+              end
+      REGISTRIES.reduce(nil){|a, r| a || r.reverse_lookup(klass) }
+    end
+
+    def self.add_plugin_dir(dir)
+      REGISTRIES.each do |r|
+        r.paths.push(dir)
+      end
+      nil
+    end
+
+    def self.new_input(type)
+      new_impl('input', INPUT_REGISTRY, type)
+    end
+
+    def self.new_output(type)
+      new_impl('output', OUTPUT_REGISTRY, type)
+    end
+
+    def self.new_filter(type)
+      new_impl('filter', FILTER_REGISTRY, type)
+    end
+
+    def self.new_buffer(type, parent: nil)
+      new_impl('buffer', BUFFER_REGISTRY, type, parent)
+    end
+
+    def self.new_sd(type, parent: nil)
+      new_impl('sd', SD_REGISTRY, type, parent)
+    end
+
+    def self.new_metrics(type, parent: nil)
+      new_impl('metrics', METRICS_REGISTRY, type, parent)
+    end
+
+    class << self
+      # This should be defined for fluent-plugin-config-formatter type arguments.
+      alias_method :new_service_discovery, :new_sd
+    end
+
+    def self.new_parser(type, parent: nil)
+      if type[0] == '/' && type[-1] == '/'
+        # This usage is not recommended for new API... create RegexpParser directly
+        require 'fluent/parser'
+        impl = Fluent::TextParser.lookup(type)
+        impl.extend FeatureAvailabilityChecker
+        impl
+      else
+        new_impl('parser', PARSER_REGISTRY, type, parent)
+      end
+    end
+
+    def self.new_formatter(type, parent: nil)
+      new_impl('formatter', FORMATTER_REGISTRY, type, parent)
+    end
+
+    def self.new_storage(type, parent: nil)
+      new_impl('storage', STORAGE_REGISTRY, type, parent)
+    end
+
+    def self.register_impl(kind, registry, type, value)
+      if !value.is_a?(Class) && !value.respond_to?(:call)
+        raise Fluent::ConfigError, "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class, or callable."
+      end
+      registry.register(type, value)
+      $log.trace "registered #{kind} plugin '#{type}'" if defined?($log)
+      nil
+    end
+
+    def self.new_impl(kind, registry, type, parent=nil)
+      # "'type' not found" is handled by registry
+      obj = registry.lookup(type)
+      impl = case
+             when obj.is_a?(Class)
+               obj.new
+             when obj.respond_to?(:call) && obj.arity == 0
+               obj.call
+             else
+               raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)."
+             end
+      if parent && impl.respond_to?("owner=")
+        impl.owner = parent
+      end
+      impl.extend FeatureAvailabilityChecker
+      impl
+    end
+
+    module FeatureAvailabilityChecker
+      def configure(conf)
+        super
+
+        # extend plugin instance by this module
+        # to run this check after all #configure methods of plugins and plugin helpers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                    owner.system_config
+                  elsif self.respond_to?(:system_config)
+                    self.system_config
+                  else
+                    nil
+                  end
+
+        if sysconf && sysconf.workers > 1 && !self.multi_workers_ready?
+          type = Fluent::Plugin.lookup_type_from_class(self.class)
+          raise Fluent::ConfigError, "Plugin '#{type}' does not support multi workers configuration (#{self.class})"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -0,0 +1,191 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'openssl'
+require 'socket'
+
+require 'fluent/tls'
+
+# this module is only for Socket/Server/HttpServer plugin helpers
+module Fluent
+  module PluginHelper
+    module CertOption
+      def cert_option_create_context(version, insecure, ciphers, conf)
+        cert, key, extra = cert_option_server_validate!(conf)
+
+        ctx = OpenSSL::SSL::SSLContext.new
+        # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+        # https://bugs.ruby-lang.org/issues/9424
+        ctx.set_params({}) unless insecure
+
+        if conf.client_cert_auth
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+        end
+
+        ctx.ca_file = conf.ca_path
+        ctx.cert = cert
+        ctx.key = key
+        if extra && !extra.empty?
+          ctx.extra_chain_cert = extra
+        end
+        if conf.cert_verifier
+          sandbox = Class.new
+          ctx.verify_callback = if File.exist?(conf.cert_verifier)
+                                  verifier = File.read(conf.cert_verifier)
+                                  sandbox.instance_eval(verifier, File.basename(conf.cert_verifier))
+                                else
+                                  sandbox.instance_eval(conf.cert_verifier)
+                                end
+        end
+
+        Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
+        ctx.ciphers = ciphers unless insecure
+
+        ctx
+      end
+
+      def cert_option_server_validate!(conf)
+        case
+        when conf.cert_path
+          raise Fluent::ConfigError, "private_key_path is required when cert_path is specified" unless conf.private_key_path
+          log.warn "For security reason, setting private_key_passphrase is recommended when cert_path is specified" unless conf.private_key_passphrase
+          cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)
+
+        when conf.ca_cert_path
+          raise Fluent::ConfigError, "ca_private_key_path is required when ca_cert_path is specified" unless conf.ca_private_key_path
+          log.warn "For security reason, setting ca_private_key_passphrase is recommended when ca_cert_path is specified" unless conf.ca_private_key_passphrase
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_by_ca(
+            conf.ca_cert_path,
+            conf.ca_private_key_path,
+            conf.ca_private_key_passphrase,
+            generate_opts
+          )
+
+        when conf.insecure
+          log.warn "insecure TLS communication server is configured (using 'insecure' mode)"
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_self_signed(generate_opts)
+
+        else
+          raise Fluent::ConfigError, "no valid cert options configured. specify either 'cert_path', 'ca_cert_path' or 'insecure'"
+        end
+      end
+
+      def cert_option_load(cert_path, private_key_path, private_key_passphrase)
+        key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase)
+        certs = cert_option_certificates_from_file(cert_path)
+        cert = certs.shift
+        return cert, key, certs
+      end
+
+      def cert_option_cert_generation_opts_from_conf(conf)
+        {
+          private_key_length: conf.generate_private_key_length,
+          country: conf.generate_cert_country,
+          state: conf.generate_cert_state,
+          locality: conf.generate_cert_locality,
+          common_name: conf.generate_cert_common_name || ::Socket.gethostname,
+          expiration: conf.generate_cert_expiration,
+          digest: conf.generate_cert_digest,
+        }
+      end
+
+      def cert_option_generate_pair(opts, issuer = nil)
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        issuer ||= subject
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + opts[:expiration]
+        cert.public_key = key
+        cert.version = 2
+        cert.serial = rand(2**(8*10))
+        cert.issuer = issuer
+        cert.subject  = subject
+
+        return cert, key
+      end
+
+      def cert_option_add_extensions(cert, extensions)
+        ef = OpenSSL::X509::ExtensionFactory.new
+        extensions.each do |ext|
+          oid, value = ext
+          cert.add_extension ef.create_extension(oid, value)
+        end
+      end
+
+      def cert_option_generate_ca_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:TRUE']
+        ])
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key
+      end
+
+      def cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, generate_opts)
+        ca_key = OpenSSL::PKey::read(File.read(ca_key_path), ca_key_passphrase)
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(ca_cert_path))
+        cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server'],
+          ['keyUsage', 'digitalSignature,keyEncipherment'],
+          ['extendedKeyUsage', 'serverAuth']
+        ])
+
+        cert.sign(ca_key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_generate_server_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server']
+        ])
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        if list.length == 0
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
+        end
+        list
+      end
+    end
+  end
+end