fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/storage_local.rb

@@ -0,0 +1,164 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+
+require 'fileutils'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class LocalStorage < Storage
+      Fluent::Plugin.register_storage('local', self)
+
+      DEFAULT_DIR_MODE = 0755
+      DEFAULT_FILE_MODE = 0644
+
+      config_param :path, :string, default: nil
+      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+        v.to_i(8)
+      end
+      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+        v.to_i(8)
+      end
+      config_param :pretty_print, :bool, default: false
+
+      attr_reader :store # for test
+
+      def initialize
+        super
+        @store = {}
+        @multi_workers_available = nil
+      end
+
+      def configure(conf)
+        super
+
+        @on_memory = false
+        if @path
+          if File.exist?(@path) && File.file?(@path)
+            @multi_workers_available = false
+          elsif File.exist?(@path) && File.directory?(@path)
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+            @multi_workers_available = true
+          else # path file/directory doesn't exist
+            if @path.end_with?('.json') # file
+              @multi_workers_available = false
+            else # directory
+              @path = File.join(@path, "worker#{fluentd_worker_id}", "storage.json")
+              @multi_workers_available = true
+            end
+          end
+        elsif root_dir = owner.plugin_root_dir
+          basename = (conf.arg && !conf.arg.empty?) ? "storage.#{conf.arg}.json" : "storage.json"
+          @path = File.join(root_dir, basename)
+          @multi_workers_available = true
+        else
+          if @persistent
+            raise Fluent::ConfigError, "Plugin @id or path for <storage> required when 'persistent' is true"
+          else
+            if @autosave
+              log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            else
+              log.info "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
+            end
+            @on_memory = true
+            @multi_workers_available = true
+          end
+        end
+
+        if !@on_memory
+          dir = File.dirname(@path)
+          FileUtils.mkdir_p(dir, mode: @dir_mode) unless Dir.exist?(dir)
+          if File.exist?(@path)
+            raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
+            begin
+              data = open(@path, 'r:utf-8') { |io| io.read }
+              if data.empty?
+                log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
+                return
+              end
+              data = Yajl::Parser.parse(data)
+              raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
+            rescue => e
+              log.error "failed to read data from plugin storage file", path: @path, error: e
+              raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
+            end
+          else
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.stat(dir).writable?
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "local plugin storage with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def load
+        return if @on_memory
+        return unless File.exist?(@path)
+        begin
+          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json = Yajl::Parser.parse(json_string)
+          unless json.is_a?(Hash)
+            log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
+            log.debug "broken content", content: json_string
+            return
+          end
+          @store = json
+        rescue => e
+          log.error "failed to load data for plugin storage from file", path: @path, error: e
+        end
+      end
+
+      def save
+        return if @on_memory
+        tmp_path = @path + '.tmp'
+        begin
+          json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
+          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
+          File.rename(tmp_path, @path)
+        rescue => e
+          log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
+        end
+      end
+
+      def get(key)
+        @store[key.to_s]
+      end
+
+      def fetch(key, defval)
+        @store.fetch(key.to_s, defval)
+      end
+
+      def put(key, value)
+        @store[key.to_s] = value
+      end
+
+      def delete(key)
+        @store.delete(key.to_s)
+      end
+
+      def update(key, &block)
+        @store[key.to_s] = block.call(@store[key.to_s])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/string_util.rb

@@ -14,21 +14,9 @@
 #    limitations under the License.
 #
 
-require "string-scrub" unless "".respond_to?(:scrub)
+require 'fluent/compat/string_util'
 
 module Fluent
-  module StringUtil
-    def match_regexp(regexp, string)
-      begin
-        return regexp.match(string)
-      rescue ArgumentError => e
-        raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-        $log.info "invalid byte sequence is replaced in `#{string}`"
-        string = string.scrub('?')
-        retry
-      end
-      return true
-    end
-    module_function :match_regexp
-  end
+  # obsolete
+  StringUtil = Fluent::Compat::StringUtil
 end

data/lib/fluent/plugin.rb

@@ -14,169 +14,170 @@
 #    limitations under the License.
 #
 
+require 'fluent/registry'
 require 'fluent/config/error'
 
 module Fluent
-  class PluginClass
-    # This class is refactored using Fluent::Registry at v0.14
+  module Plugin
+    SEARCH_PATHS = []
 
-    def initialize
-      @input = {}
-      @output = {}
-      @filter = {}
-      @buffer = {}
-    end
+    # plugins for fluentd:         fluent/plugin/type_NAME.rb
+    # plugins for fluentd plugins: fluent/plugin/type/NAME.rb
+    #   ex: storage, buffer chunk, ...
 
-    def register_input(type, klass)
-      register_impl('input', @input, type, klass)
-    end
+    # first class plugins (instantiated by Engine)
+    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_',         dir_search_prefix: 'in_')
+    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')
+    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')
 
-    def register_output(type, klass)
-      register_impl('output', @output, type, klass)
-    end
+    # feature plugin: second class plugins (instantiated by Plugins or Helpers)
+    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')
+    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
+    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
+    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
 
-    def register_filter(type, klass)
-      register_impl('filter', @filter, type, klass)
-    end
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
 
-    def register_buffer(type, klass)
-      register_impl('buffer', @buffer, type, klass)
+    def self.register_input(type, klass)
+      register_impl('input', INPUT_REGISTRY, type, klass)
     end
 
-    def register_parser(type, klass)
-      TextParser.register_template(type, klass)
+    def self.register_output(type, klass)
+      register_impl('output', OUTPUT_REGISTRY, type, klass)
     end
 
-    def register_formatter(type, klass)
-      TextFormatter.register_template(type, klass)
+    def self.register_filter(type, klass)
+      register_impl('filter', FILTER_REGISTRY, type, klass)
     end
 
-    def new_input(type)
-      new_impl('input', @input, type)
+    def self.register_buffer(type, klass)
+      register_impl('buffer', BUFFER_REGISTRY, type, klass)
     end
 
-    def new_output(type)
-      new_impl('output', @output, type)
+    def self.register_parser(type, klass_or_proc)
+      if klass_or_proc.is_a?(Regexp)
+        # This usage is not recommended for new API
+        require 'fluent/parser'
+        register_impl('parser', PARSER_REGISTRY, type, Proc.new { Fluent::TextParser::RegexpParser.new(klass_or_proc) })
+      else
+        register_impl('parser', PARSER_REGISTRY, type, klass_or_proc)
+      end
     end
 
-    def new_filter(type)
-      new_impl('filter', @filter, type)
+    def self.register_formatter(type, klass_or_proc)
+      if klass_or_proc.respond_to?(:call) && klass_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+        # This usage is not recommended for new API
+        require 'fluent/formatter'
+        register_impl('formatter', FORMATTER_REGISTRY, type, Proc.new { Fluent::TextFormatter::ProcWrappedFormatter.new(klass_or_proc) })
+      else
+        register_impl('formatter', FORMATTER_REGISTRY, type, klass_or_proc)
+      end
     end
 
-    def new_buffer(type)
-      new_impl('buffer', @buffer, type)
+    def self.register_storage(type, klass)
+      register_impl('storage', STORAGE_REGISTRY, type, klass)
     end
 
-    def new_parser(type)
-      require 'fluent/parser'
-      TextParser.lookup(type)
+    def self.lookup_type_from_class(klass_or_its_name)
+      klass = if klass_or_its_name.is_a? Class
+                klass_or_its_name
+              elsif klass_or_its_name.is_a? String
+                eval(klass_or_its_name) # const_get can't handle qualified klass name (ex: A::B)
+              else
+                raise ArgumentError, "invalid argument type #{klass_or_its_name.class}: #{klass_or_its_name}"
+              end
+      REGISTRIES.reduce(nil){|a, r| a || r.reverse_lookup(klass) }
     end
 
-    def new_formatter(type)
-      require 'fluent/formatter'
-      TextFormatter.lookup(type)
+    def self.add_plugin_dir(dir)
+      REGISTRIES.each do |r|
+        r.paths.push(dir)
+      end
+      nil
     end
 
-    def load_plugins
-      dir = File.join(File.dirname(__FILE__), "plugin")
-      load_plugin_dir(dir)
+    def self.new_input(type)
+      new_impl('input', INPUT_REGISTRY, type)
     end
 
-    def load_plugin_dir(dir)
-      dir = File.expand_path(dir)
-      Dir.entries(dir).sort.each {|fname|
-        if fname =~ /\.rb$/
-          require File.join(dir, fname)
-        end
-      }
-      nil
+    def self.new_output(type)
+      new_impl('output', OUTPUT_REGISTRY, type)
     end
 
-    def load_plugin(type, name)
-      try_load_plugin(name, type)
+    def self.new_filter(type)
+      new_impl('filter', FILTER_REGISTRY, type)
     end
 
-    def lookup_name_from_class(klass_or_str)
-      klass = if klass_or_str.class == String
-                eval(klass_or_str) # const_get can't handle A::B
-              else
-                klass_or_str
-              end
+    def self.new_buffer(type, parent: nil)
+      new_impl('buffer', BUFFER_REGISTRY, type, parent)
+    end
 
-      @input.each { |name, plugin|
-        return name if plugin == klass
-      }
-      @output.each { |name, plugin|
-        return name if plugin == klass
-      }
-      @filter.each { |name, plugin|
-        return name if plugin == klass
-      }
+    def self.new_parser(type, parent: nil)
+      if type[0] == '/' && type[-1] == '/'
+        # This usage is not recommended for new API... create RegexpParser directly
+        require 'fluent/parser'
+        impl = Fluent::TextParser.lookup(type)
+        impl.extend FeatureAvailabilityChecker
+        impl
+      else
+        new_impl('parser', PARSER_REGISTRY, type, parent)
+      end
+    end
 
-      nil
+    def self.new_formatter(type, parent: nil)
+      new_impl('formatter', FORMATTER_REGISTRY, type, parent)
     end
 
-    private
-    def register_impl(name, map, type, klass)
-      map[type] = klass
-      $log.trace { "registered #{name} plugin '#{type}'" }
-      nil
+    def self.new_storage(type, parent: nil)
+      new_impl('storage', STORAGE_REGISTRY, type, parent)
     end
 
-    def new_impl(name, map, type)
-      if klass = map[type]
-        return klass.new
-      end
-      try_load_plugin(name, type)
-      if klass = map[type]
-        return klass.new
-      end
-      raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"
-    end
-
-    def try_load_plugin(name, type)
-      case name
-      when 'input'
-        path = "fluent/plugin/in_#{type}"
-      when 'output'
-        path = "fluent/plugin/out_#{type}"
-      when 'filter'
-        path = "fluent/plugin/filter_#{type}"
-      when 'buffer'
-        path = "fluent/plugin/buf_#{type}"
-      else
-        return
+    def self.register_impl(kind, registry, type, value)
+      if !value.is_a?(Class) && !value.respond_to?(:call)
+        raise Fluent::ConfigError, "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class, or callable."
       end
+      registry.register(type, value)
+      $log.trace "registered #{kind} plugin '#{type}'" if defined?($log)
+      nil
+    end
 
-      # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map {|lp|
-        lpath = File.join(lp, "#{path}.rb")
-        File.exist?(lpath) ? lpath : nil
-      }.compact
-      unless files.empty?
-        # prefer newer version
-        require File.expand_path(files.sort.last)
-        return
+    def self.new_impl(kind, registry, type, parent=nil)
+      # "'type' not found" is handled by registry
+      obj = registry.lookup(type)
+      impl = case
+             when obj.is_a?(Class)
+               obj.new
+             when obj.respond_to?(:call) && obj.arity == 0
+               obj.call
+             else
+               raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)."
+             end
+      if parent && impl.respond_to?("owner=")
+        impl.owner = parent
       end
-
-      # search gems
-      specs = Gem::Specification.find_all { |spec|
-        spec.contains_requirable_file? path
-      }
-
-      # prefer newer version
-      specs = specs.sort_by { |spec| spec.version }
-      if spec = specs.last
-        spec.require_paths.each { |lib|
-          file = "#{spec.full_gem_path}/#{lib}/#{path}"
-          if File.exist?("#{file}.rb")
-            require file
-            return
-          end
-        }
+      impl.extend FeatureAvailabilityChecker
+      impl
+    end
+
+    module FeatureAvailabilityChecker
+      def configure(conf)
+        super
+
+        # extend plugin instance by this module
+        # to run this check after all #configure methods of plugins and plugin helpers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                    owner.system_config
+                  elsif self.respond_to?(:system_config)
+                    self.system_config
+                  else
+                    nil
+                  end
+
+        if sysconf && sysconf.workers > 1 && !self.multi_workers_ready?
+          type = Fluent::Plugin.lookup_type_from_class(self.class)
+          raise Fluent::ConfigError, "Plugin '#{type}' does not support multi workers configuration (#{self.class})"
+        end
       end
     end
   end
-
-  Plugin = PluginClass.new
 end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -0,0 +1,178 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'openssl'
+require 'socket'
+
+# this module is only for Socket/Server plugin helpers
+module Fluent
+  module PluginHelper
+    module CertOption
+      def cert_option_create_context(version, insecure, ciphers, conf)
+        cert, key, extra = cert_option_server_validate!(conf)
+
+        ctx = OpenSSL::SSL::SSLContext.new(version)
+        unless insecure
+          # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+          # https://bugs.ruby-lang.org/issues/9424
+          ctx.set_params({})
+
+          ctx.ciphers = ciphers
+        end
+
+        if conf.client_cert_auth
+            ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+        end
+
+        ctx.ca_file = conf.ca_path
+        ctx.cert = cert
+        ctx.key = key
+        if extra && !extra.empty?
+          ctx.extra_chain_cert = extra
+        end
+
+        ctx
+      end
+
+      def cert_option_server_validate!(conf)
+        case
+        when conf.cert_path
+          raise Fluent::ConfigError, "private_key_path is required when cert_path is specified" unless conf.private_key_path
+          log.warn "For security reason, setting private_key_passphrase is recommended when cert_path is specified" unless conf.private_key_passphrase
+          cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)
+
+        when conf.ca_cert_path
+          raise Fluent::ConfigError, "ca_private_key_path is required when ca_cert_path is specified" unless conf.ca_private_key_path
+          log.warn "For security reason, setting ca_private_key_passphrase is recommended when ca_cert_path is specified" unless conf.ca_private_key_passphrase
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_by_ca(
+            conf.ca_cert_path,
+            conf.ca_private_key_path,
+            conf.ca_private_key_passphrase,
+            generate_opts
+          )
+
+        when conf.insecure
+          log.warn "insecure TLS communication server is configured (using 'insecure' mode)"
+          generate_opts = cert_option_cert_generation_opts_from_conf(conf)
+          cert_option_generate_server_pair_self_signed(generate_opts)
+
+        else
+          raise Fluent::ConfigError, "no valid cert options configured. specify either 'cert_path', 'ca_cert_path' or 'insecure'"
+        end
+      end
+
+      def cert_option_load(cert_path, private_key_path, private_key_passphrase)
+        key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase)
+        certs = cert_option_certificates_from_file(cert_path)
+        cert = certs.shift
+        return cert, key, certs
+      end
+
+      def cert_option_cert_generation_opts_from_conf(conf)
+        {
+          private_key_length: conf.generate_private_key_length,
+          country: conf.generate_cert_country,
+          state: conf.generate_cert_state,
+          locality: conf.generate_cert_locality,
+          common_name: conf.generate_cert_common_name || ::Socket.gethostname,
+          expiration: conf.generate_cert_expiration,
+          digest: conf.generate_cert_digest,
+        }
+      end
+
+      def cert_option_generate_pair(opts, issuer = nil)
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        issuer ||= subject
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + opts[:expiration]
+        cert.public_key = key
+        cert.version = 2
+        cert.serial = rand(2**(8*10))
+        cert.issuer = issuer
+        cert.subject  = subject
+
+        return cert, key
+      end
+
+      def cert_option_add_extensions(cert, extensions)
+        ef = OpenSSL::X509::ExtensionFactory.new
+        extensions.each do |ext|
+          oid, value = ext
+          cert.add_extension ef.create_extension(oid, value)
+        end
+      end
+
+      def cert_option_generate_ca_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:TRUE']
+        ])
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key
+      end
+
+      def cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, generate_opts)
+        ca_key = OpenSSL::PKey::read(File.read(ca_key_path), ca_key_passphrase)
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(ca_cert_path))
+        cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server'],
+          ['keyUsage', 'digitalSignature,keyEncipherment'],
+          ['extendedKeyUsage', 'serverAuth']
+        ])
+
+        cert.sign(ca_key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_generate_server_pair_self_signed(generate_opts)
+        cert, key = cert_option_generate_pair(generate_opts)
+        raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
+
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server']
+        ])
+
+        cert.sign(key, generate_opts[:digest].to_s)
+        return cert, key, nil
+      end
+
+      def cert_option_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n?', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        list
+      end
+    end
+  end
+end