fluentd 0.12.40 → 0.14.0

data/lib/fluent/plugin.rb

@@ -14,169 +14,148 @@
 #    limitations under the License.
 #
 
+require 'fluent/registry'
 require 'fluent/config/error'
 
 module Fluent
-  class PluginClass
-    # This class is refactored using Fluent::Registry at v0.14
+  module Plugin
+    SEARCH_PATHS = []
 
-    def initialize
-      @input = {}
-      @output = {}
-      @filter = {}
-      @buffer = {}
-    end
+    # plugins for fluentd:         fluent/plugin/type_NAME.rb
+    # plugins for fluentd plugins: fluent/plugin/type/NAME.rb
+    #   ex: storage, buffer chunk, ...
 
-    def register_input(type, klass)
-      register_impl('input', @input, type, klass)
-    end
+    # first class plugins (instantiated by Engine)
+    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_')
+    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_')
+    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_')
 
-    def register_output(type, klass)
-      register_impl('output', @output, type, klass)
-    end
+    # feature plugin: second class plugins (instanciated by Plugins or Helpers)
+    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_')
+    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_')
+    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_')
+    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_')
 
-    def register_filter(type, klass)
-      register_impl('filter', @filter, type, klass)
-    end
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
 
-    def register_buffer(type, klass)
-      register_impl('buffer', @buffer, type, klass)
+    def self.register_input(type, klass)
+      register_impl('input', INPUT_REGISTRY, type, klass)
     end
 
-    def register_parser(type, klass)
-      TextParser.register_template(type, klass)
+    def self.register_output(type, klass)
+      register_impl('output', OUTPUT_REGISTRY, type, klass)
     end
 
-    def register_formatter(type, klass)
-      TextFormatter.register_template(type, klass)
+    def self.register_filter(type, klass)
+      register_impl('filter', FILTER_REGISTRY, type, klass)
     end
 
-    def new_input(type)
-      new_impl('input', @input, type)
+    def self.register_buffer(type, klass)
+      register_impl('buffer', BUFFER_REGISTRY, type, klass)
     end
 
-    def new_output(type)
-      new_impl('output', @output, type)
+    def self.register_parser(type, klass_or_proc)
+      if klass_or_proc.is_a?(Regexp)
+        # This usage is not recommended for new API
+        require 'fluent/parser'
+        register_impl('parser', PARSER_REGISTRY, type, Proc.new { Fluent::TextParser::RegexpParser.new(klass_or_proc) })
+      else
+        register_impl('parser', PARSER_REGISTRY, type, klass_or_proc)
+      end
     end
 
-    def new_filter(type)
-      new_impl('filter', @filter, type)
+    def self.register_formatter(type, klass_or_proc)
+      if klass_or_proc.respond_to?(:call) && klass_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+        # This usage is not recommended for new API
+        require 'fluent/formatter'
+        register_impl('formatter', FORMATTER_REGISTRY, type, Proc.new { Fluent::TextFormatter::ProcWrappedFormatter.new(klass_or_proc) })
+      else
+        register_impl('formatter', FORMATTER_REGISTRY, type, klass_or_proc)
+      end
     end
 
-    def new_buffer(type)
-      new_impl('buffer', @buffer, type)
+    def self.register_storage(type, klass)
+      register_impl('storage', STORAGE_REGISTRY, type, klass)
     end
 
-    def new_parser(type)
-      require 'fluent/parser'
-      TextParser.lookup(type)
+    def self.lookup_type_from_class(klass_or_its_name)
+      klass = if klass_or_its_name.is_a? Class
+                klass_or_its_name
+              elsif klass_or_its_name.is_a? String
+                eval(klass_or_its_name) # const_get can't handle qualified klass name (ex: A::B)
+              else
+                raise ArgumentError, "invalid argument type #{klass_or_its_name.class}: #{klass_or_its_name}"
+              end
+      REGISTRIES.reduce(nil){|a, r| a || r.reverse_lookup(klass) }
     end
 
-    def new_formatter(type)
-      require 'fluent/formatter'
-      TextFormatter.lookup(type)
+    def self.add_plugin_dir(dir)
+      REGISTRIES.each do |r|
+        r.paths.push(dir)
+      end
+      nil
     end
 
-    def load_plugins
-      dir = File.join(File.dirname(__FILE__), "plugin")
-      load_plugin_dir(dir)
+    def self.new_input(type)
+      new_impl('input', INPUT_REGISTRY, type)
     end
 
-    def load_plugin_dir(dir)
-      dir = File.expand_path(dir)
-      Dir.entries(dir).sort.each {|fname|
-        if fname =~ /\.rb$/
-          require File.join(dir, fname)
-        end
-      }
-      nil
+    def self.new_output(type)
+      new_impl('output', OUTPUT_REGISTRY, type)
     end
 
-    def load_plugin(type, name)
-      try_load_plugin(name, type)
+    def self.new_filter(type)
+      new_impl('filter', FILTER_REGISTRY, type)
     end
 
-    def lookup_name_from_class(klass_or_str)
-      klass = if klass_or_str.class == String
-                eval(klass_or_str) # const_get can't handle A::B
-              else
-                klass_or_str
-              end
+    def self.new_buffer(type, parent: nil)
+      new_impl('buffer', BUFFER_REGISTRY, type, parent)
+    end
 
-      @input.each { |name, plugin|
-        return name if plugin == klass
-      }
-      @output.each { |name, plugin|
-        return name if plugin == klass
-      }
-      @filter.each { |name, plugin|
-        return name if plugin == klass
-      }
+    def self.new_parser(type, parent: nil)
+      require 'fluent/parser'
 
-      nil
+      if type[0] == '/' && type[-1] == '/'
+        # This usage is not recommended for new API... create RegexpParser directly
+        require 'fluent/parser'
+        Fluent::TextParser.lookup(type)
+      else
+        new_impl('parser', PARSER_REGISTRY, type, parent)
+      end
     end
 
-    private
-    def register_impl(name, map, type, klass)
-      map[type] = klass
-      $log.trace { "registered #{name} plugin '#{type}'" }
-      nil
+    def self.new_formatter(type, parent: nil)
+      new_impl('formatter', FORMATTER_REGISTRY, type, parent)
     end
 
-    def new_impl(name, map, type)
-      if klass = map[type]
-        return klass.new
-      end
-      try_load_plugin(name, type)
-      if klass = map[type]
-        return klass.new
-      end
-      raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"
-    end
-
-    def try_load_plugin(name, type)
-      case name
-      when 'input'
-        path = "fluent/plugin/in_#{type}"
-      when 'output'
-        path = "fluent/plugin/out_#{type}"
-      when 'filter'
-        path = "fluent/plugin/filter_#{type}"
-      when 'buffer'
-        path = "fluent/plugin/buf_#{type}"
-      else
-        return
-      end
+    def self.new_storage(type, parent: nil)
+      new_impl('storage', STORAGE_REGISTRY, type, parent)
+    end
 
-      # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map {|lp|
-        lpath = File.join(lp, "#{path}.rb")
-        File.exist?(lpath) ? lpath : nil
-      }.compact
-      unless files.empty?
-        # prefer newer version
-        require File.expand_path(files.sort.last)
-        return
+    def self.register_impl(kind, registry, type, value)
+      if !value.is_a?(Class) && !value.respond_to?(:call)
+        raise Fluent::ConfigError, "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class, or callable."
       end
+      registry.register(type, value)
+      $log.trace "registered #{kind} plugin '#{type}'" if defined?($log)
+      nil
+    end
 
-      # search gems
-      specs = Gem::Specification.find_all { |spec|
-        spec.contains_requirable_file? path
-      }
-
-      # prefer newer version
-      specs = specs.sort_by { |spec| spec.version }
-      if spec = specs.last
-        spec.require_paths.each { |lib|
-          file = "#{spec.full_gem_path}/#{lib}/#{path}"
-          if File.exist?("#{file}.rb")
-            require file
-            return
-          end
-        }
+    def self.new_impl(kind, registry, type, parent=nil)
+      # "'type' not found" is handled by registry
+      obj = registry.lookup(type)
+      impl = case
+             when obj.is_a?(Class)
+               obj.new
+             when obj.respond_to?(:call) && obj.arity == 0
+               obj.call
+             else
+               raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)."
+             end
+      if parent && impl.respond_to?("owner=")
+        impl.owner = parent
       end
+      impl
     end
   end
-
-  Plugin = PluginClass.new
 end

data/lib/fluent/plugin/bare_output.rb

@@ -0,0 +1,63 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class BareOutput < Base
+      # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
+      # This output plugin base class is only for meta-output plugins
+      # which cannot be implemented on MultiOutput.
+      # E.g,: forest, config-expander
+
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      attr_reader :num_errors, :emit_count, :emit_records
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+      end
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/base.rb

@@ -0,0 +1,121 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module Plugin
+    class Base
+      include Configurable
+      include SystemConfig::Mixin
+
+      State = Struct.new(:configure, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
+
+      def initialize
+        super
+        @_state = State.new(false, false, false, false, false, false, false, false)
+      end
+
+      def has_router?
+        false
+      end
+
+      def configure(conf)
+        super
+        @_state ||= State.new(false, false, false, false, false, false, false, false)
+        @_state.configure = true
+        self
+      end
+
+      def start
+        @_state.start = true
+        self
+      end
+
+      def stop
+        @_state.stop = true
+        self
+      end
+
+      def before_shutdown
+        @_state.before_shutdown = true
+        self
+      end
+
+      def shutdown
+        @_state.shutdown = true
+        self
+      end
+
+      def after_shutdown
+        @_state.after_shutdown = true
+        self
+      end
+
+      def close
+        @_state.close = true
+        self
+      end
+
+      def terminate
+        @_state.terminate = true
+        self
+      end
+
+      def configured?
+        @_state.configure
+      end
+
+      def started?
+        @_state.start
+      end
+
+      def stopped?
+        @_state.stop
+      end
+
+      def before_shutdown?
+        @_state.before_shutdown
+      end
+
+      def shutdown?
+        @_state.shutdown
+      end
+
+      def after_shutdown?
+        @_state.after_shutdown
+      end
+
+      def closed?
+        @_state.close
+      end
+
+      def terminated?
+        @_state.terminate
+      end
+
+      def inspect
+        # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
+        # Original commit comment says that:
+        #   To emulate normal inspect behavior `ruby -e'o=Object.new;p o;p (o.__id__<<1).to_s(16)'`.
+        #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788
+        "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
+      end
+    end
+  end
+end