fluentd 0.14.2 → 0.14.5

data/lib/fluent/config/configure_proxy.rb

@@ -208,6 +208,12 @@ module Fluent
         end
       end
 
+      def option_value_type!(name, opts, key, klass)
+        if opts.has_key?(key) && !opts[key].is_a?(klass)
+          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+        end
+      end
+
       def parameter_configuration(name, type = nil, **kwargs, &block)
         name = name.to_sym
 
@@ -216,7 +222,7 @@ module Fluent
         opts.merge!(kwargs)
 
         if block && type
-          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
+          raise ArgumentError, "#{name}: both of block and type cannot be specified"
         end
 
         begin
@@ -224,8 +230,19 @@ module Fluent
           block ||= @type_lookup.call(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
+        end
+
+        option_value_type!(name, opts, :desc, String)
+        option_value_type!(name, opts, :alias, Symbol)
+        option_value_type!(name, opts, :deprecated, String)
+        option_value_type!(name, opts, :obsoleted, String)
+        if type == :enum
+          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
+            raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
+          end
         end
+        option_value_type!(name, opts, :value_type, Symbol) # hash, array
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -235,6 +252,10 @@ module Fluent
           config_set_desc(name, opts[:desc])
         end
 
+        if opts[:deprecated] && opts[:obsoleted]
+          raise ArgumentError, "#{name}: both of deprecated and obsoleted cannot be specified at once"
+        end
+
         [name, block, opts]
       end
 
@@ -296,7 +317,7 @@ module Fluent
 
       def config_section(name, **kwargs, &block)
         unless block_given?
-          raise ArgumentError, "#{self.name}: config_section requires block parameter"
+          raise ArgumentError, "#{name}: config_section requires block parameter"
         end
         name = name.to_sym
 
@@ -305,10 +326,10 @@ module Fluent
 
         if sub_proxy.init?
           if sub_proxy.argument && !sub_proxy.defaults.has_key?(sub_proxy.argument.first)
-            raise ArgumentError, "#{self.name}: init is specified, but default value of argument is missing"
+            raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
           end
           if sub_proxy.params.keys.any?{|param_name| !sub_proxy.defaults.has_key?(param_name)}
-            raise ArgumentError, "#{self.name}: init is specified, but there're parameters without default values"
+            raise ArgumentError, "#{name}: init is specified, but there're parameters without default values"
           end
         end
 

data/lib/fluent/config/error.rb

@@ -20,4 +20,7 @@ module Fluent
 
   class ConfigParseError < ConfigError
   end
+
+  class ObsoletedParameterError < ConfigError
+  end
 end

data/lib/fluent/config/section.rb

@@ -126,6 +126,7 @@ module Fluent
             logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
+          # argument should NOT be deprecated... (argument always has a value: '')
         end
 
         proxy.params.each_pair do |name, defval|
@@ -138,6 +139,20 @@ module Fluent
                     conf[opts[:alias].to_s]
                   end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
           end
           unless section_params.has_key?(varname)
             logger.error "config error in:\n#{conf}" if logger

data/lib/fluent/event_router.rb

@@ -136,10 +136,12 @@ module Fluent
       def initialize
         @filters = []
         @output = nil
+        @optimizer = FilterOptimizer.new
       end
 
       def add_filter(filter)
         @filters << filter
+        @optimizer.filters = @filters
       end
 
       def set_output(output)
@@ -147,12 +149,83 @@ module Fluent
       end
 
       def emit_events(tag, es)
-        processed = es
-        @filters.each { |filter|
-          processed = filter.filter_stream(tag, processed)
-        }
+        processed = @optimizer.filter_stream(tag, es)
         @output.emit_events(tag, processed)
       end
+
+      class FilterOptimizer
+        def initialize(filters = [])
+          @filters = filters
+          @optimizable = nil
+        end
+
+        def filters=(filters)
+          @filters = filters
+          reset_optimization
+        end
+
+        def filter_stream(tag, es)
+          if optimizable?
+            optimized_filter_stream(tag, es)
+          else
+            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+          end
+        end
+
+        private
+
+        def optimized_filter_stream(tag, es)
+          new_es = MultiEventStream.new
+          es.each do |time, record|
+            filtered_record = record
+            filtered_time = time
+
+            catch :break_loop do
+              @filters.each do |filter|
+                if filter.has_filter_with_time
+                  begin
+                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record && filtered_time
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                else
+                  begin
+                    filtered_record = filter.filter(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                end
+              end
+
+              new_es.add(filtered_time, filtered_record)
+            end
+          end
+          new_es
+        end
+
+        def optimizable?
+          return @optimizable unless @optimizable.nil?
+          @optimizable = if filters_having_filter_stream.empty?
+                           true
+                         else
+                           $log.info "Filtering works with worse performance, because #{filters_having_filter_stream.map(&:class)} uses `#filter_stream` method."
+                           false
+                         end
+        end
+
+        def filters_having_filter_stream
+          @filters_having_filter_stream ||= @filters.select do |filter|
+            filter.class.instance_methods(false).include?(:filter_stream)
+          end
+        end
+
+        def reset_optimization
+          @optimizable = nil
+          @filters_having_filter_stream = nil
+        end
+      end
     end
 
     def find(tag)

data/lib/fluent/plugin/base.rb

@@ -24,11 +24,11 @@ module Fluent
       include Configurable
       include SystemConfig::Mixin
 
-      State = Struct.new(:configure, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
+      State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
 
       def initialize
         super
-        @_state = State.new(false, false, false, false, false, false, false, false)
+        @_state = State.new(false, false, false, false, false, false, false, false, false)
       end
 
       def has_router?
@@ -37,7 +37,7 @@ module Fluent
 
       def configure(conf)
         super
-        @_state ||= State.new(false, false, false, false, false, false, false, false)
+        @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true
         self
       end
@@ -47,6 +47,11 @@ module Fluent
         self
       end
 
+      def after_start
+        @_state.after_start = true
+        self
+      end
+
       def stop
         @_state.stop = true
         self
@@ -85,6 +90,10 @@ module Fluent
         @_state.start
       end
 
+      def after_started?
+        @_state.after_start
+      end
+
       def stopped?
         @_state.stop
       end

data/lib/fluent/plugin/buffer.rb

@@ -55,7 +55,11 @@ module Fluent
       # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
       config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
 
-      Metadata = Struct.new(:timekey, :tag, :variables)
+      Metadata = Struct.new(:timekey, :tag, :variables) do
+        def empty?
+          timekey.nil? && tag.nil? && variables.nil?
+        end
+      end
 
       # for tests
       attr_accessor :stage_size, :queue_size

data/lib/fluent/plugin/filter.rb

@@ -30,22 +30,64 @@ module Fluent
 
       helpers :event_emitter
 
+      attr_reader :has_filter_with_time
+
+      def initialize
+        super
+        @has_filter_with_time = has_filter_with_time?
+      end
+
       def filter(tag, time, record)
         raise NotImplementedError, "BUG: filter plugins MUST implement this method"
       end
 
+      def filter_with_time(tag, time, record)
+        raise NotImplementedError, "BUG: filter plugins MUST implement this method"
+      end
+
       def filter_stream(tag, es)
         new_es = MultiEventStream.new
-        es.each do |time, record|
-          begin
-            filtered_record = filter(tag, time, record)
-            new_es.add(time, filtered_record) if filtered_record
-          rescue => e
-            router.emit_error_event(tag, time, record, e)
+        if @has_filter_with_time
+          es.each do |time, record|
+            begin
+              filtered_time, filtered_record = filter_with_time(tag, time, record)
+              new_es.add(filtered_time, filtered_record) if filtered_time && filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        else
+          es.each do |time, record|
+            begin
+              filtered_record = filter(tag, time, record)
+              new_es.add(time, filtered_record) if filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
           end
         end
         new_es
       end
+
+      private
+
+      def has_filter_with_time?
+        implmented_methods = self.class.instance_methods(false)
+        # Plugins that override `filter_stream` don't need check,
+        # because they may not call `filter` or `filter_with_time`
+        # for example fluentd/lib/fluent/plugin/filter_record_transformer.rb
+        return nil if implmented_methods.include?(:filter_stream)
+        case
+        when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
+          raise "BUG: Filter plugins MUST be implemented either `filter` or `filter_with_time`"
+        when implmented_methods.include?(:filter)
+          false
+        when implmented_methods.include?(:filter_with_time)
+          true
+        else
+          raise NotImplementedError, "BUG: Filter plugins MUST be implmented either `filter` or `filter_with_time`"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -204,7 +204,9 @@ module Fluent::Plugin
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              placeholders.store("${#{k}}", v) # foo
+              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
+                placeholders.store("${#{k}}", v) # foo
+              end
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?

data/lib/fluent/plugin/filter_stdout.rb

@@ -34,11 +34,7 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :inject, :formatter)
       super
-    end
-
-    def start
       @formatter = formatter_create(conf: @config.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
-      super
     end
 
     def filter_stream(tag, es)

data/lib/fluent/plugin/in_debug_agent.rb

@@ -14,11 +14,11 @@
 #    limitations under the License.
 #
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
+module Fluent::Plugin
   class DebugAgentInput < Input
-    Plugin.register_input('debug_agent', self)
+    Fluent::Plugin.register_input('debug_agent', self)
 
     def initialize
       require 'drb/drb'
@@ -30,13 +30,13 @@ module Fluent
     config_param :port, :integer, default: 24230
     config_param :unix_path, :string, default: nil
     #config_param :unix_mode  # TODO
-    config_param :object, :string, default: 'Engine'
+    config_param :object, :string, default: 'Fluent::Engine'
 
     def configure(conf)
       super
       if @unix_path
         unless ::Fluent::FileUtil.writable?(@unix_path)
-          raise ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
+          raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
         end
       end
     end

data/lib/fluent/plugin/in_dummy.rb

@@ -30,6 +30,8 @@ module Fluent::Plugin
 
     desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
+    desc "The number of events in event stream of each emits."
+    config_param :size, :integer, default: 1
     desc "It configures how many events to generate per second."
     config_param :rate, :integer, default: 1
     desc "If specified, each generated event has an auto-incremented key field."
@@ -97,7 +99,13 @@ module Fluent::Plugin
 
     def emit(num)
       begin
-        num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+        if @size > 1
+          num.times do
+            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
+          end
+        else
+          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
+        end
       rescue => _
         # ignore all errors not to stop emits by emit errors
       end