fluentd 0.12.40 → 1.6.2

data/lib/fluent/event_router.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/match'
 require 'fluent/event'
+require 'fluent/filter'
 
 module Fluent
   #
@@ -45,7 +46,6 @@ module Fluent
       @match_cache = MatchCache.new
       @default_collector = default_collector
       @emit_error_handler = emit_error_handler
-      @chain = NullOutputChain.instance
     end
 
     attr_accessor :default_collector
@@ -71,6 +71,12 @@ module Fluent
       attr_reader :pattern_str
     end
 
+    def suppress_missing_match!
+      if @default_collector.respond_to?(:suppress_missing_match!)
+        @default_collector.suppress_missing_match!
+      end
+    end
+
     # called by Agent to add new match pattern and collector
     def add_rule(pattern, collector)
       @match_rules << Rule.new(pattern, collector)
@@ -87,7 +93,7 @@ module Fluent
     end
 
     def emit_stream(tag, es)
-      match(tag).emit(tag, es, @chain)
+      match(tag).emit_events(tag, es)
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -102,7 +108,7 @@ module Fluent
 
     def match(tag)
       collector = @match_cache.get(tag) {
-        c = find(tag) || @default_collector
+        find(tag) || @default_collector
       }
       collector
     end
@@ -136,22 +142,99 @@ module Fluent
       def initialize
         @filters = []
         @output = nil
+        @optimizer = FilterOptimizer.new
       end
 
       def add_filter(filter)
         @filters << filter
+        @optimizer.filters = @filters
       end
 
       def set_output(output)
         @output = output
       end
 
-      def emit(tag, es, chain)
-        processed = es
-        @filters.each { |filter|
-          processed = filter.filter_stream(tag, processed)
-        }
-        @output.emit(tag, processed, chain)
+      def emit_events(tag, es)
+        processed = @optimizer.filter_stream(tag, es)
+        @output.emit_events(tag, processed)
+      end
+
+      class FilterOptimizer
+        def initialize(filters = [])
+          @filters = filters
+          @optimizable = nil
+        end
+
+        def filters=(filters)
+          @filters = filters
+          reset_optimization
+        end
+
+        def filter_stream(tag, es)
+          if optimizable?
+            optimized_filter_stream(tag, es)
+          else
+            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+          end
+        end
+
+        private
+
+        def optimized_filter_stream(tag, es)
+          new_es = MultiEventStream.new
+          es.each do |time, record|
+            filtered_record = record
+            filtered_time = time
+
+            catch :break_loop do
+              @filters.each do |filter|
+                if filter.has_filter_with_time
+                  begin
+                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record && filtered_time
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                else
+                  begin
+                    filtered_record = filter.filter(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                end
+              end
+
+              new_es.add(filtered_time, filtered_record)
+            end
+          end
+          new_es
+        end
+
+        def optimizable?
+          return @optimizable unless @optimizable.nil?
+          fs_filters = filters_having_filter_stream
+          @optimizable = if fs_filters.empty?
+                           true
+                         else
+                           # skip log message when filter is only 1, because its performance is same as non optimized chain.
+                           if @filters.size > 1 && fs_filters.size >= 1
+                             $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
+                           end
+                           false
+                         end
+        end
+
+        def filters_having_filter_stream
+          @filters_having_filter_stream ||= @filters.select do |filter|
+            filter.class.instance_methods(false).include?(:filter_stream)
+          end
+        end
+
+        def reset_optimization
+          @optimizable = nil
+          @filters_having_filter_stream = nil
+        end
       end
     end
 
@@ -159,7 +242,7 @@ module Fluent
       pipeline = nil
       @match_rules.each_with_index { |rule, i|
         if rule.match?(tag)
-          if rule.collector.is_a?(Filter)
+          if rule.collector.is_a?(Plugin::Filter)
             pipeline ||= Pipeline.new
             pipeline.add_filter(rule.collector)
           else

data/lib/fluent/filter.rb

@@ -14,56 +14,8 @@
 #    limitations under the License.
 #
 
-require 'fluent/config'
-require 'fluent/configurable'
-require 'fluent/engine'
-require 'fluent/event'
-require 'fluent/log'
+require 'fluent/compat/filter'
 
 module Fluent
-  class Filter
-    include Configurable
-    include PluginId
-    include PluginLoggerMixin
-
-    attr_accessor :router
-
-    def initialize
-      super
-    end
-
-    def configure(conf)
-      super
-
-      if label_name = conf['@label']
-        label = Engine.root_agent.find_label(label_name)
-        @router = label.event_router
-      elsif @router.nil?
-        @router = Engine.root_agent.event_router
-      end
-    end
-
-    def start
-    end
-
-    def shutdown
-    end
-
-    def filter(tag, time, record)
-      raise NotImplementedError, "Implement this method in child class"
-    end
-
-    def filter_stream(tag, es)
-      new_es = MultiEventStream.new
-      es.each { |time, record|
-        begin
-          filtered_record = filter(tag, time, record)
-          new_es.add(time, filtered_record) if filtered_record
-        rescue => e
-          router.emit_error_event(tag, time, record, e)
-        end
-      }
-      new_es
-    end
-  end
+  Filter = Fluent::Compat::Filter
 end

data/lib/fluent/formatter.rb

@@ -14,299 +14,10 @@
 #    limitations under the License.
 #
 
-require 'fluent/configurable'
-require 'fluent/env'
-require 'fluent/registry'
-require 'fluent/mixin'
+require 'fluent/compat/formatter'
 
 module Fluent
-  class Formatter
-    include Configurable
-
-    def configure(conf)
-      super
-    end
-
-    def format(tag, time, record)
-      raise NotImplementedError, "Implement this method in child class"
-    end
-  end
-
-  module TextFormatter
-    module HandleTagAndTimeMixin
-      def self.included(klass)
-        klass.instance_eval {
-          config_param :include_time_key, :bool, default: false
-          config_param :time_key, :string, default: 'time'
-          config_param :time_format, :string, default: nil
-          config_param :include_tag_key, :bool, default: false
-          config_param :tag_key, :string, default: 'tag'
-          config_param :localtime, :bool, default: true
-          config_param :timezone, :string, default: nil
-        }
-      end
-
-      def configure(conf)
-        super
-
-        if conf['utc']
-          @localtime = false
-        end
-        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
-      end
-
-      def filter_record(tag, time, record)
-        if @include_tag_key
-          record[@tag_key] = tag
-        end
-        if @include_time_key
-          record[@time_key] = @timef.format(time)
-        end
-      end
-    end
-
-    class OutFileFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :output_time, :bool, default: true
-      config_param :output_tag, :bool, default: true
-      config_param :delimiter, default: "\t" do |val|
-        case val
-        when /SPACE/i then ' '
-        when /COMMA/i then ','
-        else "\t"
-        end
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        header = ''
-        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
-        header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
-      end
-    end
-
-    class StdoutFormatter < Formatter
-      config_param :output_type, :string, default: 'json'
-
-      def configure(conf)
-        super
-
-        @formatter = Plugin.new_formatter(@output_type)
-        @formatter.configure(conf)
-      end
-
-      def format(tag, time, record)
-        header = "#{Time.now.localtime} #{tag}: "
-        "#{header}#{@formatter.format(tag, time, record)}"
-      end
-    end
-
-    module StructuredFormatMixin
-      def self.included(klass)
-        klass.instance_eval {
-          config_param :time_as_epoch, :bool, default: false
-        }
-      end
-
-      def configure(conf)
-        super
-
-        if @time_as_epoch
-          if @include_time_key
-            @include_time_key = false
-          else
-            $log.warn "include_time_key is false so ignore time_as_epoch"
-            @time_as_epoch = false
-          end
-        end
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        record[@time_key] = time if @time_as_epoch
-        format_record(record)
-      end
-    end
-
-    class JSONFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      config_param :json_parser, :string, default: 'oj'
-      config_param :add_newline, :bool, default: true
-
-      def configure(conf)
-        super
-
-        begin
-          raise LoadError unless @json_parser == 'oj'
-          require 'oj'
-          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @dump_proc = Oj.method(:dump)
-        rescue LoadError
-          @dump_proc = Yajl.method(:dump)
-        end
-
-        # format json is used on various highload environment, so re-define method to skip if check
-        unless @add_newline
-          define_singleton_method(:format, method(:format_without_nl))
-        end
-      end
-
-      def format_record(record)
-        "#{@dump_proc.call(record)}\n"
-      end
-
-      def format_without_nl(tag, time, record)
-        @dump_proc.call(record)
-      end
-    end
-
-    class HashFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      config_param :add_newline, :bool, default: true
-
-      def format_record(record)
-        if @add_newline
-          "#{record.to_s}\n"
-        else
-          record.to_s
-        end
-      end
-    end
-
-    class MessagePackFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
-        record.to_msgpack
-      end
-    end
-
-    class LabeledTSVFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :delimiter, :string, default: "\t"
-      config_param :label_delimiter, :string, default: ":"
-      config_param :add_newline, :bool, default: true
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        formatted = record.inject('') { |result, pair|
-          result << @delimiter if result.length.nonzero?
-          result << "#{pair.first}#{@label_delimiter}#{pair.last}"
-        }
-        formatted << "\n".freeze if @add_newline
-        formatted
-      end
-    end
-
-    class CsvFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :delimiter, default: ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
-      end
-      config_param :force_quotes, :bool, default: true
-      config_param :fields, :array, value_type: :string
-      config_param :add_newline, :bool, default: true
-
-      def initialize
-        super
-        require 'csv'
-      end
-
-      def configure(conf)
-        super
-        @fields = fields.select { |f| !f.empty? }
-        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
-
-        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        row = @fields.inject([]) do |memo, key|
-            memo << record[key]
-            memo
-        end
-        line = CSV.generate_line(row, @generate_opts)
-        line.chomp! unless @add_newline
-        line
-      end
-    end
-
-    class SingleValueFormatter < Formatter
-      config_param :message_key, :string, default: 'message'
-      config_param :add_newline, :bool, default: true
-
-      def format(tag, time, record)
-        text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
-        text
-      end
-    end
-
-    class ProcWrappedFormatter < Formatter
-      def initialize(proc)
-        @proc = proc
-      end
-
-      def configure(conf)
-      end
-
-      def format(tag, time, record)
-        @proc.call(tag, time, record)
-      end
-    end
-
-    TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
-    {
-      'out_file' => Proc.new { OutFileFormatter.new },
-      'stdout' => Proc.new { StdoutFormatter.new },
-      'json' => Proc.new { JSONFormatter.new },
-      'hash' => Proc.new { HashFormatter.new },
-      'msgpack' => Proc.new { MessagePackFormatter.new },
-      'ltsv' => Proc.new { LabeledTSVFormatter.new },
-      'csv' => Proc.new { CsvFormatter.new },
-      'single_value' => Proc.new { SingleValueFormatter.new },
-    }.each { |name, factory|
-      TEMPLATE_REGISTRY.register(name, factory)
-    }
-
-    def self.register_template(name, factory_or_proc)
-      factory = if factory_or_proc.is_a?(Class) # XXXFormatter
-                  Proc.new { factory_or_proc.new }
-                elsif factory_or_proc.arity == 3 # Proc.new { |tag, time, record| }
-                  Proc.new { ProcWrappedFormatter.new(factory_or_proc) }
-                else # Proc.new { XXXFormatter.new }
-                  factory_or_proc
-                end
-
-      TEMPLATE_REGISTRY.register(name, factory)
-    end
-
-    def self.lookup(format)
-      TEMPLATE_REGISTRY.lookup(format).call
-    end
-
-    # Keep backward-compatibility
-    def self.create(conf)
-      format = conf['format']
-      if format.nil?
-        raise ConfigError, "'format' parameter is required"
-      end
-
-      formatter = lookup(format)
-      if formatter.respond_to?(:configure)
-        formatter.configure(conf)
-      end
-      formatter
-    end
-  end
+  Formatter = Fluent::Compat::Formatter
+  TextFormatter = Fluent::Compat::TextFormatter
+  # deprecate_constant is ruby 2.3 feature
 end

data/lib/fluent/input.rb

@@ -14,38 +14,8 @@
 #    limitations under the License.
 #
 
-require 'fluent/config'
-require 'fluent/configurable'
-require 'fluent/engine'
-require 'fluent/log'
+require 'fluent/compat/input'
 
 module Fluent
-  class Input
-    include Configurable
-    include PluginId
-    include PluginLoggerMixin
-
-    attr_accessor :router
-
-    def initialize
-      super
-    end
-
-    def configure(conf)
-      super
-
-      if label_name = conf['@label']
-        label = Engine.root_agent.find_label(label_name)
-        @router = label.event_router
-      elsif @router.nil?
-        @router = Engine.root_agent.event_router
-      end
-    end
-
-    def start
-    end
-
-    def shutdown
-    end
-  end
+  Input = Fluent::Compat::Input
 end

data/lib/fluent/label.rb

@@ -18,8 +18,8 @@ require 'fluent/agent'
 
 module Fluent
   class Label < Agent
-    def initialize(name, opts = {})
-      super(opts)
+    def initialize(name, log:)
+      super(log: log)
 
       @context = name
       @root_agent = nil
@@ -27,6 +27,14 @@ module Fluent
 
     attr_accessor :root_agent
 
+    def configure(conf)
+      super
+
+      if conf.elements('match').size == 0
+        raise ConfigError, "Missing <match> sections in <label #{@context}> section"
+      end
+    end
+
     def emit_error_event(tag, time, record, e)
       @root_agent.emit_error_event(tag, time, record, e)
     end

data/lib/fluent/load.rb

@@ -9,26 +9,26 @@ require 'json'
 require 'yajl'
 require 'uri'
 require 'msgpack'
+require 'strptime'
 begin
   require 'sigdump/setup'
 rescue
   # ignore setup error on Win or similar platform which doesn't support signal
 end
 require 'cool.io'
+
+require 'fluent/time'
 require 'fluent/env'
 require 'fluent/version'
 require 'fluent/log'
-require 'fluent/status'
 require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/rpc'
 require 'fluent/mixin'
-require 'fluent/process'
 require 'fluent/plugin'
 require 'fluent/parser'
 require 'fluent/formatter'
 require 'fluent/event'
-require 'fluent/buffer'
 require 'fluent/input'
 require 'fluent/output'
 require 'fluent/filter'