fluentd 0.12.40 → 0.14.0

data/lib/fluent/input.rb

@@ -14,38 +14,8 @@
 #    limitations under the License.
 #
 
-require 'fluent/config'
-require 'fluent/configurable'
-require 'fluent/engine'
-require 'fluent/log'
+require 'fluent/compat/input'
 
 module Fluent
-  class Input
-    include Configurable
-    include PluginId
-    include PluginLoggerMixin
-
-    attr_accessor :router
-
-    def initialize
-      super
-    end
-
-    def configure(conf)
-      super
-
-      if label_name = conf['@label']
-        label = Engine.root_agent.find_label(label_name)
-        @router = label.event_router
-      elsif @router.nil?
-        @router = Engine.root_agent.event_router
-      end
-    end
-
-    def start
-    end
-
-    def shutdown
-    end
-  end
+  Input = Fluent::Compat::Input
 end

data/lib/fluent/label.rb

@@ -18,8 +18,8 @@ require 'fluent/agent'
 
 module Fluent
   class Label < Agent
-    def initialize(name, opts = {})
-      super(opts)
+    def initialize(name, log:)
+      super(log: log)
 
       @context = name
       @root_agent = nil

data/lib/fluent/load.rb

@@ -9,16 +9,18 @@ require 'json'
 require 'yajl'
 require 'uri'
 require 'msgpack'
+require 'strptime'
 begin
   require 'sigdump/setup'
 rescue
   # ignore setup error on Win or similar platform which doesn't support signal
 end
 require 'cool.io'
+
+require 'fluent/time'
 require 'fluent/env'
 require 'fluent/version'
 require 'fluent/log'
-require 'fluent/status'
 require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/rpc'
@@ -28,7 +30,6 @@ require 'fluent/plugin'
 require 'fluent/parser'
 require 'fluent/formatter'
 require 'fluent/event'
-require 'fluent/buffer'
 require 'fluent/input'
 require 'fluent/output'
 require 'fluent/filter'

data/lib/fluent/log.rb

@@ -53,9 +53,34 @@ module Fluent
       end
     end
 
-    def initialize(out=STDERR, level=LEVEL_TRACE, opts={})
-      @out = out
-      @level = level
+    def initialize(logger, opts={})
+      # overwrites logger.level= so that config reloading resets level of Fluentd::Log
+      orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
+      me = self
+      # The original ruby logger sets the number as each log level like below.
+      # DEBUG = 0
+      # INFO  = 1
+      # WARN  = 2
+      # ERROR = 3
+      # FATAL = 4
+      # Serverengine use this original log number. In addition to this, serverengine sets -1 as TRACE level.
+      # TRACE = -1
+      #
+      # On the other hand, in fluentd side, it sets the number like below.
+      # TRACE = 0
+      # DEBUG = 1
+      # INFO  = 2
+      # WARN  = 3
+      # ERROR = 4
+      # FATAL = 5
+      #
+      # Then fluentd's level is set as serverengine's level + 1.
+      # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
+      logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
+
+      @logger = logger
+      @out = logger.instance_variable_get(:@logdev)
+      @level = logger.level + 1
       @debug_mode = false
       @self_event = false
       @tag = 'fluent'
@@ -67,16 +92,40 @@ module Fluent
 
       # Fluent::Engine requires Fluent::Log, so we must take that object lazily
       @engine = Fluent.const_get('Engine')
+      @optional_header = nil
+      @optional_attrs = nil
 
-      if opts.has_key?(:suppress_repeated_stacktrace)
-        @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
-      end
+      @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
+    end
+
+    def dup
+      dl_opts = {}
+      dl_opts[:log_level] = @level - 1
+      logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
+      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace)
+      clone.tag = @tag
+      clone.time_format = @time_format
+      # optional headers/attrs are not copied, because new PluginLogger should have another one of it
+      clone
     end
 
     attr_accessor :out
     attr_accessor :level
     attr_accessor :tag
     attr_accessor :time_format
+    attr_accessor :optional_header, :optional_attrs
+
+    def logdev=(logdev)
+      @out = logdev
+      @logger.instance_variable_set(:@logdev, logdev)
+      nil
+    end
+
+    def reopen!
+      # do noting in @logger.reopen! because it's already reopened in Supervisor.load_config
+      @logger.reopen! if @logger
+      nil
+    end
 
     def enable_debug(b=true)
       @debug_mode = b
@@ -229,7 +278,7 @@ module Fluent
     end
 
     def puts(msg)
-      @out.puts(msg)
+      @logger << msg + "\n"
       @out.flush
       msg
     rescue
@@ -240,18 +289,38 @@ module Fluent
     def write(data)
       @out.write(data)
     end
-    # We need `#<<` method to use this logger class with other
-    # libraries such as aws-sdk
-    alias << write
 
     def flush
       @out.flush
     end
 
+    def reset
+      @out.reset if @out.respond_to?(:reset)
+    end
+
+    private
+
+    def dump_stacktrace(backtrace, level)
+      return if @level > level
+
+      time = Time.now
+      line = caller_line(time, 5, level)
+      if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        puts ["  ", line, 'suppressed same stacktrace'].join
+      else
+        backtrace.each { |msg|
+          puts ["  ", line, msg].join
+        }
+        Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+      end
+
+      nil
+    end
+
     def event(level, args)
       time = Time.now
-      message = ''
-      map = {}
+      message = @optional_header ? @optional_header.dup : ''
+      map = @optional_attrs ? @optional_attrs.dup : {}
       args.each {|a|
         if a.is_a?(Hash)
           a.each_pair {|k,v|
@@ -263,7 +332,11 @@ module Fluent
       }
 
       map.each_pair {|k,v|
-        message << " #{k}=#{v.inspect}"
+        if k == "error".freeze && v.is_a?(Exception) && !map.has_key?("error_class")
+          message << " error_class=#{v.class.to_s} error=#{v.to_s.inspect}"
+        else
+          message << " #{k}=#{v.inspect}"
+        end
       }
 
       unless @threads_exclude_events.include?(Thread.current)
@@ -291,25 +364,6 @@ module Fluent
       end
       return log_msg
     end
-
-    private
-
-    def dump_stacktrace(backtrace, level)
-      return if @level > level
-
-      time = Time.now
-      line = caller_line(time, 5, level)
-      if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
-        puts ["  ", line, 'suppressed same stacktrace'].join
-      else
-        backtrace.each { |msg|
-          puts ["  ", line, msg].join
-        }
-        Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
-      end
-
-      nil
-    end
   end
 
 
@@ -323,7 +377,9 @@ module Fluent
       @logger = logger
       @level = @logger.level
       @depth_offset = 2
-      @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
+      if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
+        @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
+      end
 
       enable_color @logger.enable_color?
     end
@@ -342,7 +398,8 @@ module Fluent
     extend Forwardable
     def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
       :disable_events, :tag, :tag=, :time_format, :time_format=,
-      :event, :caller_line, :puts, :write, :<<, :flush, :out, :out=
+      :event, :caller_line, :puts, :write, :flush, :reset, :out, :out=,
+      :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
   end
 
 
@@ -350,7 +407,7 @@ module Fluent
     def self.included(klass)
       klass.instance_eval {
         desc 'Allows the user to set different levels of logging for each plugin.'
-        config_param :log_level, :string, default: nil, alias: :@log_level
+        config_param :@log_level, :string, default: nil, alias: :log_level # 'log_level' will be warned as deprecated
       }
     end
 
@@ -365,12 +422,24 @@ module Fluent
     def configure(conf)
       super
 
-      if @log_level
+      if level = conf['@log_level']
         unless @log.is_a?(PluginLogger)
-          @log = PluginLogger.new($log)
+          @log = PluginLogger.new($log.dup)
         end
-        @log.level = @log_level
+        @log.level = level
+        @log.optional_header = "[#{self.class.name}#{plugin_id_configured? ? "(" + @id + ")" : ""}] "
+        @log.optional_attrs = {}
       end
     end
+
+    def start
+      @log.reset
+      super
+    end
+
+    def terminate
+      super
+      @log.reset
+    end
   end
 end

data/lib/fluent/match.rb

@@ -15,42 +15,6 @@
 #
 
 module Fluent
-  class Match
-    def initialize(pattern_str, output)
-      patterns = pattern_str.split(/\s+/).map {|str|
-        MatchPattern.create(str)
-      }
-      if patterns.length == 1
-        @pattern = patterns[0]
-      else
-        @pattern = OrMatchPattern.new(patterns)
-      end
-      @output = output
-    end
-
-    attr_reader :output
-
-    def emit(tag, es)
-      chain = NullOutputChain.instance
-      @output.emit(tag, es, chain)
-    end
-
-    def start
-      @output.start
-    end
-
-    def shutdown
-      @output.shutdown
-    end
-
-    def match(tag)
-      if @pattern.match(tag)
-        return true
-      end
-      return false
-    end
-  end
-
   class MatchPattern
     def self.create(str)
       if str == '**'

data/lib/fluent/mixin.rb

@@ -26,6 +26,16 @@ module Fluent
       @tc2 = 0
       @tc2_str = nil
 
+      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
+        define_singleton_method(:format) {|time|
+          format_with_subsec(time)
+        }
+      else
+        define_singleton_method(:format) {|time|
+          format_without_subsec(time)
+        }
+      end
+
       if formatter = Fluent::Timezone.formatter(timezone, format)
         define_singleton_method(:format_nocache) {|time|
           formatter.call(time)
@@ -56,7 +66,7 @@ module Fluent
       end
     end
 
-    def format(time)
+    def format_without_subsec(time)
       if @tc1 == time
         return @tc1_str
       elsif @tc2 == time
@@ -74,6 +84,28 @@ module Fluent
       end
     end
 
+    def format_with_subsec(time)
+      if Fluent::EventTime.eq?(@tc1, time)
+        return @tc1_str
+      elsif Fluent::EventTime.eq?(@tc2, time)
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    def format(time)
+      # will be overridden in initialize
+    end
+
     def format_nocache(time)
       # will be overridden in initialize
     end
@@ -101,13 +133,18 @@ module Fluent
     attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
     def configure(conf)
       super
-      if remove_tag_prefix = conf['remove_tag_prefix']
-        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
-      end
 
-      if remove_tag_suffix = conf['remove_tag_suffix']
-        @remove_tag_suffix = Regexp.new(Regexp.escape(remove_tag_suffix) + '$')
-      end
+      @remove_tag_prefix = if conf.has_key?('remove_tag_prefix')
+                             Regexp.new('^' + Regexp.escape(conf['remove_tag_prefix']))
+                           else
+                             nil
+                           end
+
+      @remove_tag_suffix = if conf.has_key?('remove_tag_suffix')
+                             Regexp.new(Regexp.escape(conf['remove_tag_suffix']) + '$')
+                           else
+                             nil
+                           end
 
       @add_tag_prefix = conf['add_tag_prefix']
       @add_tag_suffix = conf['add_tag_suffix']
@@ -130,6 +167,8 @@ module Fluent
 
     def configure(conf)
       @include_time_key = false
+      @localtime = false
+      @timezone = nil
 
       super
 
@@ -192,4 +231,75 @@ module Fluent
       record[@tag_key] = tag if @include_tag_key
     end
   end
+
+  module TypeConverter
+    Converters = {
+      'string' => lambda { |v| v.to_s },
+      'integer' => lambda { |v| v.to_i },
+      'float' => lambda { |v| v.to_f },
+      'bool' => lambda { |v|
+        case v.downcase
+        when 'true', 'yes', '1'
+          true
+        else
+          false
+        end
+      },
+      'time' => lambda { |v, time_parser|
+        time_parser.parse(v)
+      },
+      'array' => lambda { |v, delimiter|
+        v.to_s.split(delimiter)
+      }
+    }
+
+    def self.included(klass)
+      klass.instance_eval {
+        config_param :types, :string, default: nil
+        config_param :types_delimiter, :string, default: ','
+        config_param :types_label_delimiter, :string, default: ':'
+      }
+    end
+
+    def configure(conf)
+      super
+
+      @type_converters = nil
+      @type_converters = parse_types_parameter unless @types.nil?
+    end
+
+    private
+
+    def convert_type(name, value)
+      converter = @type_converters[name]
+      converter.nil? ? value : converter.call(value)
+    end
+
+    def parse_types_parameter
+      converters = {}
+
+      @types.split(@types_delimiter).each { |pattern_name|
+        name, type, format = pattern_name.split(@types_label_delimiter, 3)
+        raise ConfigError, "Type is needed" if type.nil?
+
+        case type
+        when 'time'
+          require 'fluent/parser'
+          t_parser = Fluent::TextParser::TimeParser.new(format)
+          converters[name] = lambda { |v|
+            Converters[type].call(v, t_parser)
+          }
+        when 'array'
+          delimiter = format || ','
+          converters[name] = lambda { |v|
+            Converters[type].call(v, delimiter)
+          }
+        else
+          converters[name] = Converters[type]
+        end
+      }
+
+      converters
+    end
+  end
 end