fluentd 0.14.6 → 0.14.7

data/lib/fluent/registry.rb

@@ -28,7 +28,7 @@ module Fluent
       @paths = [DEFAULT_PLUGIN_PATH]
     end
 
-    attr_reader :kind, :paths
+    attr_reader :kind, :paths, :map, :dir_search_prefix
 
     def register(type, value)
       type = type.to_sym

data/lib/fluent/root_agent.rb

@@ -163,7 +163,7 @@ module Fluent
     end
 
     def shutdown # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
-      # Thesee method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
+      # These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
       # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
       # Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
 
@@ -242,6 +242,7 @@ module Fluent
 
     def add_label(name)
       label = Label.new(name, log: log)
+      raise ConfigError, "Section <label #{name}> appears twice" if @labels[name]
       label.root_agent = self
       @labels[name] = label
     end

data/lib/fluent/supervisor.rb

@@ -230,11 +230,17 @@ module Fluent
       log_path = params['log_path']
       chuser = params['chuser']
       chgroup = params['chgroup']
+      log_rotate_age = params['log_rotate_age']
+      log_rotate_size = params['log_rotate_size']
       rpc_endpoint = system_config.rpc_endpoint
       enable_get_dump = system_config.enable_get_dump
 
       log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
-      logger_initializer = Supervisor::LoggerInitializer.new(log_path, log_level, chuser, chgroup, log_opts)
+      logger_initializer = Supervisor::LoggerInitializer.new(
+        log_path, log_level, chuser, chgroup, log_opts,
+        log_rotate_age: log_rotate_age,
+        log_rotate_size: log_rotate_size
+      )
       # this #init sets initialized logger to $log
       logger_initializer.init
       logger = $log
@@ -295,42 +301,48 @@ module Fluent
     end
 
     class LoggerInitializer
-      def initialize(path, level, chuser, chgroup, opts)
+      def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
         @path = path
         @level = level
         @chuser = chuser
         @chgroup = chgroup
         @opts = opts
+        @log_rotate_age = log_rotate_age
+        @log_rotate_size = log_rotate_size
       end
 
       def init
         if @path && @path != "-"
-          @io = File.open(@path, "a")
+          @logdev = if @log_rotate_age || @log_rotate_size
+                     Fluent::LogDeviceIO.new(@path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
+                   else
+                     File.open(@path, "a")
+                   end
           if @chuser || @chgroup
             chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
             chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
             File.chown(chuid, chgid, @path)
           end
         else
-          @io = STDOUT
+          @logdev = STDOUT
         end
 
         dl_opts = {}
         # subtract 1 to match serverengine daemon logger side logging severity.
         dl_opts[:log_level] = @level - 1
-        logger = ServerEngine::DaemonLogger.new(@io, dl_opts)
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
         $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
       end
 
       def stdout?
-        @io == STDOUT
+        @logdev == STDOUT
       end
 
       def reopen!
         if @path && @path != "-"
-          @io.reopen(@path, "a")
+          @logdev.reopen(@path, "a")
         end
         self
       end
@@ -381,6 +393,8 @@ module Fluent
       @process_name = nil
 
       @log_level = opt[:log_level]
+      @log_rotate_age = opt[:log_rotate_age]
+      @log_rotate_size = opt[:log_rotate_size]
       @suppress_interval = opt[:suppress_interval]
       @suppress_config_dump = opt[:suppress_config_dump]
       @without_source = opt[:without_source]
@@ -388,7 +402,11 @@ module Fluent
 
       @suppress_repeated_stacktrace = opt[:suppress_repeated_stacktrace]
       log_opts = {suppress_repeated_stacktrace: @suppress_repeated_stacktrace}
-      @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
+      @log = LoggerInitializer.new(
+        @log_path, @log_level, @chuser, @chgroup, log_opts,
+        log_rotate_age: @log_rotate_age,
+        log_rotate_size: @log_rotate_size
+      )
       @finished = false
     end
 
@@ -512,6 +530,8 @@ module Fluent
       params['inline_config'] = @inline_config
       params['log_path'] = @log_path
       params['log_level'] = @log_level
+      params['log_rotate_age'] = @log_rotate_age
+      params['log_rotate_size'] = @log_rotate_size
       params['chuser'] = @chuser
       params['chgroup'] = @chgroup
       params['use_v1_config'] = @use_v1_config

data/lib/fluent/test/base.rb

@@ -76,10 +76,3 @@ module Fluent
     end
   end
 end
-
-Test::Unit::Assertions.module_eval do
-  def assert_equal_event_time(a, b)
-    assert_equal(a.sec, b.sec)
-    assert_equal(a.nsec, b.nsec)
-  end
-end

data/lib/fluent/test/driver/base.rb

@@ -40,6 +40,7 @@ module Fluent
           else
             @instance = klass
           end
+          @instance.under_plugin_development = true
 
           @logs = []
 

data/lib/fluent/test/driver/output.rb

@@ -18,6 +18,7 @@ require 'fluent/test/driver/base_owner'
 require 'fluent/test/driver/event_feeder'
 
 require 'fluent/plugin/output'
+require 'timeout'
 
 module Fluent
   module Test
@@ -43,6 +44,7 @@ module Fluent
           super(**kwargs, &block)
           if @flush_buffer_at_cleanup
             @instance.force_flush
+            Timeout.timeout(10){ sleep 0.1 until !@instance.buffer || @instance.buffer.queue.size == 0 }
           end
         end
 
@@ -52,6 +54,7 @@ module Fluent
 
         def flush
           @instance.force_flush
+          Timeout.timeout(10){ sleep 0.1 until !@instance.buffer || @instance.buffer.queue.size == 0 }
         end
 
         def instance_hook_after_started

data/lib/fluent/test/helpers.rb

@@ -21,6 +21,17 @@ require 'fluent/time'
 module Fluent
   module Test
     module Helpers
+      # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      def assert_equal_event_time(expected, actual, message = nil)
+        message = build_message(message, <<EOT, expected, actual)
+<?> expected but was
+<?>.
+EOT
+        assert_block(message) do
+          expected.is_a?(Fluent::EventTime) && actual.is_a?(Fluent::EventTime) && expected.sec == actual.sec && expected.nsec == actual.nsec
+        end
+      end
+
       def config_element(name = 'test', argument = '', params = {}, elements = [])
         Fluent::Config::Element.new(name, argument, params, elements)
       end
@@ -37,6 +48,13 @@ module Fluent
         end
       end
 
+      def with_timezone(tz)
+        oldtz, ENV['TZ'] = ENV['TZ'], tz
+        yield
+      ensure
+        ENV['TZ'] = oldtz
+      end
+
       def time2str(time, localtime: false, format: nil)
         if format
           if localtime

data/lib/fluent/test/input_test.rb

@@ -24,6 +24,7 @@ module Fluent
       def initialize(klass, &block)
         super(klass, &block)
         @emit_streams = []
+        @event_streams = []
         @expects = nil
         # for checking only the number of emitted records during run
         @expected_emits_length = nil
@@ -42,7 +43,7 @@ module Fluent
 
       attr_accessor :expected_emits_length
       attr_accessor :run_timeout
-      attr_reader :emit_streams
+      attr_reader :emit_streams, :event_streams
 
       def emits
         all = []
@@ -135,7 +136,7 @@ module Fluent
               end
             end
 
-            # Set runnning timeout to avoid infinite loop caused by some errors.
+            # Set running timeout to avoid infinite loop caused by some errors.
             started_at = Time.now
             register_run_breaking_condition do
               Time.now >= started_at + @run_timeout
@@ -165,6 +166,7 @@ module Fluent
 
       private
       def emit_stream(tag, es)
+        @event_streams << es
         @emit_streams << [tag, es.to_a]
       end
     end

data/lib/fluent/test/log.rb

@@ -21,13 +21,15 @@ module Fluent
   module Test
     class DummyLogDevice
       attr_reader :logs
+      attr_accessor :flush_logs
 
       def initialize
         @logs = []
+        @flush_logs = true
       end
 
       def reset
-        @logs = []
+        @logs = [] if @flush_logs
       end
 
       def tty?

data/lib/fluent/time.rb

@@ -16,7 +16,10 @@
 
 require 'time'
 require 'msgpack'
+require 'strptime'
 require 'fluent/timezone'
+require 'fluent/configurable'
+require 'fluent/config/error'
 
 module Fluent
   class EventTime
@@ -103,8 +106,207 @@ module Fluent
     end
   end
 
+  module TimeMixin
+    TIME_PARAMETERS = [
+      [:time_format, :string, {default: nil}],
+      [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
+      [:utc,       :bool, {default: false}], # to turn :localtime false
+      [:timezone, :string, {default: nil}],
+    ]
+    TIME_FULL_PARAMETERS = [
+      # To avoid to define :time_type twice (in plugin_helper/inject)
+      [:time_type, :enum, {default: :string, list: [:string, :unixtime, :float]}],
+    ] + TIME_PARAMETERS
+
+    module TimeParameters
+      include Fluent::Configurable
+      TIME_FULL_PARAMETERS.each do |name, type, opts|
+        config_param name, type, opts
+      end
+
+      def configure(conf)
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        super
+
+        Fluent::Timezone.validate!(@timezone) if @timezone
+      end
+    end
+
+    module Parser
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeParser.new(type) if type != :string
+        return TimeParser.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeParser.new(format, localtime, timezone)
+      end
+    end
+
+    module Formatter
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_formatter_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeFormatter.new(type) if type != :string
+        return TimeFormatter.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeFormatter.new(format, localtime, timezone)
+      end
+    end
+  end
+
+  class TimeParser
+    class TimeParseError < StandardError; end
+
+    def initialize(format = nil, localtime = true, timezone = nil)
+      if format.nil? && (timezone || !localtime)
+        raise Fluent::ConfigError, "specifying timezone requires time format"
+      end
+
+      @cache1_key = nil
+      @cache1_time = nil
+      @cache2_key = nil
+      @cache2_time = nil
+
+      format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
+
+      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
+      offset_diff = case
+                    when format_with_timezone then nil
+                    when timezone  then Time.now.localtime.utc_offset - Time.zone_offset(timezone)
+                    when localtime then 0
+                    else Time.now.localtime.utc_offset # utc
+                    end
+
+      strptime = format && (Strptime.new(time_format) rescue nil)
+
+      @parse = case
+               when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
+               when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
+               when strptime then ->(v){ t = strptime.exec(v);         Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+               when format   then ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+               else ->(v){ Fluent::EventTime.parse(v) }
+               end
+    end
+
+    # TODO: new cache mechanism using format string
+    def parse(value)
+      unless value.is_a?(String)
+        raise TimeParseError, "value must be string: #{value}"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      else
+        begin
+          time = @parse.call(value)
+        rescue => e
+          raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+        end
+        @cache1_key = @cache2_key
+        @cache1_time = @cache2_time
+        @cache2_key = value
+        @cache2_time = time
+        return time
+      end
+    end
+    alias :call :parse
+  end
+
+  class NumericTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:parse, method(:parse_unixtime))
+        define_singleton_method(:call, method(:parse_unixtime))
+      else # :float
+        define_singleton_method(:parse, method(:parse_float))
+        define_singleton_method(:call, method(:parse_float))
+      end
+    end
+
+    def parse_unixtime(value)
+      unless value.is_a?(String)
+        raise TimeParseError, "value must be a string: #{value}"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        time = Fluent::EventTime.new(value.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+
+    # rough benchmark result to compare handmade parser vs Fluent::EventTime.from_time(Time.at(value.to_r))
+    # full: with 9-digits of nsec after dot
+    # msec: with 3-digits of msec after dot
+    # 10_000_000 times loop on MacBookAir
+    ## parse_by_myself(full): 12.162475 sec
+    ## parse_by_myself(msec): 15.050435 sec
+    ## parse_by_to_r  (full): 28.722362 sec
+    ## parse_by_to_r  (msec): 28.232856 sec
+    def parse_float(value)
+      unless value.is_a?(String)
+        raise TimeParseError, "value must be a string: #{value}"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        sec_s, nsec_s, _ = value.split('.', 3) # throw away second-dot and later
+        nsec_s = nsec_s[0..9]
+        nsec_s += '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        time = Fluent::EventTime.new(sec_s.to_i, nsec_s.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+  end
+
   class TimeFormatter
-    def initialize(format, localtime, timezone = nil)
+    def initialize(format = nil, localtime = true, timezone = nil)
       @tc1 = 0
       @tc1_str = nil
       @tc2 = 0
@@ -172,4 +374,33 @@ module Fluent
       @format_nocache.call(time)
     end
   end
+
+  class NumericTimeFormatter < TimeFormatter
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:format, method(:format_unixtime))
+        define_singleton_method(:call, method(:format_unixtime))
+      else # :float
+        define_singleton_method(:format, method(:format_float))
+        define_singleton_method(:call, method(:format_float))
+      end
+    end
+
+    def format_unixtime(time)
+      time.to_i.to_s
+    end
+
+    def format_float(time)
+      if time.is_a?(Fluent::EventTime) || time.is_a?(Time)
+        # 10.015 secs for 10_000_000 times call on MacBookAir
+        nsec_s = time.nsec.to_s
+        nsec_s = '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        "#{time.sec}.#{nsec_s}"
+      else # integer (or float?)
+        time.to_f.to_s
+      end
+    end
+  end
 end