fluentd 0.10.52 → 0.10.53

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6871e2cbaea0e7068a4671988446b2cbbf8d5efb
-  data.tar.gz: 7ec21b7f21fa8787b7a577894849df0ea5acf22b
+  metadata.gz: b76b903de67ead21383835723d9b289f437de46e
+  data.tar.gz: b1b787a0cc37983a878136b9731d5ad83eea40cc
 SHA512:
-  metadata.gz: 924081ce1193ab72e08f05bc9ab758716d2aea3d988b1c2f735781eda2432acc78c5fea0bf40032aedc54f72067a5a39784e23cb9fa3a15ac4064792aff45dc7
-  data.tar.gz: 726dbda087652aa3c88f17f3a7f6dcd34d4dc7dab5a576efbf08883f10256f3e2ee3fa0bae375a69a3f769fd47deb36f22bb256f6defbd607f1bd6596649dc1c
+  metadata.gz: 05b4ab5c735e1d6d01c059dde695faa7513180f1520b56f7078eedf9b1491fb00b9fd29aaec0f87f28784750a5c5aa210c5aaff2f5c9fe7740fad857c92b30ee
+  data.tar.gz: 075c930dd72a40427330313e11bf4e3fe5d4399ea91a44df072cbe6fb2605b584549b9075ae55234a999fd737ff805b24f0285969574cca485f360a9b9b6ffa8

data/AUTHORS

@@ -1 +1,2 @@
 FURUHASHI Sadayuki <frsyuki _at_ gmail.com>
+NAKAGAWA Masahiro <repeatedly _at_ gmail.com>

data/ChangeLog

@@ -1,3 +1,17 @@
+Release 0.10.53 - 2014/08/21
+
+* in_tail: Fix forget to detach Closer timer object
+* in_debug_agent: Fix config_param type of unix_path
+* in_syslog: Add include_source_host and source_host_Key options
+* config: Dump V1 escaped parameter correctly at start phase
+* config: Fix bug for config_param of bool value without defaults
+* config: Add 'alias' option to config_section and config_param
+* buffer: Add 'disable_retry_limit' option
+* engine: Reduce memory usage when retry_limit is large
+* parser: Add apache_error format
+* supervisor: Dump Fluentd version at dry run mode
+* test: InputTestDriver wait 'run' until input plugin emits all test data
+
 Release 0.10.52 - 2014/07/15
 
 * in_tail: Fix typo of warning argument. lb to line

data/README.md

@@ -1,10 +1,10 @@
 Fluentd: Open-Source Data Collector
 ===================================
 
-[<img src="https://travis-ci.org/fluent/fluentd.png" />](https://travis-ci.org/fluent/fluentd) [<img src="https://codeclimate.com/github/fluent/fluentd.png " />](https://codeclimate.com/github/fluent/fluentd)
+[<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 
 
-[Fluentd](http://fluentd.org/) collects events from various data sources and writes them to files, database or other types of storages. You can simplify your data stream, and have robust data collection mechanism instantly:
+[Fluentd](http://fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](http://www.fluentd.org/blog/unified-logging-layer)).
 
 <p align="center">
 <img src="http://docs.fluentd.org/images/fluentd-architecture.png" width="500px"/>
@@ -20,6 +20,10 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
     $ fluentd -c conf/fluent.conf &
     $ echo '{"json":"message"}' | fluent-cat debug.test
 
+## Fluentd UI: Admin GUI
+
+[Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
+
 ## More Information
 
 - Website: http://fluentd.org/

data/lib/fluent/config/configure_proxy.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :param_name, :required, :multi, :argument, :params, :defaults, :sections
+      attr_accessor :name, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
       # config_param :desc, :string, :default => '....'
       # config_set_default :buffer_type, :memory
       #
@@ -12,7 +12,7 @@ module Fluent
       #   config_param :power, :integer
       # end
       #
-      # config_section :child, param_name: 'children', required: false, multi: true do
+      # config_section :child, param_name: 'children', required: false, multi: true, alias: 'node' do
       #   config_param :name, :string
       #   config_param :power, :integer, default: nil
       #   config_section :item do
@@ -26,6 +26,7 @@ module Fluent
         @param_name = (opts[:param_name] || @name).to_sym
         @required = opts[:required]
         @multi = opts[:multi]
+        @alias = opts[:alias]
 
         @argument = nil # nil: ignore argument
         @params = {}

data/lib/fluent/config/element.rb

@@ -12,12 +12,14 @@ module Fluent
           self[k] = v
         }
         @unused = unused || attrs.keys
+        @v1_config = false
       end
 
-      attr_accessor :name, :arg, :elements, :unused
+      attr_accessor :name, :arg, :elements, :unused, :v1_config
 
       def add_element(name, arg='')
         e = Element.new(name, arg, {}, [])
+        e.v1_config = @v1_config
         @elements << e
         e
       end
@@ -36,7 +38,9 @@ module Fluent
       end
 
       def +(o)
-        Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
+        e = Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
+        e.v1_config = @v1_config
+        e
       end
 
       def each_element(*names, &block)
@@ -82,7 +86,11 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          out << "#{nindent}#{k} #{v}\n"
+          if @v1_config
+            out << "#{nindent}#{k} #{Element.unescape_parameter(v)}\n"
+          else
+            out << "#{nindent}#{k} #{v}\n"
+          end
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)
@@ -90,6 +98,12 @@ module Fluent
         out << "#{indent}</#{@name}>\n"
         out
       end
+
+      def self.unescape_parameter(v)
+        result = ''
+        v.each_char { |c| result << LiteralParser.unescape_char(c) }
+        result
+      end
     end
   end
 end

data/lib/fluent/config/literal_parser.rb

@@ -21,6 +21,29 @@ module Fluent
     require 'irb/ruby-lex'  # RubyLex
 
     class LiteralParser < BasicParser
+      def self.unescape_char(c)
+        case c
+        when '"'
+          '\"'
+        when "'"
+          "\\'"
+        when '\\'
+          '\\\\'
+        when "\r"
+          '\r'
+        when "\n"
+          '\n'
+        when "\t"
+          '\t'
+        when "\f"
+          '\f'
+        when "\b"
+          '\b'
+        else
+          c
+        end
+      end
+
       def initialize(strscan, eval_context)
         super(strscan)
         @eval_context = eval_context

data/lib/fluent/config/section.rb

@@ -86,7 +86,12 @@ module Fluent
         proxy.params.each_pair do |name, defval|
           varname = name.to_sym
           block, opts = defval
-          if val = conf[name.to_s]
+          if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
+            val = if conf.has_key?(name.to_s)
+                    conf[name.to_s]
+                  else
+                    conf[opts[:alias].to_s]
+                  end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
           end
           unless section_params.has_key?(varname)
@@ -97,7 +102,7 @@ module Fluent
 
         proxy.sections.each do |name, subproxy|
           varname = subproxy.param_name.to_sym
-          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s }
+          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
 
           if subproxy.required? && elements.size < 1
             logger.error "config error in:\n#{conf}"

data/lib/fluent/config/types.rb

@@ -71,7 +71,7 @@ module Fluent
   })
 
   Configurable.register_type(:hash, Proc.new { |val, opts|
-    param = JSON.load(val)
+    param = val.is_a?(String) ? JSON.load(val) : val
     if param.class != Hash
       raise ConfigError, "hash required but got #{val.inspect}"
     end
@@ -79,7 +79,7 @@ module Fluent
   })
 
   Configurable.register_type(:array, Proc.new { |val, opts|
-    param = JSON.load(val)
+    param = val.is_a?(String) ? JSON.load(val) : val
     if param.class != Array
       raise ConfigError, "array required but got #{val.inspect}"
     end

data/lib/fluent/config/v1_parser.rb

@@ -39,6 +39,7 @@ module Fluent
       def parse!
         attrs, elems = parse_element(true, nil)
         root = Element.new('ROOT', '', attrs, elems)
+        root.v1_config = true
 
         spacing
         unless eof?
@@ -86,7 +87,9 @@ module Fluent
             e_arg ||= ''
             # call parse_element recursively
             e_attrs, e_elems = parse_element(false, e_name)
-            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+            new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
+            new_e.v1_config = true
+            elems << new_e
 
           elsif root_element && skip(/(\@include|include)#{SPACING}/)
             if !prev_match.start_with?('@')

data/lib/fluent/event.rb

@@ -23,8 +23,9 @@ module Fluent
       false
     end
 
-    #def each(&block)
-    #end
+    def each(&block)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
 
     def to_msgpack_stream
       out = ''
@@ -56,7 +57,10 @@ module Fluent
     end
   end
 
-
+  # EventStream from entries: Array of [time, record]
+  #
+  # Use this class for many events data with a tag
+  # and its representation is [ [time, record], [time, record], .. ]
   class ArrayEventStream < EventStream
     def initialize(entries)
       @entries = entries
@@ -79,15 +83,18 @@ module Fluent
       @entries.each(&block)
       nil
     end
-
-    #attr_reader :entries
-    #
-    #def to_a
-    #  @entries
-    #end
   end
 
-
+  # EventStream from entries: numbers of pairs of time and record.
+  #
+  # This class can handle many events more efficiently than ArrayEventStream
+  # because this class generate less objects than ArrayEventStream.
+  #
+  # Use this class as below, in loop of data-enumeration:
+  #  1. initialize blank stream:
+  #     streams[tag] ||= MultiEventStream
+  #  2. add events
+  #     stream[tag].add(time, record)
   class MultiEventStream < EventStream
     def initialize
       @time_array = []
@@ -173,25 +180,5 @@ module Fluent
     end
 
   end
-
-  #class IoEventStream < EventStream
-  #  def initialize(io, num)
-  #    @io = io
-  #    @num = num
-  #    @u = MessagePack::Unpacker.new(@io)
-  #  end
-  #
-  #  def repeatable?
-  #    false
-  #  end
-  #
-  #  def each(&block)
-  #    return nil if @num == 0
-  #    @u.each {|obj|
-  #      block.call(obj[0], obj[1])
-  #      break if @array.size >= @num
-  #    }
-  #  end
-  #end
 end
 

data/lib/fluent/mixin.rb

@@ -120,46 +120,33 @@ module Fluent
     def configure(conf)
       super
 
+      @include_time_key = false
+
       if s = conf['include_time_key']
-        b = Config.bool_value(s)
-        if s.empty?
-          b = true
-        elsif b == nil
-          raise ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter"
-        end
-        @include_time_key = b
+        include_time_key = Config.bool_value(s)
+        raise ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil?
+
+        @include_time_key = include_time_key
       end
 
       if @include_time_key
-        if time_key = conf['time_key']
-          @time_key = time_key
-        end
-        unless @time_key
-          @time_key = 'time'
-        end
+        @time_key     = conf['time_key'] || 'time'
+        @time_format  = conf['time_format']
 
-        if time_format = conf['time_format']
-          @time_format = time_format
-        end
-
-        if localtime = conf['localtime']
+        if    conf['localtime']
           @localtime = true
-        elsif utc = conf['utc']
+        elsif conf['utc']
           @localtime = false
         end
 
         @timef = TimeFormatter.new(@time_format, @localtime)
-
-      else
-        @include_time_key = false
       end
     end
 
     def filter_record(tag, time, record)
       super
-      if @include_time_key
-        record[@time_key] = @timef.format(time)
-      end
+
+      record[@time_key] = @timef.format(time) if @include_time_key
     end
   end
 
@@ -171,34 +158,22 @@ module Fluent
     def configure(conf)
       super
 
-      if s = conf['include_tag_key']
-        b = Config.bool_value(s)
-        if s.empty?
-          b = true
-        elsif b == nil
-          raise ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter"
-        end
-        @include_tag_key = b
-      end
+      @include_tag_key = false
 
-      if @include_tag_key
-        if tag_key = conf['tag_key']
-          @tag_key = tag_key
-        end
-        unless @tag_key
-          @tag_key = 'tag'
-        end
+      if s = conf['include_tag_key']
+        include_tag_key = Config.bool_value(s)
+        raise ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter" if include_tag_key.nil?
 
-      else
-        @include_tag_key = false
+        @include_tag_key = include_tag_key
       end
+
+      @tag_key = conf['tag_key'] || 'tag' if @include_tag_key
     end
 
     def filter_record(tag, time, record)
       super
-      if @include_tag_key
-        record[@tag_key] = tag
-      end
+
+      record[@tag_key] = tag if @include_tag_key
     end
   end
 end

data/lib/fluent/output.rb

@@ -165,8 +165,8 @@ module Fluent
       @next_flush_time = 0
       @last_retry_time = 0
       @next_retry_time = 0
-      @error_history = []
-      @error_history.extend(MonitorMixin)
+      @num_errors = 0
+      @num_errors_lock = Mutex.new
       @secondary_limit = 8
       @emit_count = 0
     end
@@ -174,6 +174,7 @@ module Fluent
     config_param :buffer_type, :string, :default => 'memory'
     config_param :flush_interval, :time, :default => 60
     config_param :try_flush_interval, :float, :default => 1
+    config_param :disable_retry_limit, :bool, :default => false
     config_param :retry_limit, :integer, :default => 17
     config_param :retry_wait, :time, :default => 1.0
     config_param :max_retry_wait, :time, :default => nil
@@ -286,11 +287,11 @@ module Fluent
       end
 
       begin
-        retrying = !@error_history.empty?
+        retrying = !@num_errors.zero?
 
         if retrying
-          @error_history.synchronize do
-            if retrying = !@error_history.empty?  # re-check in synchronize
+          @num_errors_lock.synchronize do
+            if retrying = !@num_errors.zero? # re-check in synchronize
               if @next_retry_time >= time
                 # allow retrying for only one thread
                 return time + @try_flush_interval
@@ -298,13 +299,13 @@ module Fluent
               # assume next retry failes and
               # clear them if when it succeeds
               @last_retry_time = time
-              @error_history << time
+              @num_errors += 1
               @next_retry_time += calc_retry_wait
             end
           end
         end
 
-        if @secondary && @error_history.size > @retry_limit
+        if @secondary && !@disable_retry_limit && @num_errors > @retry_limit
           has_next = flush_secondary(@secondary)
         else
           has_next = @buffer.pop(self)
@@ -312,7 +313,7 @@ module Fluent
 
         # success
         if retrying
-          @error_history.clear
+          @num_errors = 0
           # Note: don't notify to other threads to prevent
           #       burst to recovered server
           $log.warn "retry succeeded.", :instance=>object_id
@@ -326,20 +327,20 @@ module Fluent
 
       rescue => e
         if retrying
-          error_count = @error_history.size
+          error_count = @num_errors
         else
           # first error
           error_count = 0
-          @error_history.synchronize do
-            if @error_history.empty?
+          @num_errors_lock.synchronize do
+            if @num_errors.zero?
               @last_retry_time = time
-              @error_history << time
+              @num_errors += 1
               @next_retry_time = time + calc_retry_wait
             end
           end
         end
 
-        if error_count < @retry_limit
+        if @disable_retry_limit || error_count < @retry_limit
           $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
           $log.warn_backtrace e.backtrace
 
@@ -357,7 +358,7 @@ module Fluent
             $log.warn "secondary retry count exceededs limit."
             $log.warn_backtrace e.backtrace
             write_abort
-            @error_history.clear
+            @num_errors = 0
           end
 
         else
@@ -365,7 +366,7 @@ module Fluent
           $log.warn "retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
-          @error_history.clear
+          @num_errors = 0
         end
 
         return @next_retry_time
@@ -388,11 +389,11 @@ module Fluent
 
     def calc_retry_wait
       # TODO retry pattern
-      wait = if @error_history.size <= @retry_limit
-               @retry_wait * (2 ** (@error_history.size-1))
+      wait = if @disable_retry_limit || @num_errors <= @retry_limit
+               @retry_wait * (2 ** (@num_errors - 1))
              else
                # secondary retry
-               @retry_wait * (2 ** (@error_history.size-2-@retry_limit))
+               @retry_wait * (2 ** (@num_errors - 2 - @retry_limit))
              end
       retry_wait = wait + (rand * (wait / 4.0) - (wait / 8.0))
       @max_retry_wait ? [retry_wait, @max_retry_wait].min : retry_wait

data/lib/fluent/parser.rb

@@ -257,8 +257,6 @@ module Fluent
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      # 'configure()' may raise errors for unexpected configurations
       attr_accessor :estimate_current_event
 
       def initialize
@@ -624,6 +622,7 @@ module Fluent
     TEMPLATE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/parser_')
     {
       'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+      'apache_error' => Proc.new { RegexpParser.new(/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/) },
       'apache2' => Proc.new { ApacheParser.new },
       'syslog' => Proc.new { SyslogParser.new },
       'json' => Proc.new { JSONParser.new },
@@ -650,10 +649,15 @@ module Fluent
 
     def initialize
       @parser = nil
+      @estimate_current_event = nil
     end
 
     attr_reader :parser
 
+    # SET false BEFORE CONFIGURE, to return nil when time not parsed
+    # 'configure()' may raise errors for unexpected configurations
+    attr_accessor :estimate_current_event
+
     def configure(conf, required=true)
       format = conf['format']
 
@@ -687,6 +691,10 @@ module Fluent
         @parser = factory.call
       end
 
+      if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
+        @parser.estimate_current_event = @estimate_current_event
+      end
+
       if @parser.respond_to?(:configure)
         @parser.configure(conf)
       end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -26,7 +26,7 @@ module Fluent
 
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :port, :integer, :default => 24230
-    config_param :unix_path, :integer, :default => nil
+    config_param :unix_path, :string, :default => nil
     #config_param :unix_mode  # TODO
     config_param :object, :string, :default => 'Engine'
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -217,7 +217,7 @@ module Fluent
       'output_plugin' => 'is_a?(::Fluent::Output)',
       'buffer_queue_length' => '@buffer.queue_size',
       'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
-      'retry_count' => '@error_history.size',
+      'retry_count' => '@num_errors',
       'config' => 'config',
     }
 

data/lib/fluent/plugin/in_syslog.rb

@@ -78,6 +78,8 @@ module Fluent
         raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
       end
     end
+    config_param :include_source_host, :bool, :default => false
+    config_param :source_host_key, :string, :default => 'source_host'.freeze
 
     def configure(conf)
       super
@@ -137,6 +139,7 @@ module Fluent
           return
         end
 
+        record[@source_host_key] = addr[2] if @include_source_host
         emit(pri, time, record)
       }
     rescue => e
@@ -152,6 +155,7 @@ module Fluent
         end
 
         pri = record.delete('pri')
+        record[@source_host_key] = addr[2] if @include_source_host
         emit(pri, time, record)
       }
     rescue => e

data/lib/fluent/plugin/in_tail.rb

@@ -462,6 +462,8 @@ module Fluent
         rescue => e
           @log.error e.to_s
           @log.error_backtrace(e.backtrace)
+        ensure
+          detach
         end
       end
 

data/lib/fluent/supervisor.rb

@@ -151,6 +151,8 @@ module Fluent
     private
 
     def dry_run
+      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
+
       read_config
       change_privilege
       init_engine
@@ -158,7 +160,7 @@ module Fluent
       run_configure
       exit 0
     rescue => e
-      $log.error "Dry run failed: #{e}"
+      $log.error "dry run failed: #{e}"
       exit 1
     end
 
@@ -358,6 +360,7 @@ module Fluent
       end
     end
 
+    # TODO: this method should be moved to SystemConfig class method
     def apply_system_config(opt)
       # Create NULL file to avoid $log uninitialized problem before call @log.init
       file = File.open(File::NULL)

data/lib/fluent/test/input_test.rb

@@ -29,6 +29,9 @@ module Fluent
         super(klass, &block)
         @emit_streams = []
         @expects = nil
+        # for checking only the number of emitted records during run
+        @expected_emits_length = nil
+        @run_timeout = 5
       end
 
       def expect_emit(tag, time, record)
@@ -40,6 +43,8 @@ module Fluent
         @expects ||= []
       end
 
+      attr_accessor :expected_emits_length
+      attr_accessor :run_timeout
       attr_reader :emit_streams
 
       def emits
@@ -78,15 +83,24 @@ module Fluent
         super {
           block.call if block
 
-          if @expects
-            i = 0
-            @emit_streams.each {|tag,events|
-              events.each {|time,record|
-                assert_equal(@expects[i], [tag, time, record])
+          if @expected_emits_length || @expects
+            max_length = @expected_emits_length || @expects.length
+            started_at = Time.now
+            i, j = 0, 0
+            while i < max_length && Time.now <= started_at + @run_timeout
+              if j >= @emit_streams.length
+                sleep 0.01
+                next
+              end
+
+              tag, events = @emit_streams[j]
+              events.each do |time, record|
+                assert_equal(@expects[i], [tag, time, record]) if @expects
                 i += 1
-              }
-            }
-            assert_equal @expects.length, i
+              end
+              j += 1
+            end
+            assert_equal(@expects.length, i) if @expects
           end
         }
         self

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.52'
+  VERSION = '0.10.53'
 
 end

data/spec/config/config_parser_spec.rb

@@ -299,5 +299,16 @@ describe Fluent::Config::V1Parser do
 
     # TODO: Add uri based include spec
   end
-end
 
+  describe 'unescape parameter' do
+    it 'parses dumpped configuration' do
+      original = %q!a\\\\\n\r\f\b\\'\\"z!
+      expected = %!a\\\n\r\f\b'"z!
+
+      conf = parse_text(%[k1 #{original}])
+      expect(conf['k1']).to eq(expected) # escape check
+      conf2 = parse_text(conf.to_s) # use dumpped configuration to check unescape
+      expect(conf2.elements.first['k1']).to eq(expected)
+    end
+  end
+end

data/spec/config/configurable_spec.rb

@@ -80,6 +80,33 @@ module ConfigurableSpec
       ary + [@nodes, @description1, @description2, @description3]
     end
   end
+
+  class Example0
+    include Fluent::Configurable
+
+    config_param :stringvalue, :string
+    config_param :boolvalue, :bool
+    config_param :integervalue, :integer
+    config_param :sizevalue, :size
+    config_param :timevalue, :time
+    config_param :floatvalue, :float
+    config_param :hashvalue, :hash
+    config_param :arrayvalue, :array
+  end
+
+  class Example1
+    include Fluent::Configurable
+
+    config_param :name, :string, alias: :fullname
+    config_param :bool, :bool, alias: :flag
+    config_section :detail, required: true, multi: false, alias: "information" do
+      config_param :address, :string
+    end
+
+    def get_all
+      [@name, @detail]
+    end
+  end
 end
 
 describe Fluent::Configurable do
@@ -111,6 +138,11 @@ describe Fluent::Configurable do
     end
 
     describe '#configure' do
+      it 'returns configurable object itself' do
+        b2 = ConfigurableSpec::Base2.new
+        expect(b2.configure({"name1" => "t1", "name5" => "t5"})).to be_a(ConfigurableSpec::Base2)
+      end
+
       it 'raise errors without any specifications for param without defaults' do
         b2 = ConfigurableSpec::Base2.new
         expect{ b2.configure({}) }.to raise_error(Fluent::ConfigError)
@@ -128,7 +160,7 @@ describe Fluent::Configurable do
         expect(b2a.flag2).to eq(true)
 
         b2b = ConfigurableSpec::Base2.new
-        expect{ b2b.configure({"flag1" => "false", "flag2" => "no", "name1" => "t1", "name5" => "t5"}) }.not_to raise_error()
+        expect{ b2b.configure({"flag1" => false, "flag2" => "no", "name1" => "t1", "name5" => "t5"}) }.not_to raise_error()
         expect(b2b.flag1).to eq(false)
         expect(b2b.flag2).to eq(false)
       end
@@ -368,6 +400,125 @@ describe Fluent::Configurable do
         expect(b4.description3[2].note).to eql('desc3: d-3a')
         expect(b4.description3[2].text).to eql('dddd3-3')
       end
+
+      it 'checks missing of specifications' do
+        conf0 = e('ROOT', '', {}, [])
+        ex01 = ConfigurableSpec::Example0.new
+        expect { ex01.configure(conf0) }.to raise_error(Fluent::ConfigError)
+
+        complete = {
+          "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10",
+          "sizevalue" => "10m", "timevalue" => "100s", "floatvalue" => "1.001",
+          "hashvalue" => '{"foo":1, "bar":2}',
+          "arrayvalue" => '[1,"ichi"]',
+        }
+
+        checker = lambda {|conf| ConfigurableSpec::Example0.new.configure(conf) }
+
+        expect { checker.call(complete) }.not_to raise_error()
+        expect { checker.call(complete.reject{|k,v| k == "stringvalue" }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "boolvalue"   }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "integervalue"}) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "sizevalue"   }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "timevalue"   }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "floatvalue"  }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "hashvalue"   }) }.to raise_error(Fluent::ConfigError)
+        expect { checker.call(complete.reject{|k,v| k == "arrayvalue"  }) }.to raise_error(Fluent::ConfigError)
+      end
+
+      it 'accepts configuration values as string representation' do
+        conf = {
+          "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10",
+          "sizevalue" => "10m", "timevalue" => "10m", "floatvalue" => "1.001",
+          "hashvalue" => '{"foo":1, "bar":2}',
+          "arrayvalue" => '[1,"ichi"]',
+        }
+        ex = ConfigurableSpec::Example0.new.configure(conf)
+        expect(ex.stringvalue).to eq("s1")
+        expect(ex.boolvalue).to equal(true)
+        expect(ex.integervalue).to eq(10)
+        expect(ex.sizevalue).to eq(10 * 1024 * 1024)
+        expect(ex.timevalue).to eq(10 * 60)
+        expect(ex.floatvalue).to eq(1.001)
+        expect(ex.hashvalue).to eq({"foo" => 1, "bar" => 2})
+        expect(ex.arrayvalue).to eq([1, "ichi"])
+      end
+
+      it 'accepts configuration values as ruby value representation (especially for DSL)' do
+        conf = {
+          "stringvalue" => "s1", "boolvalue" => true, "integervalue" => 10,
+          "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001,
+          "hashvalue" => {"foo" => 1, "bar" => 2},
+          "arrayvalue" => [1,"ichi"],
+        }
+        ex = ConfigurableSpec::Example0.new.configure(conf)
+        expect(ex.stringvalue).to eq("s1")
+        expect(ex.boolvalue).to equal(true)
+        expect(ex.integervalue).to eq(10)
+        expect(ex.sizevalue).to eq(10 * 1024 * 1024)
+        expect(ex.timevalue).to eq(10 * 60)
+        expect(ex.floatvalue).to eq(1.001)
+        expect(ex.hashvalue).to eq({"foo" => 1, "bar" => 2})
+        expect(ex.arrayvalue).to eq([1, "ichi"])
+      end
+
+      it 'gets both of true(yes) and false(no) for bool value parameter' do
+        conf = {
+          "stringvalue" => "s1", "integervalue" => 10,
+          "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001,
+          "hashvalue" => {"foo" => 1, "bar" => 2},
+          "arrayvalue" => [1,"ichi"],
+        }
+        ex0 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "true"}))
+        expect(ex0.boolvalue).to equal(true)
+
+        ex1 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "yes"}))
+        expect(ex1.boolvalue).to equal(true)
+
+        ex2 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => true}))
+        expect(ex2.boolvalue).to equal(true)
+
+        ex3 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "false"}))
+        expect(ex3.boolvalue).to equal(false)
+
+        ex4 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "no"}))
+        expect(ex4.boolvalue).to equal(false)
+
+        ex5 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => false}))
+        expect(ex5.boolvalue).to equal(false)
+      end
+    end
+  end
+
+  context 'class defined with config_param/config_section having :alias' do
+    describe '#initialize' do
+      it 'does not create methods for alias' do
+        ex1 = ConfigurableSpec::Example1.new
+        expect { ex1.name }.not_to raise_error()
+        expect { ex1.fullname }.to raise_error(NoMethodError)
+        expect { ex1.bool }.not_to raise_error()
+        expect { ex1.flag }.to raise_error(NoMethodError)
+        expect { ex1.detail }.not_to raise_error()
+        expect { ex1.information}.to raise_error(NoMethodError)
+      end
+    end
+
+    describe '#configure' do
+      def e(name, arg = '', attrs = {}, elements = [])
+        attrs_str_keys = {}
+        attrs.each{|key, value| attrs_str_keys[key.to_s] = value }
+        Fluent::Config::Element.new(name, arg, attrs_str_keys, elements)
+      end
+
+      it 'provides accessible data for alias attribute keys' do
+        ex1 = ConfigurableSpec::Example1.new
+        ex1.configure(e('ROOT', '', {"fullname" => "foo bar", "bool" => false}, [e('information', '', {"address" => "Mountain View 0"})]))
+        expect(ex1.name).to eql("foo bar")
+        expect(ex1.bool).not_to be_nil
+        expect(ex1.bool).to be_falsy
+        expect(ex1.detail).not_to be_nil
+        expect(ex1.detail.address).to eql("Mountain View 0")
+      end
     end
   end
 end

data/spec/config/system_config_spec.rb

@@ -0,0 +1,49 @@
+require 'fluent/configurable'
+require 'fluent/config/element'
+require 'fluent/config/section'
+require 'fluent/supervisor'
+
+describe Fluent::Supervisor::SystemConfig do
+  def parse_text(text)
+    basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+    Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+  end
+
+  it 'should not override default configurations when no parameters' do
+    conf = parse_text(<<EOS)
+<system>
+</system>
+EOS
+    sc = Fluent::Supervisor::SystemConfig.new(conf)
+    expect(sc.log_level).to be_nil
+    expect(sc.suppress_repeated_stacktrace).to be_nil
+    expect(sc.emit_error_log_interval).to be_nil
+    expect(sc.suppress_config_dump).to be_nil
+    expect(sc.without_source).to be_nil
+    expect(sc.to_opt).to eql({})
+  end
+
+  {'log_level' => 'error', 'suppress_repeated_stacktrace' => true, 'emit_error_log_interval' => 60, 'suppress_config_dump' => true, 'without_source' => true}.each { |k, v|
+    it "accepts #{k} parameter" do
+      conf = parse_text(<<EOS)
+<system>
+  #{k} #{v}
+</system>
+EOS
+      sc = Fluent::Supervisor::SystemConfig.new(conf)
+      expect(sc.instance_variable_get("@#{k}")).not_to be_nil
+      if k == 'emit_error_log_interval'
+        expect(sc.to_opt).to include(:suppress_interval)
+      else
+        expect(sc.to_opt).to include(k.to_sym)
+      end
+    end
+  }
+
+  {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
+    it "should not affect settable parameters with unknown #{k} parameter" do
+      sc = Fluent::Supervisor::SystemConfig.new({k => v})
+      expect(sc.to_opt).to be_empty
+    end
+  }
+end

data/test/plugin/test_in_forward.rb

@@ -44,7 +44,6 @@ class ForwardInputTest < Test::Unit::TestCase
       d.expected_emits.each {|tag,time,record|
         send_data [tag, 0, record].to_msgpack
       }
-      sleep 0.5
     end
   end
 
@@ -60,7 +59,6 @@ class ForwardInputTest < Test::Unit::TestCase
       d.expected_emits.each {|tag,time,record|
         send_data [tag, time, record].to_msgpack
       }
-      sleep 0.5
     end
   end
 
@@ -78,7 +76,6 @@ class ForwardInputTest < Test::Unit::TestCase
         entries << [time, record]
       }
       send_data ["tag1", entries].to_msgpack
-      sleep 0.5
     end
   end
 
@@ -96,7 +93,6 @@ class ForwardInputTest < Test::Unit::TestCase
         [time, record].to_msgpack(entries)
       }
       send_data ["tag1", entries].to_msgpack
-      sleep 0.5
     end
   end
 
@@ -112,7 +108,6 @@ class ForwardInputTest < Test::Unit::TestCase
       d.expected_emits.each {|tag,time,record|
         send_data [tag, time, record].to_json
       }
-      sleep 0.5
     end
   end
 

data/test/plugin/test_in_object_space.rb

@@ -33,15 +33,15 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
 
     time = Time.parse("2011-01-02 13:14:15").to_i
 
-    d.run do
-      sleep 2
-    end
+    d.expected_emits_length = 2
+    d.run
 
     emits = d.emits
     assert_equal true, emits.length > 0
 
-    tag, time, record = emits[0]
-    assert_equal 2, record.size
+    emits.each { |tag, time, record|
+      assert_equal d.instance.tag, tag
+      assert_equal d.instance.top, record.keys.size
+    }
   end
 end
-

data/test/plugin/test_in_stream.rb

@@ -19,7 +19,6 @@ module StreamInputTest
       d.expected_emits.each {|tag,time,record|
         send_data [tag, 0, record].to_msgpack
       }
-      sleep 0.5
     end
   end
 
@@ -35,7 +34,6 @@ module StreamInputTest
       d.expected_emits.each {|tag,time,record|
         send_data [tag, time, record].to_msgpack
       }
-      sleep 0.5
     end
   end
 
@@ -53,7 +51,6 @@ module StreamInputTest
         entries << [time, record]
       }
       send_data ["tag1", entries].to_msgpack
-      sleep 0.5
     end
   end
 
@@ -71,7 +68,6 @@ module StreamInputTest
         [time, record].to_msgpack(entries)
       }
       send_data ["tag1", entries].to_msgpack
-      sleep 0.5
     end
   end
 
@@ -87,7 +83,6 @@ module StreamInputTest
       d.expected_emits.each {|tag,time,record|
         send_data [tag, time, record].to_json
       }
-      sleep 0.5
     end
   end
 

data/test/plugin/test_in_syslog.rb

@@ -131,6 +131,24 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests)
   end
 
+  def test_msg_size_with_include_source_host
+    d = create_driver([CONFIG, 'include_source_host'].join("\n"))
+    tests = create_test_case
+
+    host = nil
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      host = u.peeraddr[2]
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests, host)
+  end
+
   def create_test_case
     # actual syslog message has "\n"
     [
@@ -139,10 +157,11 @@ class SyslogInputTest < Test::Unit::TestCase
     ]
   end
 
-  def compare_test_result(emits, tests)
+  def compare_test_result(emits, tests, host = nil)
     emits.each_index { |i|
       assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert_equal(host, emits[i][2]['source_host']) if host
     }
   end
 end

data/test/test_output.rb

@@ -27,6 +27,7 @@ module FluentOutputTest
       d = create_driver
       assert_equal 'memory', d.instance.buffer_type
       assert_equal 60, d.instance.flush_interval
+      assert_equal false, d.instance.disable_retry_limit
       assert_equal 17, d.instance.retry_limit
       assert_equal 1.0, d.instance.retry_wait
       assert_equal nil, d.instance.max_retry_wait
@@ -37,18 +38,27 @@ module FluentOutputTest
       # max_retry_wait
       d = create_driver(CONFIG + %[max_retry_wait 4])
       assert_equal 4, d.instance.max_retry_wait
+
+      # disable_retry_limit
+      d = create_driver(CONFIG + %[disable_retry_limit true])
+      assert_equal true, d.instance.disable_retry_limit
     end
 
     def test_calc_retry_wait
       # default
       d = create_driver
-      d.instance.retry_limit.times { d.instance.instance_variable_get(:@error_history) << Engine.now }
+      d.instance.retry_limit.times {
+        # "d.instance.instance_variable_get(:@num_errors) += 1" causes SyntaxError
+        d.instance.instance_eval { @num_errors += 1 }
+      }
       wait = d.instance.retry_wait * (2 ** (d.instance.retry_limit - 1))
       assert( d.instance.calc_retry_wait > wait - wait / 8.0 )
 
       # max_retry_wait
       d = create_driver(CONFIG + %[max_retry_wait 4])
-      d.instance.retry_limit.times { d.instance.instance_variable_get(:@error_history) << Engine.now }
+      d.instance.retry_limit.times {
+        d.instance.instance_eval { @num_errors += 1 }
+      }
       assert_equal 4, d.instance.calc_retry_wait
     end
 

data/test/test_parser.rb

@@ -130,6 +130,43 @@ module ParserTest
     end
   end
 
+  class ApacheErrorParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::TEMPLATE_REGISTRY.lookup('apache_error').call
+      @expected = {
+        'level' => 'error',
+        'client' => '127.0.0.1',
+        'message' => 'client denied by server configuration'
+      }
+    end
+
+    def test_call
+      @parser.call('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal(@expected, record)
+      }
+    end
+
+    def test_call_with_pid
+      @parser.call('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal(@expected.merge('pid' => '1000'), record)
+      }
+    end
+
+    def test_call_without_client
+      @parser.call('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal({
+          'level' => 'notice',
+          'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
+        }, record)
+      }
+    end
+  end
+
   class Apache2ParserTest < ::Test::Unit::TestCase
     include ParserTest
 
@@ -656,5 +693,23 @@ EOS
         i += 1
       }
     end
+
+    def test_setting_estimate_current_event_value
+      p1 = TextParser.new
+      assert_nil p1.estimate_current_event
+      assert_nil p1.parser
+
+      p1.configure('format' => 'none')
+      assert_equal true, p1.parser.estimate_current_event
+
+      p2 = TextParser.new
+      assert_nil p2.estimate_current_event
+      assert_nil p2.parser
+
+      p2.estimate_current_event = false
+
+      p2.configure('format' => 'none')
+      assert_equal false, p2.parser.estimate_current_event
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.52
+  version: 0.10.53
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-14 00:00:00.000000000 Z
+date: 2014-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -341,6 +341,7 @@ files:
 - spec/config/helper.rb
 - spec/config/literal_parser_spec.rb
 - spec/config/section_spec.rb
+- spec/config/system_config_spec.rb
 - spec/spec_helper.rb
 - test/helper.rb
 - test/plugin/data/2010/01/20100102-030405.log
@@ -409,6 +410,7 @@ test_files:
 - spec/config/helper.rb
 - spec/config/literal_parser_spec.rb
 - spec/config/section_spec.rb
+- spec/config/system_config_spec.rb
 - spec/spec_helper.rb
 - test/helper.rb
 - test/plugin/data/2010/01/20100102-030405.log