fluentd 0.12.19 → 0.12.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 251e0a23a2109a9b8434de8be77676e4acb84b0f
-  data.tar.gz: 495d9a6ca6baa5266da3c04f11f5b65336252674
+  metadata.gz: 4968b695bb1d272edf951c51e3abb784431bc722
+  data.tar.gz: 69f6d6745d9d9a09201477c9f8f84e92734b03eb
 SHA512:
-  metadata.gz: 9799adf3dbb5078f797aedf12275283b153e8201c1ab6fa4310bf80e9460911f3077499def817a82f2709cbc4a9fda82be1f88b7b0149b17d45b98cf4f8c237a
-  data.tar.gz: a1ef697d0b1ae85a414b9a3f65aaef5b108fea491c13edf0f9c17ab965d2dfd2606b276d81762c5d59f185aaba25bb74424b40d04048608dcb6b937d69411c2d
+  metadata.gz: 26de5771b3a3c686cdfc868d633bee44bc4748713327e57cfe51da3d6a2ba4e5cae28d6bc8b03e0b4419241dc60032bf653b750c8c31b11b48adf3058773fd04
+  data.tar.gz: 65245255419ef81be6697494aa4b2264011d7cbf2e89a2d3c6f7f3a767db68c2caa05c7ae267e9bce1988406e684a9f60e162706d841890c547b5ca40c417757

data/.travis.yml

@@ -5,6 +5,7 @@ rvm:
   - 2.0.0
   - 2.1
   - 2.2.3
+  - 2.3.0
   - ruby-head
   - rbx-2
 
@@ -30,3 +31,4 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-2
+    - rvm: 2.3.0

data/ChangeLog

@@ -1,5 +1,28 @@
 # v0.12
 
+## Release 0.12.20 - 2015/01/26
+
+### New features / Enhancement
+
+* in_forward: Add skip_invalid_event paramter to check and skip invalid event
+  https://github.com/fluent/fluentd/pull/776
+* in_tail: Add multiline_flush_interval parameter for periodic flush with multiline format
+  https://github.com/fluent/fluentd/pull/775
+* filter_record_transformer: Improve ruby placeholder performance and adding 'record["key"]' syntax
+  https://github.com/fluent/fluentd/pull/766
+* Add on_exit_process hook point to DetachProcessMixin
+  https://github.com/fluent/fluentd/pull/757
+* Add descriptions to BufferedOutput and TimeSlicedOutput
+  https://github.com/fluent/fluentd/pull/759
+  https://github.com/fluent/fluentd/pull/760
+
+### Bug fixes
+
+* parser: Don't use BigDecimal in JSON parsing result with Oj
+  https://github.com/fluent/fluentd/pull/778
+* config: Fix the regression of unused parameter warning inside sub section
+  https://github.com/fluent/fluentd/pull/765
+
 ## Release 0.12.19 - 2015/12/21
 
 ### New features / Enhancement

data/lib/fluent/config/element.rb

@@ -75,11 +75,13 @@ module Fluent
       end
 
       def has_key?(key)
+        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in chgeck_not_fetched.
         @unused.delete(key)
         super
       end
 
       def [](key)
+        @unused_in = false # ditto
         @unused.delete(key)
         super
       end

data/lib/fluent/output.rb

@@ -179,14 +179,22 @@ module Fluent
       @emit_count = 0
     end
 
+    desc 'The buffer type (memory, file)'
     config_param :buffer_type, :string, :default => 'memory'
+    desc 'The interval between data flushes.'
     config_param :flush_interval, :time, :default => 60
     config_param :try_flush_interval, :float, :default => 1
+    desc 'If true, the value of `retry_value` is ignored and there is no limit'
     config_param :disable_retry_limit, :bool, :default => false
+    desc 'The limit on the number of retries before buffered data is discarded'
     config_param :retry_limit, :integer, :default => 17
+    desc 'The initial intervals between write retries.'
     config_param :retry_wait, :time, :default => 1.0
+    desc 'The maximum intervals between write retries.'
     config_param :max_retry_wait, :time, :default => nil
+    desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, :default => 1
+    desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, :default => 1
 
     def configure(conf)
@@ -474,8 +482,11 @@ module Fluent
       #@ignore_old = false   # TODO
     end
 
+    desc 'The time format used as part of the file name.'
     config_param :time_slice_format, :string, :default => '%Y%m%d'
+    desc 'The amount of time Fluentd will wait for old logs to arrive.'
     config_param :time_slice_wait, :time, :default => 10*60
+    desc 'Parse the time value in the specified timezone'
     config_param :timezone, :string, :default => nil
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit

data/lib/fluent/parser.rb

@@ -250,6 +250,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
+          Oj.default_options = {:bigdecimal_load => :float}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -41,12 +41,12 @@ module Fluent
     def configure(conf)
       super
 
-      @map = {}
+      map = {}
       # <record></record> directive
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @map[k] = parse_value(v)
+          map[k] = parse_value(v)
         end
       end
 
@@ -73,6 +73,7 @@ module Fluent
         else
           PlaceholderExpander.new(placeholder_expander_params)
         end
+      @map = @placeholder_expander.preprocess_map(map)
 
       @hostname = Socket.gethostname
     end
@@ -82,17 +83,21 @@ module Fluent
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)
       tag_suffix = tag_suffix(tag_parts)
-      placeholders = {
-        'tag' => tag,
-        'tag_parts' => tag_parts,
+      placeholder_values = {
+        'tag'        => tag,
+        'tag_parts'  => tag_parts,
         'tag_prefix' => tag_prefix,
         'tag_suffix' => tag_suffix,
-        'hostname' => @hostname,
+        'hostname'   => @hostname,
       }
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        new_record = reform(time, record, placeholders)
+        placeholder_values.merge!({
+          'time'     => @placeholder_expander.time_value(time),
+          'record'   => record,
+        })
+        new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
         end
@@ -102,7 +107,7 @@ module Fluent
     rescue => e
       log.warn "failed to reform records", :error_class => e.class, :error => e.message
       log.warn_backtrace
-      log.debug "map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+      log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
 
     private
@@ -118,8 +123,8 @@ module Fluent
       value_str # emit as string
     end
 
-    def reform(time, record, opts)
-      @placeholder_expander.prepare_placeholders(time, record, opts)
+    def reform(record, placeholder_values)
+      @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
@@ -175,17 +180,29 @@ module Fluent
         @auto_typecast = params[:auto_typecast]
       end
 
-      def prepare_placeholders(time, record, opts)
-        placeholders = { '${time}' => Time.at(time).to_s }
-        record.each {|key, value| placeholders.store("${#{key}}", value) }
+      def time_value(time)
+        Time.at(time).to_s
+      end
+
+      def preprocess_map(value, force_stringify = false)
+        value
+      end
 
-        opts.each do |key, value|
+      def prepare_placeholders(placeholder_values)
+        placeholders = {}
+
+        placeholder_values.each do |key, value|
           if value.kind_of?(Array) # tag_parts, etc
             size = value.size
-            value.each_with_index { |v, idx|
+            value.each_with_index do |v, idx|
               placeholders.store("${#{key}[#{idx}]}", v)
               placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
-            }
+            end
+          elsif value.kind_of?(Hash) # record, etc
+            value.each do |k, v|
+              placeholders.store("${#{k}}", v) # foo
+              placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
+            end
           else # string, interger, float, and others?
             placeholders.store("${#{key}}", value)
           end
@@ -194,22 +211,27 @@ module Fluent
         @placeholders = placeholders
       end
 
-      def expand(str, force_stringify=false)
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def expand(str, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_unknown_placeholder($1)
+            log_if_unknown_placeholder($1)
             return @placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_unknown_placeholder($1)
+          log_if_unknown_placeholder($1)
           @placeholders[$1]
         }
       end
 
       private
-      def log_unknown_placeholder(placeholder)
+
+      def log_if_unknown_placeholder(placeholder)
         unless @placeholders.include?(placeholder)
           log.warn "unknown placeholder `#{placeholder}` found"
         end
@@ -217,41 +239,96 @@ module Fluent
     end
 
     class RubyPlaceholderExpander
-      attr_reader :placeholders, :log
+      attr_reader :log
 
       def initialize(params)
         @log = params[:log]
         @auto_typecast = params[:auto_typecast]
+        @cleanroom_expander = CleanroomExpander.new
       end
 
-      # Get placeholders as a struct
-      #
-      # @param [Time]   time        the time
-      # @param [Hash]   record      the record
-      # @param [Hash]   opts        others
-      def prepare_placeholders(time, record, opts)
-        struct = UndefOpenStruct.new(record)
-        struct.time = Time.at(time)
-        opts.each {|key, value| struct.__send__("#{key}=", value) }
-        @placeholders = struct
+      def time_value(time)
+        Time.at(time)
       end
 
-      def expand(_str_for_eval_, force_stringify=false)
-        if @auto_typecast and !force_stringify
-          _single_placeholder_matched_ = _str_for_eval_.match(/\A\${([^}]+)}\z/)
-          if _single_placeholder_matched_
-            return eval _single_placeholder_matched_[1], @placeholders.instance_eval { binding }
+      # Preprocess record map to convert into ruby string expansion
+      #
+      # @param [Hash|String|Array] value record map config
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def preprocess_map(value, force_stringify = false)
+        new_value = nil
+        if value.is_a?(String)
+          if @auto_typecast and !force_stringify
+            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
+              new_value = single_placeholder_matched[1]
+            end
+          end
+          unless new_value
+            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+          end
+        elsif value.is_a?(Hash)
+          new_value = {}
+          value.each_pair do |k, v|
+            new_value[preprocess_map(k, true)] = preprocess_map(v)
           end
+        elsif value.is_a?(Array)
+          new_value = []
+          value.each_with_index do |v, i|
+            new_value[i] = preprocess_map(v)
+          end
+        else
+          new_value = value
         end
-        _interpolated_for_eval_ = _str_for_eval_.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        eval "\"#{_interpolated_for_eval_}\"", @placeholders.instance_eval { binding }
+        new_value
+      end
+
+      def prepare_placeholders(placeholder_values)
+        @tag = placeholder_values['tag']
+        @time = placeholder_values['time']
+        @record = placeholder_values['record']
+        @tag_parts = placeholder_values['tag_parts']
+        @tag_prefix = placeholder_values['tag_prefix']
+        @tag_suffix = placeholder_values['tag_suffix']
+        @hostname = placeholder_values['hostname']
+      end
+
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      def expand(str, force_stringify = false)
+        @cleanroom_expander.expand(
+          str,
+          @tag,
+          @time,
+          @record,
+          @tag_parts,
+          @tag_prefix,
+          @tag_suffix,
+          @hostname,
+        )
       rescue => e
-        log.warn "failed to expand `#{_str_for_eval_}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
         log.warn_backtrace
         nil
       end
 
-      class UndefOpenStruct < OpenStruct
+      class CleanroomExpander
+        def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
+          tags = tag_parts # for old version compatibility
+          @record = record # for old version compatibility
+          instance_eval(__str_to_eval__)
+        end
+
+        # for old version compatibility
+        def method_missing(name)
+          key = name.to_s
+          if @record.has_key?(key)
+            @record[key]
+          else
+            raise NameError, "undefined local variable or method `#{key}'"
+          end
+        end
+
         (Object.instance_methods).each do |m|
           undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
         end

data/lib/fluent/plugin/in_forward.rb

@@ -38,6 +38,8 @@ module Fluent
     config_param :chunk_size_warn_limit, :size, :default => nil
     desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, :default => nil
+    desc 'Skip an event if incoming event is invalid.'
+    config_param :skip_invalid_event, :bool, :default => false
 
     def configure(conf)
       super
@@ -136,7 +138,7 @@ module Fluent
         return
       end
 
-      tag = msg[0].to_s
+      tag = msg[0]
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
@@ -149,27 +151,37 @@ module Fluent
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries)
+        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
         router.emit_stream(tag, es)
         option = msg[2]
 
       elsif entries.class == Array
         # Forward
-        es = MultiEventStream.new
-        entries.each {|e|
-          record = e[1]
-          next if record.nil?
-          time = e[0].to_i
-          time = (now ||= Engine.now) if time == 0
-          es.add(time, record)
-        }
+        es = if @skip_invalid_event
+               check_and_skip_invalid_event(tag, entries, source)
+             else
+               es = MultiEventStream.new
+               entries.each { |e|
+                 record = e[1]
+                 next if record.nil?
+                 time = e[0]
+                 time = (now ||= Engine.now) if time.to_i == 0
+                 es.add(time, record)
+               }
+               es
+             end
         router.emit_stream(tag, es)
         option = msg[2]
 
       else
         # Message
+        time = msg[1]
         record = msg[2]
+        if @skip_invalid_event && invalid_event?(tag, time, record)
+          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
+          return msg[3] # retry never succeeded so return ack and drop incoming event.
+        end
         return if record.nil?
-        time = msg[1]
         time = Engine.now if time == 0
         router.emit(tag, time, record)
         option = msg[3]
@@ -179,6 +191,22 @@ module Fluent
       option
     end
 
+    def invalid_event?(tag, time, record)
+      !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
+    end
+
+    def check_and_skip_invalid_event(tag, es, source)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        if invalid_event?(tag, time, record)
+          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
+          next
+        end
+        new_es.add(time, record)
+      }
+      new_es
+    end
+
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 

data/lib/fluent/plugin/in_tail.rb

@@ -40,6 +40,8 @@ module Fluent
     config_param :refresh_interval, :time, :default => 60
     desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, :default => 1000
+    desc 'The interval of flushing the buffer for multiline format'
+    config_param :multiline_flush_interval, :time, :default => nil
 
     attr_reader :paths
 
@@ -147,7 +149,8 @@ module Fluent
     end
 
     def setup_watcher(path, pe)
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), &method(:receive_lines))
+      line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -277,6 +280,7 @@ module Fluent
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
       if @parser.has_firstline?
+        tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
         lines.each { |line|
           if @parser.firstline?(line)
             if lb
@@ -308,7 +312,7 @@ module Fluent
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
@@ -323,10 +327,12 @@ module Fluent
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
         @io_handler = nil
         @log = log
+
+        @line_buffer_timer_flusher = line_buffer_timer_flusher
       end
 
       attr_reader :path
-      attr_accessor :line_buffer
+      attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
 
       def tag
@@ -358,6 +364,7 @@ module Fluent
 
       def on_notify
         @rotate_handler.on_notify if @rotate_handler
+        @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher
         return unless @io_handler
         @io_handler.on_notify
       end
@@ -595,6 +602,30 @@ module Fluent
           @log.error_backtrace
         end
       end
+
+
+      class LineBufferTimerFlusher
+        def initialize(log, flush_interval, &flush_method)
+          @log = log
+          @flush_interval = flush_interval
+          @flush_method = flush_method
+          @start = nil
+        end
+
+        def on_notify(tw)
+          if @start && @flush_interval
+            if Time.now - @start >= @flush_interval
+              @flush_method.call(tw)
+              tw.line_buffer = nil
+              @start = nil
+            end
+          end
+        end
+
+        def reset_timer
+          @start = Time.now
+        end
+      end
     end
 
 

data/lib/fluent/process.rb

@@ -289,6 +289,9 @@ module Fluent
     def on_detach_process(i)
     end
 
+    def on_exit_process(i)
+    end
+
     private
 
     def detach_process_impl(num, &block)
@@ -326,6 +329,7 @@ module Fluent
           #forward_thread.join  # TODO this thread won't stop because parent doesn't close pipe
           fin.wait
 
+          on_exit_process(i)
           exit! 0
         ensure
           $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid

data/lib/fluent/test/input_test.rb

@@ -102,7 +102,7 @@ module Fluent
         false
       end
 
-      def run(&block)
+      def run(num_waits = 10, &block)
         m = method(:emit_stream)
         Engine.define_singleton_method(:emit_stream) {|tag,es|
           m.call(tag, es)
@@ -110,7 +110,7 @@ module Fluent
         instance.router.define_singleton_method(:emit_stream) {|tag,es|
           m.call(tag, es)
         }
-        super {
+        super(num_waits) {
           block.call if block
 
           if @expected_emits_length || @expects || @run_post_conditions

data/lib/fluent/test/output_test.rb

@@ -69,9 +69,9 @@ module Fluent
         (@expected_buffer ||= '') << str
       end
 
-      def run(&block)
+      def run(num_waits = 10, &block)
         result = nil
-        super {
+        super(num_waits) {
           es = ArrayEventStream.new(@entries)
           buffer = @instance.format_stream(@tag, es)
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.19'
+  VERSION = '0.12.20'
 
 end

data/test/plugin/test_filter_record_transformer.rb

@@ -340,6 +340,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             multiple    ${source}${source}
             with_prefix prefix-${source}
             with_suffix ${source}-suffix
+            with_quote  source[""]
           </record>
         ]
         msgs = [
@@ -353,23 +354,28 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           { :single      => "string",
             :multiple    => "stringstring",
             :with_prefix => "prefix-string",
-            :with_suffix => "string-suffix" },
+            :with_suffix => "string-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => 123.to_s,
             :multiple    => "#{123.to_s}#{123.to_s}",
             :with_prefix => "prefix-#{123.to_s}",
-            :with_suffix => "#{123.to_s}-suffix" },
+            :with_suffix => "#{123.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => [1, 2].to_s,
             :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
             :with_prefix => "prefix-#{[1, 2].to_s}",
-            :with_suffix => "#{[1, 2].to_s}-suffix" },
+            :with_suffix => "#{[1, 2].to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => {a:1, b:2}.to_s,
             :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
             :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
-            :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+            :with_suffix => "#{{a:1, b:2}.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
           { :single      => nil.to_s,
             :multiple    => "#{nil.to_s}#{nil.to_s}",
             :with_prefix => "prefix-#{nil.to_s}",
-            :with_suffix => "#{nil.to_s}-suffix" },
+            :with_suffix => "#{nil.to_s}-suffix",
+            :with_quote  => %Q{source[""]} },
         ]
         actual_results = []
         es = emit(config, msgs)
@@ -379,6 +385,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             :multiple    => r["multiple"],
             :with_prefix => r["with_prefix"],
             :with_suffix => r["with_suffix"],
+            :with_quote  => r["with_quote"],
           }
         end
         assert_equal(expected_results, actual_results)
@@ -436,6 +443,27 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         end
         assert_equal(expected_results, actual_results)
       end
+
+      test %Q[record["key"] with enable_ruby #{enable_ruby}] do
+        config = %[
+          enable_ruby #{enable_ruby}
+          auto_typecast yes
+          <record>
+            _timestamp ${record["@timestamp"]}
+            _foo_bar   ${record["foo.bar"]}
+          </record>
+        ]
+        d = create_driver(config)
+        record = {
+          "foo.bar"    => "foo.bar",
+          "@timestamp" => 10,
+        }
+        es = d.run { d.emit(record, @time) }.filtered
+        es.each do |t, r|
+          assert { r['_timestamp'] == record['@timestamp'] }
+          assert { r['_foo_bar'] == record['foo.bar'] }
+        end
+      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do
@@ -458,7 +486,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       ]
       es = emit(config) { |d|
-        mock(d.instance.log).warn("failed to expand `${unknown['bar']}`", anything)
+        mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
       }
       es.each do |t, r|
         assert_nil(r['message'])
@@ -495,4 +523,36 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       end
     end
   end
+
+  test "compatibility test (enable_ruby yes)" do
+    config = %[
+      enable_ruby yes
+      auto_typecast yes
+      <record>
+        _message   prefix-${message}-suffix
+        _time      ${Time.at(time)}
+        _number    ${number == '-' ? 0 : number}
+        _match     ${/0x[0-9a-f]+/.match(hex)[0]}
+        _timestamp ${__send__("@timestamp")}
+        _foo_bar   ${__send__('foo.bar')}
+      </record>
+    ]
+    d = create_driver(config)
+    record = {
+      "number"     => "-",
+      "hex"        => "0x10",
+      "foo.bar"    => "foo.bar",
+      "@timestamp" => 10,
+      "message"    => "10",
+    }
+    es = d.run { d.emit(record, @time) }.filtered
+    es.each do |t, r|
+      assert { r['_message'] == "prefix-#{record['message']}-suffix" }
+      assert { r['_time'] == Time.at(@time) }
+      assert { r['_number'] == 0 }
+      assert { r['_match'] == record['hex'] }
+      assert { r['_timestamp'] == record['@timestamp'] }
+      assert { r['_foo_bar'] == record['foo.bar'] }
+    end
+  end
 end

data/test/plugin/test_in_forward.rb

@@ -64,6 +64,28 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_message_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag, time, record| [tag, time, record] }
+      # These entries are skipped
+      entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
+
+      entries.each { |tag, time, record|
+        # Without ack, logs are sometimes not saved to logs during test.
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s, true
+      }
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
+  end
+
   def test_forward
     d = create_driver
 
@@ -81,6 +103,25 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag, time, record| [time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
   def test_packed_forward
     d = create_driver
 
@@ -98,6 +139,29 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_packed_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map { |tag ,time, record| [time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      packed_entries = ''
+      entries.each { |time, record|
+        Fluent::Engine.msgpack_factory.packer(packed_entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
   def test_message_json
     d = create_driver
 

data/test/plugin/test_in_tail.rb

@@ -12,6 +12,11 @@ class TailInputTest < Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
   end
 
+  def teardown
+    super
+    Fluent::Engine.stop
+  end
+
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = %[
@@ -288,16 +293,61 @@ class TailInputTest < Test::Unit::TestCase
         f.puts "s test8"
       }
       sleep 1
+
+      emits = d.emits
+      assert(emits.length == 3)
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+      assert_equal({"message1" => "test5"}, emits[1][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+
+      sleep 3
+      emits = d.emits
+      assert(emits.length == 3)
     end
 
     emits = d.emits
-    assert(emits.length > 0)
-    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
-    assert_equal({"message1" => "test5"}, emits[1][2])
-    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert(emits.length == 4)
     assert_equal({"message1" => "test8"}, emits[3][2])
   end
 
+  def test_multiline_with_flush_interval
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+    ]
+
+    assert_equal 2, d.instance.multiline_flush_interval
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+
+      emits = d.emits
+      assert(emits.length == 3)
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+      assert_equal({"message1" => "test5"}, emits[1][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+
+      sleep 3
+      emits = d.emits
+      assert(emits.length == 4)
+      assert_equal({"message1" => "test8"}, emits[3][2])
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
 
@@ -434,7 +484,7 @@ class TailInputTest < Test::Unit::TestCase
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -450,7 +500,7 @@ class TailInputTest < Test::Unit::TestCase
       end
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

data/test/test_buffer.rb

@@ -574,7 +574,7 @@ module FluentBufferTest
 
         begin
           # with block, emit events to full queue causes sleep loop
-          timeout(1) {
+          Timeout.timeout(1) {
             assert db.emit('key', data, chain)
           }
           flunk("timeout must happen")

data/test/test_parser.rb

@@ -85,7 +85,7 @@ module ParserTest
     def test_parse_with_invalid_argument
       parser = TextParser::TimeParser.new(nil)
 
-      [[], {}, nil, true, 10000].each { |v|
+      [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
         assert_raise Fluent::ParserError do
           parser.parse(v)
         end
@@ -386,6 +386,14 @@ module ParserTest
       }
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_large_float(data)
+      @parser.configure('json_parser' => data)
+      @parser.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
+        assert_equal(Float, record['num'].class)
+      }
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_parse_without_time(data)
       time_at_start = Time.now.to_i

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.19
+  version: 0.12.20
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-21 00:00:00.000000000 Z
+date: 2016-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack