fluentd 0.10.49 → 0.10.50

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d9ea1a003d545a18baf0c61a85fb216d183cb07
-  data.tar.gz: c3ccc3a4665acdf5726287ce6b29e2e6af5c5194
+  metadata.gz: dcc68a7c0e54890fee8388d37e9801edac3fb77f
+  data.tar.gz: 992e8adb5d7c043f91f58e095aaf1d6cb5c7b363
 SHA512:
-  metadata.gz: cd8c5f25fc9ca7cf31ac5d0fb2de908cee79204deb11e821e31774b8aecc112557ebcc0f0d2ddc033fcb4b189e31b05ff9fbe1a07d04ad4630a2b39980038f8b
-  data.tar.gz: af93420f7ed26fdf703d11723aa27e76a43528b90b67879687e8bb5fa94bb9bba55fbad61dbeaa10dc2218de776cc39df3e13372204a50359008a4abcebdf6f3
+  metadata.gz: 84b5ed1c244b405de5ed3d637fbad708cbb361a42a7efd13793c2992f7aad66c9c9d49256b105545e65e2627d5e3de64838e0df05d9e0d6bd82e6c47d7f56230
+  data.tar.gz: ad3c062f06ec7b472f7efb72acc693c9b71e83fc4eaf1df4c6c23ca5820543cb72bfea59778f3f5cb42f1510927f1f52c9f674a8bee3debdb3f86c4cc75e5950

data/ChangeLog

@@ -1,3 +1,14 @@
+Release 0.10.50 - 2014/06/17
+
+* in_tail: Fix race condition at shutdown
+* in_tail: read_from_head now works without pos_file
+* in_tail: Fix multiline format parsing without format_firstline
+* formatter: Add add_newline option to SingleValueFormatter
+* parser: Add block based API to support multi-events parsing
+* config: Fix encoding mismatch when @include used in v1
+* engine: Serialize shutdown order. Now shutdown input plugins first
+* Update rspec to 3.x
+
 Release 0.10.49 - 2014/06/05
 
 * in_http: Add format option to support various input data format

data/fluentd.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock")
   gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("rspec", ["~> 2.13"])
+  gem.add_development_dependency("rspec", ["~> 3.0.0"])
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])

data/lib/fluent/config/v1_parser.rb

@@ -142,16 +142,16 @@ module Fluent
             basepath = File.dirname(path)
             fname = File.basename(path)
             data = File.read(path)
+            data.force_encoding('UTF-8')
             ss = StringScanner.new(data)
             V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
           }
-
         else
+          require 'open-uri'
           basepath = '/'
           fname = path
-          require 'open-uri'
-          data = nil
-          open(uri) { |f| data = f.read }
+          data = open(uri) { |f| f.read }
+          data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
         end

data/lib/fluent/engine.rb

@@ -22,7 +22,8 @@ module Fluent
       @sources = []
       @match_cache = {}
       @match_cache_keys = []
-      @started = []
+      @started_sources = []
+      @started_matches = []
       @default_loop = nil
       @engine_stopped = false
 
@@ -254,25 +255,40 @@ module Fluent
     def start
       @matches.each {|m|
         m.start
-        @started << m
+        @started_matches << m
       }
       @sources.each {|s|
         s.start
-        @started << s
+        @started_sources << s
       }
     end
 
     def shutdown
-      @started.map {|s|
+      # Shutdown Input plugin first to prevent emitting to terminated Output plugin
+      @started_sources.map { |s|
         Thread.new do
           begin
             s.shutdown
           rescue => e
-            $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
+            $log.warn "unexpected error while shutting down", :plugin => s.class, :plugin_id => s.plugin_id, :error_class => e.class, :error => e
             $log.warn_backtrace
           end
         end
-      }.each {|t|
+      }.each { |t|
+        t.join
+      }
+      # Output plugin as filter emits records at shutdown so emit problem still exist.
+      # This problem will be resolved after actual filter mechanizm.
+      @started_matches.map { |m|
+        Thread.new do
+          begin
+            m.shutdown
+          rescue => e
+            $log.warn "unexpected error while shutting down", :plugin => m.output.class, :plugin_id => m.output.plugin_id, :error_class => e.class, :error => e
+            $log.warn_backtrace
+          end
+        end
+      }.each { |t|
         t.join
       }
     end

data/lib/fluent/formatter.rb

@@ -126,9 +126,12 @@ module Fluent
       include Configurable
 
       config_param :message_key, :string, :default => 'message'
+      config_param :add_newline, :bool, :default => true
 
       def format(tag, time, record)
-        record[@message_key]
+        text = record[@message_key].to_s
+        text << "\n" if @add_newline
+        text
       end
     end
 

data/lib/fluent/mixin.rb

@@ -67,20 +67,6 @@ module Fluent
     def format_nocache(time)
       # will be overridden in initialize
     end
-
-    def self.configure(conf)
-      if time_format = conf['time_format']
-        @time_format = time_format
-      end
-
-      if localtime = conf['localtime']
-        @localtime = true
-      elsif utc = conf['utc']
-        @localtime = false
-      end
-
-      @timef = new(@time_format, @localtime)
-    end
   end
 
 

data/lib/fluent/parser.rb

@@ -147,7 +147,12 @@ module Fluent
       def call(text)
         m = @regexp.match(text)
         unless m
-          return nil, nil
+          if block_given?
+            yield nil, nil
+            return
+          else
+            return nil, nil
+          end
         end
 
         time = nil
@@ -170,7 +175,11 @@ module Fluent
 
         time ||= Engine.now
 
-        return time, record
+        if block_given?
+          yield time, record
+        else # keep backward compatibility. will be removed at v1
+          return time, record
+        end
       end
     end
 
@@ -202,9 +211,17 @@ module Fluent
           time = Engine.now
         end
 
-        return time, record
+        if block_given?
+          yield time, record
+        else
+          return time, record
+        end
       rescue Yajl::ParseError
-        return nil, nil
+        if block_given?
+          yield nil, nil
+        else
+          return nil, nil
+        end
       end
     end
 
@@ -267,7 +284,11 @@ module Fluent
       config_param :delimiter, :string, :default => "\t"
 
       def call(text)
-        return values_map(text.split(@delimiter))
+        if block_given?
+          yield values_map(text.split(@delimiter))
+        else
+          return values_map(text.split(@delimiter))
+        end
       end
     end
 
@@ -291,7 +312,11 @@ module Fluent
           values.push(value)
         end
 
-        return values_map(values)
+        if block_given?
+          yield values_map(values)
+        else
+          return values_map(values)
+        end
       end
     end
 
@@ -302,7 +327,11 @@ module Fluent
       end
 
       def call(text)
-        return values_map(CSV.parse_line(text))
+        if block_given?
+          yield values_map(CSV.parse_line(text))
+        else
+          return values_map(CSV.parse_line(text))
+        end
       end
     end
 
@@ -314,7 +343,11 @@ module Fluent
       def call(text)
         record = {}
         record[@message_key] = text
-        return Engine.now, record
+        if block_given?
+          yield Engine.now, record
+        else
+          return Engine.now, record
+        end
       end
     end
 
@@ -331,7 +364,12 @@ module Fluent
       def call(text)
         m = REGEXP.match(text)
         unless m
-          return nil, nil
+          if block_given?
+            yield nil, nil
+            return
+          else
+            return nil, nil
+          end
         end
 
         host = m['host']
@@ -369,7 +407,11 @@ module Fluent
           "agent" => agent,
         }
 
-        return time, record
+        if block_given?
+          yield time, record
+        else
+          return time, record
+        end
       end
     end
 
@@ -396,16 +438,24 @@ module Fluent
 
         if @format_firstline
           check_format_regexp(@format_firstline, 'format_firstline')
-          @regex = Regexp.new(@format_firstline[1..-2])
+          @firstline_regex = Regexp.new(@format_firstline[1..-2])
         end
       end
 
-      def call(text)
-        @parser.call(text)
+      def call(text, &block)
+        if block
+          @parser.call(text, &block)
+        else
+          @parser.call(text)
+        end
+      end
+
+      def has_firstline?
+        !!@format_firstline
       end
 
       def firstline?(text)
-        @regex.match(text)
+        @firstline_regex.match(text)
       end
 
       private
@@ -506,7 +556,6 @@ module Fluent
         end
 
         @parser = RegexpParser.new(regexp, conf)
-
       else
         # built-in template
         begin
@@ -524,8 +573,12 @@ module Fluent
       return true
     end
 
-    def parse(text)
-      return @parser.call(text)
+    def parse(text, &block)
+      if block
+        @parser.call(text, &block)
+      else # keep backward compatibility. Will be removed at v1
+        return @parser.call(text)
+      end
     end
   end
 end

data/lib/fluent/plugin/buf_memory.rb

@@ -22,7 +22,7 @@ module Fluent
       @data.force_encoding('ASCII-8BIT')
       now = Time.now.utc
       u1 = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
-      @unique_id = [u1 >> 32, u1 & u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
+      @unique_id = [u1 >> 32, u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
       super(key)
     end
 

data/lib/fluent/plugin/in_http.rb

@@ -54,13 +54,6 @@ module Fluent
     end
 
     class KeepaliveManager < Coolio::TimerWatcher
-      class TimerValue
-        def initialize
-          @value = 0
-        end
-        attr_accessor :value
-      end
-
       def initialize(timeout)
         super(1, true)
         @cons = {}

data/lib/fluent/plugin/in_syslog.rb

@@ -131,14 +131,14 @@ module Fluent
       pri = m[1].to_i
       text = m[2]
 
-      time, record = @parser.parse(text)
-      unless time && record
-        log.warn "pattern not match: #{text.inspect}"
-        return
-      end
-
-      emit(pri, time, record)
+      @parser.parse(text) { |time, record|
+        unless time && record
+          log.warn "pattern not match: #{text.inspect}"
+          return
+        end
 
+        emit(pri, time, record)
+      }
     rescue
       log.warn data.dump, :error=>$!.to_s
       log.debug_backtrace

data/lib/fluent/plugin/in_tail.rb

@@ -129,7 +129,7 @@ module Fluent
     end
 
     def setup_watcher(path, pe)
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, method(:update_watcher), &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, method(:update_watcher), &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -158,7 +158,7 @@ module Fluent
         if tw
           tw.unwatched = unwatched
           if immediate
-            close_watcher(tw)
+            close_watcher(tw, false)
           else
             close_watcher_after_rotate_wait(tw)
           end
@@ -173,8 +173,12 @@ module Fluent
       close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
     end
 
-    def close_watcher(tw)
-      tw.close
+    # TailWatcher#close is called by another thread at shutdown phase.
+    # It causes 'can't modify string; temporarily locked' error in IOHandler
+    # so adding close_io argument to avoid this problem.
+    # At shutdown, IOHandler's io will be released automatically after detached the event loop
+    def close_watcher(tw, close_io = true)
+      tw.close(close_io)
       flush_buffer(tw)
       if tw.unwatched && @pf
         @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
@@ -189,17 +193,18 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        time, record = parse_line(lb)
-        if time && record
-          tag = if @tag_prefix || @tag_suffix
-                  @tag_prefix + tw.tag + @tag_suffix
-                else
-                  @tag
-                end
-          Engine.emit(tag, time, record)
-        else
-          log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
-        end
+        @parser.parse(lb) { |time, record|
+          if time && record
+            tag = if @tag_prefix || @tag_suffix
+                    @tag_prefix + tw.tag + @tag_suffix
+                  else
+                    @tag
+                  end
+            Engine.emit(tag, time, record)
+          else
+            log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
+          end
+        }
       end
     end
 
@@ -226,19 +231,16 @@ module Fluent
       end
     end
 
-    def parse_line(line)
-      return @parser.parse(line)
-    end
-
     def convert_line_to_event(line, es)
       begin
         line.chomp!  # remove \n
-        time, record = parse_line(line)
-        if time && record
-          es.add(time, record)
-        else
-          log.warn "pattern not match: #{line.inspect}"
-        end
+        @parser.parse(line) { |time, record|
+          if time && record
+            es.add(time, record)
+          else
+            log.warn "pattern not match: #{line.inspect}"
+          end
+        }
       rescue => e
         log.warn line.dump, :error => e.to_s
         log.debug_backtrace(e.backtrace)
@@ -256,29 +258,43 @@ module Fluent
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
-      lines.each { |line|
-        if @parser.parser.firstline?(line)
-          if lb
-            convert_line_to_event(lb, es)
-          end
-          lb = line
-        else
-          if lb.nil?
-            log.warn "got incomplete line before first line from #{tail_watcher.path}: #{lb.inspect}"
+      if @parser.parser.has_firstline?
+        lines.each { |line|
+          if @parser.parser.firstline?(line)
+            if lb
+              convert_line_to_event(lb, es)
+            end
+            lb = line
           else
-            lb << line
+            if lb.nil?
+              log.warn "got incomplete line before first line from #{tail_watcher.path}: #{lb.inspect}"
+            else
+              lb << line
+            end
           end
+        }
+      else
+        lb ||= ''
+        lines.each do |line|
+          lb << line
+          @parser.parse(lb) { |time, record|
+            if time && record
+              convert_line_to_event(lb, es)
+              lb = ''
+            end
+          }
         end
-      }
+      end
       tail_watcher.line_buffer = lb
       es
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, update_watcher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, update_watcher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
+        @read_from_head = read_from_head
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
@@ -313,8 +329,8 @@ module Fluent
         @stat_trigger.detach if @stat_trigger.attached?
       end
 
-      def close
-        if @io_handler
+      def close(close_io = true)
+        if close_io && @io_handler
           @io_handler.on_notify
           @io_handler.close
         end
@@ -354,7 +370,7 @@ module Fluent
               # seek to the end of the any files.
               # logs may duplicate without this seek because it's not sure the file is
               # existent file or rotated new file.
-              pos = fsize
+              pos = @read_from_head ? 0 : fsize
               @pe.update(inode, pos)
             end
             io.seek(pos)