fluentd 0.10.30 → 0.10.31

data/ChangeLog

@@ -1,4 +1,12 @@
 
+Release 0.10.31 - 2013/01/31
+
+* out_forward: supports TCP heartbeat
+* in_tail reads rotated files from its head rather than its end when pos_file option is set
+* out_/in_ forward set O_NONBLOCK flag to the UDP heartbeat socket
+* Improved match cache expiration algorithm
+
+
 Release 0.10.30 - 2012/12/06
 
 * Shows configuration file at start time

data/README.rdoc

@@ -52,7 +52,7 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 == Meta
 
 Web site::  http://fluentd.org/
-Documents:: http://fluentd.org/doc/
+Documents:: http://docs.fluentd.org/
 Source repository:: http://github.com/fluent
 Discussion:: http://groups.google.com/group/fluentd
 Author:: Sadayuki Furuhashi

data/VERSION

@@ -1 +1 @@
-0.10.30
+0.10.31

data/lib/fluent/engine.rb

@@ -23,10 +23,13 @@ class EngineClass
     @matches = []
     @sources = []
     @match_cache = {}
+    @match_cache_keys = []
     @started = []
     @default_loop = nil
   end
 
+  MATCH_CACHE_SIZE = 1024
+
   attr_reader :matches, :sources
 
   def init
@@ -106,9 +109,13 @@ class EngineClass
     target = @match_cache[tag]
     unless target
       target = match(tag) || NoMatchMatch.new
-      if @match_cache.size < 1024  # TODO size limit
-        @match_cache[tag] = target
+      # this is not thread-safe but inconsistency doesn't
+      # cause serious problems while locking causes.
+      if @match_cache_keys.size >= MATCH_CACHE_SIZE
+        @match_cache_keys.delete @match_cache_keys.shift
       end
+      @match_cache[tag] = target
+      @match_cache_keys << tag
     end
     target.emit(tag, es)
   rescue

data/lib/fluent/event.rb

@@ -65,7 +65,7 @@ class ArrayEventStream < EventStream
   end
 
   def empty?
-    @time_array.empty?
+    @entries.empty?
   end
 
   def each(&block)

data/lib/fluent/load.rb

@@ -1,5 +1,6 @@
 require 'thread'
 require 'socket'
+require 'fcntl'
 require 'time'
 require 'monitor'
 require 'stringio'

data/lib/fluent/output.rb

@@ -141,19 +141,8 @@ class OutputThread
     }
   end
 
-  if ConditionVariable.new.method(:wait).arity == 1
-    $log.warn "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
-    require 'timeout'
-    def cond_wait(sec)
-      Timeout.timeout(sec) {
-        @cond.wait(@mutex)
-      }
-    rescue Timeout::Error
-    end
-  else
-    def cond_wait(sec)
-      @cond.wait(@mutex, sec)
-    end
+  def cond_wait(sec)
+    @cond.wait(@mutex, sec)
   end
 end
 

data/lib/fluent/parser.rb

@@ -213,7 +213,7 @@ class TextParser
       regexp = regexp_or_proc
       factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
     else
-      factory = Proc.new { regexp_or_proc }
+      factory = regexp_or_proc
     end
 
     TEMPLATE_FACTORIES[name] = factory

data/lib/fluent/plugin/in_forward.rb

@@ -41,6 +41,7 @@ class ForwardInput < Input
 
     @usock = SocketUtil.create_udp_socket(@bind)
     @usock.bind(@bind, @port)
+    @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
     @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
     @loop.attach(@hbr)
 
@@ -102,6 +103,11 @@ class ForwardInput < Input
   #   3: object record
   # }
   def on_message(msg)
+    if msg.nil?
+      # for future TCP heartbeat_request
+      return
+    end
+
     # TODO format error
     tag = msg[0].to_s
     entries = msg[1]
@@ -191,7 +197,11 @@ class ForwardInput < Input
     end
 
     def on_readable
-      msg, addr = @io.recvfrom(1024)
+      begin
+        msg, addr = @io.recvfrom(1024)
+      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+        return
+      end
       host = addr[3]
       port = addr[1]
       @callback.call(host, port, msg)
@@ -202,7 +212,10 @@ class ForwardInput < Input
 
   def on_heartbeat_request(host, port, msg)
     #$log.trace "heartbeat request from #{host}:#{port}"
-    @usock.send "\0", 0, host, port
+    begin
+      @usock.send "\0", 0, host, port
+    rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+    end
   end
 end
 

data/lib/fluent/plugin/in_syslog.rb

@@ -77,7 +77,6 @@ class SyslogInput < Input
 
     parser = TextParser.new
     if parser.configure(conf, false)
-      parser.use_template('syslog')
       @parser = parser
     else
       @parser = nil

data/lib/fluent/plugin/in_tail.rb

@@ -191,12 +191,22 @@ class TailInput < Input
           stat = io.stat
           fsize = stat.size
           inode = stat.ino
-          if inode == @pe.read_inode
+
+          last_inode = @pe.read_inode
+          if inode == last_inode
             # seek to the saved position
             pos = @pe.read_pos
+          elsif last_inode != 0
+            # this is FilePositionEntry and fluentd once started.
+            # read data from the head of the rotated file.
+            # logs never duplicate because this file is a rotated new file.
+            pos = 0
+            @pe.update(inode, pos)
           else
-            # seek to the end of the file.
-            # logs never duplicate but may be lost if fluentd is down.
+            # this is MemoryPositionEntry or this is the first time fluentd started.
+            # seek to the end of the any files.
+            # logs may duplicate without this seek because it's not sure the file is
+            # existent file or rotated new file.
             pos = fsize
             @pe.update(inode, pos)
           end

data/lib/fluent/plugin/out_forward.rb

@@ -30,6 +30,16 @@ class ForwardOutput < ObjectBufferedOutput
   end
 
   config_param :send_timeout, :time, :default => 60
+  config_param :heartbeat_type, :default => :udp do |val|
+    case val.downcase
+    when 'tcp'
+      :tcp
+    when 'udp'
+      :udp
+    else
+      raise ConfigError, "forward output heartbeat type should be 'tcp' or 'udp'"
+    end
+  end
   config_param :heartbeat_interval, :time, :default => 1
   config_param :recover_wait, :time, :default => 10
   config_param :hard_timeout, :time, :default => 60
@@ -89,10 +99,13 @@ class ForwardOutput < ObjectBufferedOutput
 
     @loop = Coolio::Loop.new
 
-    # Assume all hosts are same protocol.
-    @usock = SocketUtil.create_udp_socket(@nodes.first.host)
-    @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
-    @loop.attach(@hb)
+    if @heartbeat_type == :udp
+      # assuming all hosts use udp
+      @usock = SocketUtil.create_udp_socket(@nodes.first.host)
+      @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+      @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
+      @loop.attach(@hb)
+    end
 
     @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
     @loop.attach(@timer)
@@ -105,7 +118,7 @@ class ForwardOutput < ObjectBufferedOutput
     @loop.watchers.each {|w| w.detach }
     @loop.stop
     @thread.join
-    @usock.close
+    @usock.close if @usock
   end
 
   def run
@@ -196,6 +209,22 @@ class ForwardOutput < ObjectBufferedOutput
   # MessagePack FixArray length = 2
   FORWARD_HEADER = [0x92].pack('C')
 
+  #FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
+  def send_heartbeat_tcp(node)
+    sock = connect(node)
+    begin
+      opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+      opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+      # don't send any data to not cause a compatibility problem
+      #sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+      #sock.write FORWARD_TCP_HEARTBEAT_DATA
+      node.heartbeat(true)
+    ensure
+      sock.close
+    end
+  end
+
   def send_data(node, tag, es)
     sock = connect(node)
     begin
@@ -258,9 +287,13 @@ class ForwardOutput < ObjectBufferedOutput
         rebuild_weight_array
       end
       begin
-        #$log.trace "sending heartbeat #{n.host}:#{n.port}"
-        @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
-      rescue
+        #$log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
+        if @heartbeat_type == :tcp
+          send_heartbeat_tcp(n)
+        else
+          @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
+        end
+      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
         # TODO log
         $log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", :error=>$!.to_s
       end
@@ -275,7 +308,11 @@ class ForwardOutput < ObjectBufferedOutput
     end
 
     def on_readable
-      msg, addr = @io.recvfrom(1024)
+      begin
+        msg, addr = @io.recvfrom(1024)
+      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+        return
+      end
       host = addr[3]
       port = addr[1]
       sockaddr = Socket.pack_sockaddr_in(port, host)
@@ -496,4 +533,3 @@ end
 
 
 end
-

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.30'
+VERSION = '0.10.31'
 
 end

data/test/helper.rb

@@ -7,5 +7,15 @@ class Test::Unit::TestCase
   include RR::Adapters::TestUnit
 end
 
-$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_WARN)
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
 
+$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_WARN)

data/test/plugin/in_syslog.rb

@@ -1,4 +1,5 @@
 require 'fluent/test'
+require 'helper'
 
 class SyslogInputTest < Test::Unit::TestCase
   def setup
@@ -23,7 +24,10 @@ class SyslogInputTest < Test::Unit::TestCase
   end
 
   def test_configure
-    {'127.0.0.1' => CONFIG, '::1' => IPv6_CONFIG}.each_pair { |k, v|
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
       d = create_driver(v)
       assert_equal 9911, d.instance.port
       assert_equal k, d.instance.bind
@@ -31,7 +35,10 @@ class SyslogInputTest < Test::Unit::TestCase
   end
 
   def test_time_format
-    {'127.0.0.1' => CONFIG, '::1' => IPv6_CONFIG}.each_pair { |k, v|
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
       d = create_driver(v)
 
       tests = [

data/test/plugin/out_forward.rb

@@ -26,10 +26,16 @@ class ForwardOutputTest < Test::Unit::TestCase
     d = create_driver
     nodes = d.instance.nodes
     assert_equal 51, d.instance.send_timeout
+    assert_equal :udp, d.instance.heartbeat_type
     assert_equal 1, nodes.length
     node = nodes.first
     assert_equal "test", node.name
     assert_equal '127.0.0.1', node.host
     assert_equal 13999, node.port
   end
+
+  def test_configure_tcp_heartbeat
+    d = create_driver(CONFIG + "\nheartbeat_type tcp")
+    assert_equal :tcp, d.instance.heartbeat_type
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.30
+  version: 0.10.31
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-06 00:00:00.000000000 Z
+date: 2013-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack