fluentd 0.10.6 → 0.10.7

data/lib/fluent/plugin/buf_file.rb

@@ -41,8 +41,9 @@ class FileBufferChunk < BufferChunk
   end
 
   def close
+    stat = @file.stat
     @file.close
-    if @file.stat.size == 0
+    if stat.size == 0
       File.unlink(@path)
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -50,10 +50,12 @@ class ForwardInput < Input
   end
 
   def shutdown
-    @lsock.close
+    @loop.watchers.each {|w| w.detach }
     @loop.stop
-    @thread.join
     @usock.close
+    TCPSocket.open('127.0.0.1', @port) {|sock| }  # FIXME @thread.join blocks without this line
+    @thread.join
+    @lsock.close
   end
 
   def listen

data/lib/fluent/plugin/in_http.rb

@@ -21,6 +21,8 @@ module Fluent
 class HttpInput < Input
   Plugin.register_input('http', self)
 
+  include DetachMultiProcessMixin
+
   require 'http/parser'
 
   def initialize
@@ -68,23 +70,28 @@ class HttpInput < Input
     end
   end
 
-  # TODO multithreading
   def start
     $log.debug "listening http on #{@bind}:#{@port}"
+    lsock = TCPServer.new(@bind, @port)
 
-    @km = KeepaliveManager.new(@keepalive_timeout)
-    @lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
+    detach_multi_process do
+      super
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
+      @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit)
 
-    @loop = Coolio::Loop.new
-    @loop.attach(@km)
-    @loop.attach(@lsock)
+      @loop = Coolio::Loop.new
+      @loop.attach(@km)
+      @loop.attach(@lsock)
 
-    @thread = Thread.new(&method(:run))
+      @thread = Thread.new(&method(:run))
+    end
   end
 
   def shutdown
-    @lsock.close
+    @loop.watchers.each {|w| w.detach }
     @loop.stop
+    @lsock.close
     @thread.join
   end
 

data/lib/fluent/plugin/in_stream.rb

@@ -35,8 +35,9 @@ class StreamInput < Input
   end
 
   def shutdown
-    @lsock.close
+    @loop.watchers.each {|w| w.detach }
     @loop.stop
+    @lsock.close
     @thread.join
   end
 

data/lib/fluent/plugin/in_tail.rb

@@ -28,6 +28,7 @@ class TailInput < Input
 
   config_param :path, :string
   config_param :tag, :string
+  config_param :pos_file, :string, :default => nil
 
   def configure(conf)
     super
@@ -37,6 +38,12 @@ class TailInput < Input
       raise ConfigError, "tail: 'path' parameter is required on tail input"
     end
 
+    if @pos_file
+      @pf_file = File.open(@pos_file, File::RDWR|File::CREAT)
+      @pf_file.sync = true
+      @pf = PositionFile.parse(@pf_file)
+    end
+
     configure_parser(conf)
   end
 
@@ -47,16 +54,23 @@ class TailInput < Input
 
   def start
     @loop = Coolio::Loop.new
-    @paths.each {|path|
+    handlers = @paths.map {|path|
       $log.debug "following tail of #{path}"
-      @loop.attach Handler.new(path, method(:receive_lines))
+      pe = @pf ? @pf[path] : NullPositionEntry.instance
+      h = Handler.new(path, pe, method(:receive_lines))
+      @loop.attach h
+    }
+    handlers.each {|h|
+      h.on_change  # initialize call
     }
     @thread = Thread.new(&method(:run))
   end
 
   def shutdown
+    @loop.watchers.each {|w| w.detach }
     @loop.stop
     @thread.join
+    @pf_file.close if @pf_file
   end
 
   def run
@@ -90,11 +104,20 @@ class TailInput < Input
     return @parser.parse(line)
   end
 
-  # seek to the end of file first.
-  # logs never duplicate but may be lost if fluent is down.
   class Handler < Coolio::StatWatcher
-    def initialize(path, callback)
-      @pos = File.stat(path).size
+    def initialize(path, pe, callback)
+      stat = File.lstat(path)
+      @pe = pe
+      @inode = stat.ino
+      if @inode == @pe.read_inode
+        # seek to the saved position
+        @pos = @pe.read_pos
+      else
+        # seek to the end of file first.
+        # logs never duplicate but may be lost if fluent is down.
+        @pos = stat.size
+        @pe.update(@inode, stat.size)
+      end
       @buffer = ''
       @callback = callback
       super(path)
@@ -102,9 +125,13 @@ class TailInput < Input
 
     def on_change
       lines = []
+      inode = nil
 
       File.open(path) {|f|
-        if f.lstat.size < @pos
+        stat = f.lstat
+        inode = stat.ino
+
+        if @inode != inode || stat.size < @pos
           # moved or deleted
           @pos = 0
         else
@@ -113,13 +140,13 @@ class TailInput < Input
 
         line = f.gets
         unless line
-          return
+          break
         end
 
         @buffer << line
         unless line[line.length-1] == ?\n
           @pos = f.pos
-          return
+          break
         end
 
         lines << @buffer
@@ -136,11 +163,108 @@ class TailInput < Input
         @pos = f.pos
       }
 
+      if @inode != inode
+        @pe.update(inode, @pos)
+        @inode = inode
+      else
+        @pe.update_pos(@pos)
+      end
+
       @callback.call(lines)
 
     rescue Errno::ENOENT
       # moved or deleted
       @pos = 0
+    # TODO rescue
+    end
+  end
+
+  # pos               inode
+  # ffffffffffffffff\tffffffff\n
+  class PositionEntry
+    POS_SIZE = 16
+    INO_OFFSET = 17
+    INO_SIZE = 8
+    LN_OFFSET = 25
+    SIZE = 26
+
+    def initialize(file, seek)
+      @file = file
+      @seek = seek
+    end
+
+    def update(ino, pos)
+      @file.pos = @seek
+      @file.write "%016x\t%08x" % [pos, ino]
+      @inode = ino
+    end
+
+    def update_pos(pos)
+      @file.pos = @seek
+      @file.write "%016x" % pos
+    end
+
+    def read_inode
+      @file.pos = @seek + INO_OFFSET
+      @file.read(8).to_i(16)
+    end
+
+    def read_pos
+      @file.pos = @seek
+      @file.read(16).to_i(16)
+    end
+  end
+
+  class PositionFile
+    def initialize(file, map, last_pos)
+      @file = file
+      @map = map
+      @last_pos = last_pos
+    end
+
+    def [](path)
+      if m = @map[path]
+        return m
+      end
+
+      @file.pos = @last_pos
+      @file.write path
+      @file.write "\t"
+      seek = @file.pos
+      @file.write "0000000000000000\t00000000\n"
+      @last_pos = @file.pos
+
+      @map[path] = PositionEntry.new(@file, seek)
+    end
+
+    def self.parse(file)
+      map = {}
+      file.pos = 0
+      file.each_line {|line|
+        m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
+        next unless m
+        path = m[1]
+        pos = m[2].to_i(16)
+        ino = m[3].to_i(16)
+        seek = file.pos - line.bytesize + path.bytesize + 1
+        map[path] = PositionEntry.new(file, seek)
+      }
+      new(file, map, file.pos)
+    end
+  end
+
+  class NullPositionEntry
+    require 'singleton'
+    include Singleton
+    def update(ino, pos)
+    end
+    def update_pos(pos)
+    end
+    def read_pos
+      0
+    end
+    def read_inode
+      0
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -18,7 +18,7 @@
 module Fluent
 
 
-class ForwardOutput < BufferedOutput
+class ForwardOutput < ObjectBufferedOutput
   Plugin.register_output('forward', self)
 
   def initialize
@@ -108,9 +108,10 @@ class ForwardOutput < BufferedOutput
 
   def shutdown
     @finished = true
-    @usock.close
+    @loop.watchers.each {|w| w.detach }
     @loop.stop
     @thread.join
+    @usock.close
   end
 
   def run
@@ -128,14 +129,14 @@ class ForwardOutput < BufferedOutput
     end
   end
 
-  def write(chunk)
+  def write_objects(tag, es)
     wlen = @weight_array.length
     wlen.times do
       node = @weight_array[@rr]
       @rr = (@rr + 1) % wlen
 
       if node.available?
-        send_data(node, chunk)
+        send_data(node, tag, es)
         return
       end
     end
@@ -147,7 +148,7 @@ class ForwardOutput < BufferedOutput
   # MessagePack FixArray length = 2
   FORWARD_HEADER = [0x92].pack('C')
 
-  def send_data(node, chunk)
+  def send_data(node, tag, es)
     sock = connect(node)
     begin
       opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
@@ -160,10 +161,10 @@ class ForwardOutput < BufferedOutput
       sock.write FORWARD_HEADER
 
       # writeRaw(tag)
-      sock.write chunk.key.to_msgpack  # tag
+      sock.write tag.to_msgpack  # tag
 
       # beginRaw(size)
-      sz = chunk.size
+      sz = es.size
       #if sz < 32
       #  # FixRaw
       #  sock.write [0xa0 | sz].pack('C')
@@ -176,7 +177,7 @@ class ForwardOutput < BufferedOutput
       #end
 
       # writeRawBody(packed_es)
-      chunk.write_to(sock)
+      es.write_to(sock)
     ensure
       sock.close
     end
@@ -202,7 +203,12 @@ class ForwardOutput < BufferedOutput
     return if @finished
     @nodes.each_pair {|sockaddr,n|
       n.tick
-      @usock.send "", 0, sockaddr
+      begin
+        @usock.send "", 0, sockaddr
+      rescue
+        # TODO log
+        $log.debug "failed to send heartbeat packet to #{Socket.unpack_sockaddr_in(sockaddr).reverse.join(':')}", :error=>$!
+      end
     }
   end
 

data/lib/fluent/process.rb

@@ -0,0 +1,469 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class ProcessDetachInterrupt < StandardError
+end
+
+
+class DetachProcessManager
+  require 'singleton'
+  include Singleton
+
+  class Broker
+    def initialize
+    end
+
+    def engine
+      Engine
+    end
+  end
+
+  def initialize
+    require 'drb'
+    DRb.start_service("drbunix:", Broker.new)
+    @parent_uri = DRb.uri
+  end
+
+  def fork(delegate_object)
+    ipr, ipw = IO.pipe
+    opr, opw = IO.pipe
+
+    pid = Process.fork
+    if pid
+      # parent process
+      ipw.close
+      opr.close
+      forward_thread = process_parent(ipr, opw, pid, delegate_object)
+      return pid, forward_thread
+    end
+
+    # child process
+    ipr.close
+    opw.close
+    forward_thread = process_child(ipw, opr, delegate_object)
+    return nil, forward_thread
+  end
+
+  def setup_delegate
+  end
+
+  private
+  def read_header(ipr)
+    sz = ipr.read(4).unpack('N')[0]
+    ipr.read(sz)
+  end
+
+  def send_header(ipw, data)
+    ipw.write [data.bytesize].pack('N')
+    ipw.write data
+    ipw.flush
+  end
+
+  private
+  def process_child(ipw, opr, delegate_object)
+    fwd = new_forwarder(ipw, 0.5)  # TODO interval
+    Engine.define_singleton_method(:emit_stream) do |tag,es|
+      fwd.emit(tag, es)
+    end
+
+    DRb.start_service("drbunix:", delegate_object)
+    child_uri = DRb.uri
+
+    send_header(ipw, child_uri)
+
+    forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
+
+    override_shared_methods(@parent_uri)
+
+    return forward_thread
+  end
+
+  def override_shared_methods(parent_uri)
+    broker = DRbObject.new_with_uri(parent_uri)
+    shared_methods.each {|(broker_accessor,target,name)|
+      remote = broker.send(broker_accessor)
+      target.define_singleton_method(name) do |*args,&block|
+        remote.send(name, *args, &block)
+      end
+    }
+  end
+
+  def shared_methods
+    [
+      #[:engine, Engine, :flush!],
+      #[:engine, Engine, :stop],
+    ]
+  end
+
+  def process_parent(ipr, opw, pid, delegate_object)
+    child_uri = read_header(ipr)
+
+    forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
+
+    #override_delegate_methods(delegate_object, child_uri)
+
+    fwd = new_forwarder(opw, 0.5)  # TODO interval
+    #delegate_object.define_singleton_method(:emit) do |tag,es,chain|
+    #  chain.next
+    #  fwd.emit(tag, es)
+    #end
+
+    forward_thread.define_singleton_method(:forwarder) do
+      fwd
+    end
+
+    return forward_thread
+  end
+
+  #def override_delegate_methods(target, child_uri)
+  #  remote = DRbObject.new_with_uri(child_uri)
+  #  delegate_methods(target).each {|name|
+  #    target.define_singleton_method(name) do |*args,&block|
+  #      remote.send(name, *args, &block)
+  #    end
+  #  }
+  #end
+  #
+  #def delegate_methods(target)
+  #  target.methods - Object.public_instance_methods
+  #end
+
+  def output_forward_main(opr, target)
+    read_event_stream(opr) {|tag,es|
+      # FIXME error
+      begin
+        target.emit(tag, es, NullOutputChain.instance)
+      rescue
+        $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+        $log.warn_backtrace
+      end
+    }
+  rescue
+    $log.error "error on output process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+    $log.error_backtrace
+    raise
+  end
+
+  def input_forward_main(ipr, pid)
+    read_event_stream(ipr) {|tag,es|
+      Engine.emit_stream(tag, es)
+    }
+  rescue
+    $log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+    $log.error_backtrace
+    raise
+  end
+
+  def read_event_stream(r, &block)
+    u = MessagePack::Unpacker.new(r)
+    cached_unpacker = MessagePack::Unpacker.new
+    begin
+      #buf = ''
+      #map = {}
+      #while true
+      #  r.readpartial(64*1024, buf)
+      #  u.feed_each(buf) {|tag,ms|
+      #    if msbuf = map[tag]
+      #      msbuf << ms
+      #    else
+      #      map[tag] = ms
+      #    end
+      #  }
+      #  unless map.empty?
+      #    map.each_pair {|tag,ms|
+      #      es = MessagePackEventStream.new(ms, cached_unpacker)
+      #      block.call(tag, es)
+      #    }
+      #    map.clear
+      #  end
+      #end
+      u.each {|tag,ms|
+        es = MessagePackEventStream.new(ms, cached_unpacker)
+        block.call(tag, es)
+      }
+    rescue EOFError
+    ensure
+      r.close
+    end
+  end
+
+  def new_forwarder(w, interval)
+    if interval < 0.2  # TODO interval
+      Forwarder.new(w)
+    else
+      DelayedForwarder.new(w, interval)
+    end
+  end
+
+  class Forwarder
+    def initialize(w)
+      @w = w
+    end
+
+    def emit(tag, es)
+      ms = es.to_msgpack_stream
+      #[tag, ms].to_msgpack(@w)  # not thread safe
+      @w.write [tag, ms].to_msgpack
+    end
+  end
+
+  class DelayedForwarder
+    def initialize(w, interval)
+      @w = w
+      @interval = interval
+      @buffer = {}
+      Thread.new(&method(:run))
+    end
+
+    def emit(tag, es)
+      if ms = @buffer[tag]
+        ms << es.to_msgpack_stream
+      else
+        @buffer[tag] = es.to_msgpack_stream
+      end
+    end
+
+    def run
+      while true
+        sleep @interval
+        @buffer.keys.each {|tag|
+          if ms = @buffer.delete(tag)
+            [tag, ms].to_msgpack(@w)
+            #@w.write [tag, ms].to_msgpack
+          end
+        }
+      end
+    rescue
+      $log.error "error on forwerder thread", :error=>$!.to_s
+      $log.error_backtrace
+      raise
+    end
+  end
+
+  class MultiForwarder
+    def initialize(forwarders)
+      @forwarders = forwarders
+      @rr = 1
+    end
+
+    def emit(tag, es)
+      forwarder = @forwarders[@rr]
+      @rr = (@rr + 1) % @forwarders.length
+      forwarder.emit(tag, es)
+    end
+  end
+end
+
+
+module DetachProcessImpl
+  def on_detach_process(i)
+  end
+
+  protected
+  def detach_process_impl(num, &block)
+    children = []
+
+    num.times do |i|
+      pid, forward_thread = DetachProcessManager.instance.fork(self)
+
+      if pid
+        # parent process
+        $log.info "detached process", :class=>self.class, :pid=>pid
+        children << [pid, forward_thread]
+        next
+      end
+
+      # child process
+      begin
+        on_detach_process(i)
+
+        block.call
+
+        # disable Engine.stop called by signal handler
+        Engine.define_singleton_method(:stop) do
+          # do nothing
+        end
+
+        # override signal handlers called by parent process
+        fin = FinishWait.new
+        trap :INT do
+          fin.stop
+        end
+        trap :TERM do
+          fin.stop
+        end
+        #forward_thread.join
+        fin.wait
+
+        exit! 0
+      ensure
+        $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
+        $log.error_backtrace
+      end
+
+      exit! 1
+    end
+
+    # parent process
+    # override shutdown method
+    define_singleton_method(:shutdown) do
+      begin
+        children.each {|pair|
+          pid = pair[0]
+          forward_thread = pair[1]
+          if pid
+            Process.kill(:TERM, pid)
+            forward_thread.join
+            Process.waitpid(pid)
+            pair[0] = nil
+          end
+        }
+      rescue
+        $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
+        $log.error_backtrace
+      end
+    end
+
+    # override emit method
+    forwarders = children.map {|pair| pair[1].forwarder }
+    if forwarders.length > 1
+      fwd = DetachProcessManager::MultiForwarder.new(forwarders)
+    else
+      fwd = forwarders[0]
+    end
+    define_singleton_method(:emit) do |tag,es,chain|
+      chain.next
+      fwd.emit(tag, es)
+    end
+  end
+
+  class FinishWait
+    def initialize
+      @finished = false
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+    end
+
+    def wait
+      @mutex.synchronize do
+        until @finished
+          @cond.wait(@mutex, 1.0)
+        end
+      end
+    end
+
+    def stop
+      return if @finished
+      @finished = true
+      @mutex.synchronize do
+        @cond.broadcast
+      end
+    end
+
+    def finished?
+      @finished
+    end
+  end
+end
+
+
+module DetachProcessMixin
+  include DetachProcessImpl
+
+  def configure(conf)
+    super
+
+    if detach_process = conf['detach_process']
+      b3v = Config.bool_value(detach_process)
+      case b3v
+      when nil
+        num = detach_process.to_i
+        if num > 1
+          $log.warn "'detach_process' parameter supports only 1 process on this plugin: #{conf}"
+        elsif num > 0
+          @detach_process = true
+        elsif detach_process =~ /0+/
+          @detach_process = false
+        else
+          @detach_process = true
+        end
+      when true
+        @detach_process = true
+      when false
+        @detach_process = false
+      end
+    end
+  end
+
+  def detach_process(&block)
+    if @detach_process
+      detach_process_impl(1, &block)
+    else
+      block.call
+    end
+  end
+end
+
+
+module DetachMultiProcessMixin
+  include DetachProcessImpl
+
+  def initialize
+    @detach_process_num = 2
+    super
+  end
+
+  def configure(conf)
+    super
+
+    if detach_process = conf['detach_process']
+      b3v = Config.bool_value(detach_process)
+      case b3v
+      when nil
+        num = detach_process.to_i
+        if num > 0
+          @detach_process = true
+          @detach_process_num = num
+        elsif detach_process =~ /0+/
+          @detach_process = false
+        else
+          @detach_process = true
+        end
+      when true
+        @detach_process = true
+      when false
+        @detach_process = false
+      end
+    end
+  end
+
+  protected
+  def detach_multi_process(&block)
+    if @detach_process
+      detach_process_impl(@detach_process_num, &block)
+    else
+      block.call
+    end
+  end
+end
+
+
+end
+