fluentd 0.10.7 → 0.10.8

data/lib/fluent/output.rb

@@ -304,6 +304,7 @@ class BufferedOutput < Output
         @error_history.clear
         # Note: don't notify to other threads to prevent
         #       burst to recovered server
+        $log.warn "retry suceeded.", :instance=>object_id
       end
 
       if has_next
@@ -328,20 +329,20 @@ class BufferedOutput < Output
       end
 
       if error_count < @retry_limit
-        $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s
+        $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s, :instance=>object_id
         $log.warn_backtrace e.backtrace
 
       elsif @secondary
         if error_count == @retry_limit
-          $log.warn "failed to flush the buffer.", :error=>e.to_s
+          $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
           $log.warn "retry count exceededs limit. falling back to secondary output."
           $log.warn_backtrace e.backtrace
           retry  # retry immediately
         elsif error_count <= @retry_limit + @secondary_limit
-          $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s
+          $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s, :instance=>object_id
           $log.warn_backtrace e.backtrace
         else
-          $log.warn "failed to flush the buffer.", :error=>e.to_s
+          $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
           $log.warn "secondary retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
@@ -349,7 +350,7 @@ class BufferedOutput < Output
         end
 
       else
-        $log.warn "failed to flush the buffer.", :error=>e.to_s
+        $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
         $log.warn "retry count exceededs limit."
         $log.warn_backtrace e.backtrace
         write_abort

data/lib/fluent/plugin.rb

@@ -109,7 +109,7 @@ class PluginClass
       return
     end
 
-    # prefer LOAD_PATH
+    # prefer LOAD_PATH than gems
     files = $LOAD_PATH.map {|lp|
       lpath = File.join(lp, "#{path}.rb")
       File.exist?(lpath) ? lpath : nil
@@ -121,17 +121,32 @@ class PluginClass
     end
 
     # search gems
-    if defined?(::Gem) && ::Gem.respond_to?(:searcher)
+    if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
+      specs = Gem::Specification.find_all {|spec|
+        spec.contains_requirable_file? path
+      }
+
+      # prefer newer version
+      specs = specs.sort_by {|spec| spec.version }
+      if spec = specs.last
+        spec.require_paths.each {|lib|
+          file = "#{spec.full_gem_path}/#{lib}/#{path}"
+          require file
+        }
+      end
+
+    # backward compatibility for rubygems < 1.8
+    elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
       #files = Gem.find_files(path).sort
       specs = Gem.searcher.find_all(path)
-      specs = specs.sort_by {|spec| spec.version }
 
       # prefer newer version
+      specs = specs.sort_by {|spec| spec.version }
       specs.reverse_each {|spec|
         files = Gem.searcher.matching_files(spec, path)
         unless files.empty?
           require files.first
-          return
+          break
         end
       }
     end

data/lib/fluent/plugin/buf_file.rb

@@ -157,11 +157,11 @@ class FileBuffer < BasicBuffer
 
   protected
   def encode_key(key)
-		URI.encode(key, /[^-_.a-zA-Z0-9]/n)
+    URI.escape(key, /[^-_.a-zA-Z0-9]/n)
   end
 
   def decode_key(encoded_key)
-		URI.decode(encoded_key, /[^-_.a-zA-Z0-9]/n)
+    URI.unescape(encoded_key)
   end
 
   def make_path(encoded_key, bq)

data/lib/fluent/process.rb

@@ -18,10 +18,6 @@
 module Fluent
 
 
-class ProcessDetachInterrupt < StandardError
-end
-
-
 class DetachProcessManager
   require 'singleton'
   include Singleton
@@ -37,13 +33,13 @@ class DetachProcessManager
 
   def initialize
     require 'drb'
-    DRb.start_service("drbunix:", Broker.new)
+    DRb.start_service(create_drb_uri, Broker.new)
     @parent_uri = DRb.uri
   end
 
   def fork(delegate_object)
-    ipr, ipw = IO.pipe
-    opr, opw = IO.pipe
+    ipr, ipw = IO.pipe  # child Engine.emit_stream -> parent Engine.emit_stream
+    opr, opw = IO.pipe  # parent target.emit -> child target.emit
 
     pid = Process.fork
     if pid
@@ -61,9 +57,6 @@ class DetachProcessManager
     return nil, forward_thread
   end
 
-  def setup_delegate
-  end
-
   private
   def read_header(ipr)
     sz = ipr.read(4).unpack('N')[0]
@@ -76,20 +69,27 @@ class DetachProcessManager
     ipw.flush
   end
 
+  def create_drb_uri
+    "drbunix:"  # TODO
+  end
+
   private
   def process_child(ipw, opr, delegate_object)
+    DRb.start_service(create_drb_uri, delegate_object)
+    child_uri = DRb.uri
+
+    send_header(ipw, child_uri)
+
+    # override target.emit_stream to write event stream to the pipe
     fwd = new_forwarder(ipw, 0.5)  # TODO interval
     Engine.define_singleton_method(:emit_stream) do |tag,es|
       fwd.emit(tag, es)
     end
 
-    DRb.start_service("drbunix:", delegate_object)
-    child_uri = DRb.uri
-
-    send_header(ipw, child_uri)
-
+    # read event stream from the pipe and forward to target.emit
     forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
 
+    # override global methods to call parent process
     override_shared_methods(@parent_uri)
 
     return forward_thread
@@ -115,16 +115,17 @@ class DetachProcessManager
   def process_parent(ipr, opw, pid, delegate_object)
     child_uri = read_header(ipr)
 
+    # read event stream from the pipe and forward to Engine.emit_stream
     forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
 
+    # note: don't override methods in parent process
+    #       because another process may fork after overriding
     #override_delegate_methods(delegate_object, child_uri)
 
+    # return forwarder for DetachProcessMixin to
+    # override target.emit and write event stream to the pipe
     fwd = new_forwarder(opw, 0.5)  # TODO interval
-    #delegate_object.define_singleton_method(:emit) do |tag,es,chain|
-    #  chain.next
-    #  fwd.emit(tag, es)
-    #end
-
+    # note: override emit method on DetachProcessMixin
     forward_thread.define_singleton_method(:forwarder) do
       fwd
     end
@@ -147,7 +148,7 @@ class DetachProcessManager
 
   def output_forward_main(opr, target)
     read_event_stream(opr) {|tag,es|
-      # FIXME error
+      # FIXME error handling
       begin
         target.emit(tag, es, NullOutputChain.instance)
       rescue
@@ -163,7 +164,13 @@ class DetachProcessManager
 
   def input_forward_main(ipr, pid)
     read_event_stream(ipr) {|tag,es|
-      Engine.emit_stream(tag, es)
+      # FIXME error handling
+      begin
+        Engine.emit_stream(tag, es)
+      rescue
+        $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+        $log.warn_backtrace
+      end
     }
   rescue
     $log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
@@ -309,7 +316,7 @@ module DetachProcessImpl
         trap :TERM do
           fin.stop
         end
-        #forward_thread.join
+        #forward_thread.join  # TODO this thread won't stop because parent doesn't close pipe
         fin.wait
 
         exit! 0
@@ -322,28 +329,29 @@ module DetachProcessImpl
     end
 
     # parent process
-    # override shutdown method
+    # override shutdown method to kill child processes
     define_singleton_method(:shutdown) do
-      begin
-        children.each {|pair|
+      children.each {|pair|
+        begin
           pid = pair[0]
           forward_thread = pair[1]
           if pid
             Process.kill(:TERM, pid)
-            forward_thread.join
+            forward_thread.join   # wait until child closes pipe
             Process.waitpid(pid)
             pair[0] = nil
           end
-        }
-      rescue
-        $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
-        $log.error_backtrace
-      end
+        rescue
+          $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
+          $log.error_backtrace
+        end
+      }
     end
 
-    # override emit method
+    # override target.emit and write event stream to the pipe
     forwarders = children.map {|pair| pair[1].forwarder }
     if forwarders.length > 1
+      # use roundrobin
       fwd = DetachProcessManager::MultiForwarder.new(forwarders)
     else
       fwd = forwarders[0]

data/lib/fluent/supervisor.rb

@@ -0,0 +1,307 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class Supervisor
+  class LoggerInitializer
+    def initialize(path, level)
+      @path = path
+      @level = level
+    end
+
+    def init
+      if @path && @path != "-"
+        @io = File.open(@path, "a")
+      else
+        @io = STDOUT
+      end
+
+      $log = Fluent::Log.new(@io, @level)
+
+      $log.enable_color(false) if @path
+      $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
+    end
+
+    def stdout?
+      @io == STDOUT
+    end
+
+    def reopen!
+      if @path && @path != "-"
+        @io.reopen(@path, "a")
+      end
+      self
+    end
+  end
+
+  def initialize(opt)
+    @config_path = opt[:config_path]
+    @log_path = opt[:log_path]
+    @log_level = opt[:log_level]
+    @daemonize = opt[:daemonize]
+    @chgroup = opt[:chgroup]
+    @chuser = opt[:chuser]
+    @libs = opt[:libs]
+    @plugin_dirs = opt[:plugin_dirs]
+    @inline_config = opt[:inline_config]
+
+    @log = LoggerInitializer.new(@log_path, @log_level)
+    @finished = false
+    @main_pid = nil
+  end
+
+  def start
+    require 'fluent/load'
+    @log.init
+
+    start_daemonize if @daemonize
+    install_supervisor_signal_handlers
+    until @finished
+      supervise do
+        read_config
+        change_privilege
+        init_engine
+        install_main_process_signal_handlers
+        run_configure
+        finish_daemonize if @daemonize
+        run_engine
+        exit 0
+      end
+    end
+  end
+
+  private
+  def start_daemonize
+    @wait_daemonize_pipe_r, @wait_daemonize_pipe_w = IO.pipe
+
+    if fork
+      # console process
+      @wait_daemonize_pipe_w.close
+      @wait_daemonize_pipe_w = nil
+      wait_daemonize
+      exit 0
+    end
+
+    # daemonize intermediate process
+    @wait_daemonize_pipe_r.close
+    @wait_daemonize_pipe_r = nil
+
+    Process.setsid
+    exit!(0) if fork
+    File.umask(0)
+
+    # supervisor process
+    @supervisor_pid = Process.pid
+  end
+
+  def wait_daemonize
+    supervisor_pid = @wait_daemonize_pipe_r.read
+    if supervisor_pid.empty?
+      # initialization failed
+      exit! 1
+    end
+
+    @wait_daemonize_pipe_r.close
+    @wait_daemonize_pipe_r = nil
+
+    # write pid file
+    File.open(@daemonize, "w") {|f|
+      f.write supervisor_pid
+    }
+  end
+
+  def finish_daemonize
+    if @wait_daemonize_pipe_w
+      STDIN.reopen("/dev/null")
+      STDOUT.reopen("/dev/null", "w")
+      STDERR.reopen("/dev/null", "w")
+      @wait_daemonize_pipe_w.write @supervisor_pid.to_s
+      @wait_daemonize_pipe_w.close
+      @wait_daemonize_pipe_w = nil
+    end
+  end
+
+  def supervise(&block)
+    start_time = Time.now
+
+    $log.info "starting fluentd-#{Fluent::VERSION}"
+    @main_pid = fork do
+      main_process(&block)
+    end
+
+    if @daemonize && @wait_daemonize_pipe_w
+      STDIN.reopen("/dev/null")
+      STDOUT.reopen("/dev/null", "w")
+      STDERR.reopen("/dev/null", "w")
+      @wait_daemonize_pipe_w.close
+      @wait_daemonize_pipe_w = nil
+    end
+
+    Process.waitpid(@main_pid)
+    @main_pid = nil
+    ecode = $?.to_i
+
+    $log.info "process finished", :code=>ecode
+
+    if !@finished && Time.now - start_time < 1
+      $log.warn "process died within 1 second. exit."
+      exit ecode
+    end
+  end
+
+  def main_process(&block)
+    begin
+      block.call
+
+    rescue Fluent::ConfigError
+      $log.error "config error", :file=>@config_path, :error=>$!.to_s
+      $log.debug_backtrace
+      unless @log.stdout?
+        console = Fluent::Log.new(STDOUT, @log_level).enable_debug
+        console.error "config error", :file=>@config_path, :error=>$!.to_s
+        console.debug_backtrace
+      end
+
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+      unless @log.stdout?
+        console = Fluent::Log.new(STDOUT, @log_level).enable_debug
+        console.error "unexpected error", :error=>$!.to_s
+        console.error_backtrace
+      end
+    end
+
+    exit! 1
+  end
+
+  def install_supervisor_signal_handlers
+    trap :INT do
+      @finished = true
+      if pid = @main_pid
+        Process.kill(:INT, pid)
+      end
+    end
+
+    trap :TERM do
+      @finished = true
+      if pid = @main_pid
+        Process.kill(:TERM, pid)
+      end
+    end
+
+    trap :HUP do
+      $log.info "restarting"
+      if pid = @main_pid
+        Process.kill(:TERM, pid)
+      end
+    end
+
+    trap :USR1 do
+      @log.reopen!
+      if pid = @main_pid
+        Process.kill(:USR1, pid)
+      end
+    end
+  end
+
+  def read_config
+    $log.info "reading config file", :path=>@config_path
+    @config_fname = File.basename(@config_path)
+    @config_basedir = File.dirname(@config_path)
+    @config_data = File.read(@config_path)
+    if @inline_config == '-'
+      @config_data << "\n" << STDIN.read
+    elsif @inline_config
+      @config_data << "\n" << @inline_config.gsub("\\n","\n")
+    end
+  end
+
+  def run_configure
+    Fluent::Engine.parse_config(@config_data, @config_fname, @config_basedir)
+  end
+
+  def change_privilege
+    if @chgroup
+      chgid = @chgroup.to_i
+      if chgid.to_s != @chgroup
+        chgid = `id -g #{@chgroup}`.to_i
+        if $?.to_i != 0
+          exit 1
+        end
+      end
+      Process::GID.change_privilege(chgid)
+    end
+
+    if @chuser
+      chuid = @chuser.to_i
+      if chuid.to_s != @chuser
+        chuid = `id -u #{@chuser}`.to_i
+        if $?.to_i != 0
+          exit 1
+        end
+      end
+      Process::UID.change_privilege(chuid)
+    end
+  end
+
+  def init_engine
+    require 'fluent/load'
+    Fluent::Engine.init
+
+    @libs.each {|lib|
+      require lib
+    }
+
+    @plugin_dirs.each {|dir|
+      if Dir.exist?(dir)
+        dir = File.expand_path(dir)
+        Fluent::Engine.load_plugin_dir(dir)
+      end
+    }
+  end
+
+  def install_main_process_signal_handlers
+    trap :INT do
+      Fluent::Engine.stop
+    end
+
+    trap :TERM do
+      Fluent::Engine.stop
+    end
+
+    trap :HUP do
+      # TODO
+    end
+
+    trap :USR1 do
+      $log.info "force flushing buffered events"
+      @log.reopen!
+      Fluent::Engine.flush!
+    end
+  end
+
+  def run_engine
+    Fluent::Engine.run
+  end
+end
+
+
+end
+