fluentd 0.10.34 → 0.10.35

data/ChangeLog

@@ -1,4 +1,13 @@
 
+Release 0.10.35 - 2013/06/12
+
+* Log: emits fluentd's log via background thread
+* Log: use #inspect for non to_msgpack-able object in log event processing
+* FailureDetector/out_forward: fixed wrong phi calculation
+* in_stream: fixed unix domain connection leak when parsing json or msgpack failed
+* buf_file/out_file: sets 0644 permission to each created file for preventing
+  File.umask(0) effect at daemonize mode
+
 Release 0.10.34 - 2013/06/03
 
 * fluentd: supports --emit-error-log-interval to suppress log messages with the

data/lib/fluent/engine.rb

@@ -27,12 +27,18 @@ class EngineClass
     @started = []
     @default_loop = nil
 
+    @log_emit_thread = nil
+    @log_event_loop_stop = false
+    @log_event_queue = []
+
     @suppress_emit_error_log_interval = 0
     @next_emit_error_log_time = nil
   end
 
   MATCH_CACHE_SIZE = 1024
 
+  LOG_EMIT_INTERVAL = 0.1
+
   attr_reader :matches, :sources
 
   def init
@@ -155,12 +161,34 @@ class EngineClass
     Time.now.to_i
   end
 
+  def log_event_loop
+    $log.disable_events(Thread.current)
+
+    while sleep(LOG_EMIT_INTERVAL)
+      break if @log_event_loop_stop
+      next if @log_event_queue.empty?
+
+      # NOTE: thead-safe of slice! depends on GVL
+      events = @log_event_queue.slice!(0..-1)
+      next if events.empty?
+
+      events.each {|tag,time,record|
+        begin
+          Engine.emit(tag, time, record)
+        rescue
+          $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error => $!
+        end
+      }
+    end
+  end
+
   def run
     begin
       start
 
       if match?($log.tag)
         $log.enable_event
+        @log_emit_thread = Thread.new(&method(:log_event_loop))
       end
 
       # for empty loop
@@ -173,12 +201,16 @@ class EngineClass
       $log.error "unexpected error", :error=>$!.to_s
       $log.error_backtrace
     ensure
+      $log.info "shutting down fluentd"
       shutdown
+      if @log_emit_thread
+        @log_event_loop_stop = true
+        @log_emit_thread.join
+      end
     end
   end
 
   def stop
-    $log.info "shutting down fluentd"
     if @default_loop
       @default_loop.stop
       @default_loop = nil
@@ -186,6 +218,11 @@ class EngineClass
     nil
   end
 
+  def push_log_event(tag, time, record)
+    return if @log_emit_thread.nil?
+    @log_event_queue.push([tag, time, record])
+  end
+
   private
   def start
     @matches.each {|m|

data/lib/fluent/env.rb

@@ -3,4 +3,5 @@ DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
 DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
 DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
 DEFAULT_LISTEN_PORT = 24224
+DEFAULT_FILE_PERMISSION = 0644
 end

data/lib/fluent/log.rb

@@ -50,6 +50,8 @@ class Log
     @tag = 'fluent'
     @time_format = '%Y-%m-%d %H:%M:%S %z '
     enable_color out.tty?
+    # TODO: This variable name is unclear so we should change to better name.
+    @threads_exclude_events = []
   end
 
   attr_accessor :out
@@ -88,6 +90,12 @@ class Log
     self
   end
 
+  # If you want to suppress event emitting in specific thread, please use this method.
+  # Events in passed thread are never emitted.
+  def disable_events(thread)
+    @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
+  end
+
   def on_trace(&block)
     return if @level > LEVEL_TRACE
     block.call if block
@@ -252,21 +260,13 @@ class Log
       end
     }
 
-    if @self_event
-      c = caller
-      if c.count(c.shift) <= 0
-        record = map.dup
-        record['message'] = message.dup
-        ### In signal trap context, 'Engine.emit' cannot get Monitor lock (Thread level lock),
-        ### and ThreadError will be raised
-        begin
-          Engine.emit("#{@tag}.#{level}", time.to_i, record)
-        rescue ThreadError # message: can't be called from trap context
-          # ignore
-        end
-        # TODO: We should check 'GET_THREAD()->interrupt_mask & TRAP_INTERRUPT_MASK'
-        #       for replacement of begin-rescue if any methods exists.
-      end
+    unless @threads_exclude_events.include?(Thread.current)
+      record = map.dup
+      record.keys.each {|key|
+        record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
+      }
+      record['message'] = message.dup
+      Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
     end
 
     map.each_pair {|k,v|

data/lib/fluent/plugin/buf_file.rb

@@ -23,7 +23,7 @@ class FileBufferChunk < BufferChunk
     super(key)
     @path = path
     @unique_id = unique_id
-    @file = File.open(@path, mode)
+    @file = File.open(@path, mode, DEFAULT_FILE_PERMISSION)
     @file.sync = true
     @size = @file.stat.size
   end

data/lib/fluent/plugin/in_stream.rb

@@ -135,10 +135,18 @@ class StreamInput < Input
 
     def on_read_json(data)
       @y << data
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+      close
     end
 
     def on_read_msgpack(data)
       @u.feed_each(data, &@on_message)
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+      close
     end
 
     def on_close

data/lib/fluent/plugin/in_tail.rb

@@ -42,7 +42,7 @@ class TailInput < Input
     end
 
     if @pos_file
-      @pf_file = File.open(@pos_file, File::RDWR|File::CREAT)
+      @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
       @pf_file.sync = true
       @pf = PositionFile.parse(@pf_file)
     else

data/lib/fluent/plugin/out_file.rb

@@ -89,7 +89,7 @@ class FileOutput < TimeSlicedOutput
 
     case @compress
     when nil
-      File.open(path, "a") {|f|
+      File.open(path, "a", DEFAULT_FILE_PERMISSION) {|f|
         chunk.write_to(f)
       }
     when :gz

data/lib/fluent/plugin/out_forward.rb

@@ -380,7 +380,7 @@ class ForwardOutput < ObjectBufferedOutput
 
     def resolve_dns!
       @sockaddr = Socket.pack_sockaddr_in(@port, @host)
-      port, resolved_host = Socket.unpack_sockaddr_in(sockaddr)
+      port, resolved_host = Socket.unpack_sockaddr_in(@sockaddr)
       return resolved_host
     end
     private :resolve_dns!
@@ -480,7 +480,7 @@ class ForwardOutput < ObjectBufferedOutput
       mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1
 
       # Calculate phi of the phi accrual failure detector
-      t = now - @last
+      t = now - @last - @heartbeat_interval + 1
       phi = PHI_FACTOR * t / mean
 
       return phi

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.34'
+VERSION = '0.10.35'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.34
+  version: 0.10.35
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-03 00:00:00.000000000 Z
+date: 2013-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -284,6 +284,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 748399488166287607
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -313,4 +316,3 @@ test_files:
 - test/plugin/out_roundrobin.rb
 - test/plugin/out_stdout.rb
 - test/plugin/out_stream.rb
-has_rdoc: false