fluentd 0.10.6 → 0.10.7

data/ChangeLog

@@ -1,4 +1,13 @@
 
+Release 0.10.7 - 2011/11/16
+
+* Supports multi-threaded on buffered output plugins ('num_threads')
+* Supports multi-process on input plugins ('detach_process')
+* Added ObjectBufferedOutput
+* Ensure to call 'shutdown' method of started plugins
+* in_tail supports 'pos_file' option that stores read position to a file
+
+
 Release 0.10.6 - 2011/11/11
 
 * Fixed --group NAME cmdline argument

data/README

@@ -0,0 +1,57 @@
+= Fluent
+
+Fluent is an event collector system. It's said that Fluent is generalized version of syslogd, which can deal with JSON object for the log message.
+
+== Archtecture
+
+Fluent collects events from various data sources and write them to files, database or other storages:
+
+    
+    Web apps  ---+                 +--> file
+                 |                 |
+                 +-->           ---+
+    /var/log  ------>  fluentd  ------> mail
+                 +-->           ---+
+                 |                 |
+    apache    ----                 +--> fluentd
+    
+
+Fluent also supports log transfer:
+
+    
+    Web server
+    +---------+
+    | fluentd -------
+    +---------+|     |
+     +---------+     |
+                     |
+    Proxy server     |    Log server, Amazon S3, HDFS, ...
+    +---------+      +--> +---------+
+    | fluentd ----------> | fluentd ||
+    +---------+|     +--> +---------+|
+     +---------+     |     +---------+
+                     |
+    Database server  |
+    +---------+      |
+    | fluentd ---------> mail
+    +---------+|
+     +---------+
+    
+
+An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time when the event occurs. Record is a JSON object.
+
+
+== Quick Start
+
+  $ gem install fluentd
+  $ fluentd &
+  $ echo '{"json":"message"}' | fluent-cat debug.test
+
+
+Web site::  http://fluentd.org/
+Documents:: http://fluentd.org/doc/
+Source repository:: http://github.com/fluent
+Author:: Sadayuki Furuhashi
+Copyright:: (c) 2011 FURUHASHI Sadayuki
+License:: Apache License, Version 2.0
+

data/Rakefile

@@ -2,8 +2,32 @@ require 'rake'
 require 'rake/testtask'
 require 'rake/clean'
 
-require 'bundler'
-Bundler::GemHelper.install_tasks
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "fluentd"
+    gemspec.summary = "Fluent event collector"
+    gemspec.author = "Sadayuki Furuhashi"
+    gemspec.email = "frsyuki@gmail.com"
+    gemspec.homepage = "http://fluentd.org/"
+    gemspec.has_rdoc = false
+    gemspec.require_paths = ["lib"]
+    gemspec.add_dependency "msgpack", "~> 0.4.4"
+    gemspec.add_dependency "json", ">= 1.4.3"
+    gemspec.add_dependency "yajl-ruby", "~> 1.0.0"
+    gemspec.add_dependency "cool.io", "~> 1.0.0"
+    gemspec.add_dependency "http_parser.rb", "~> 0.5.1"
+    gemspec.add_development_dependency "rake", ">= 0.9.2"
+    gemspec.test_files = Dir["test/**/*.rb"]
+    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
+      %w[fluent.conf VERSION AUTHORS Rakefile COPYING fluentd.gemspec]
+    gemspec.executables = ['fluentd', 'fluent-cat', 'fluent-gem']
+    gemspec.required_ruby_version = '~> 1.9.2'
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
 
 Rake::TestTask.new(:test) do |t|
   t.test_files = Dir['test/*_test.rb']
@@ -11,6 +35,21 @@ Rake::TestTask.new(:test) do |t|
   t.ruby_opts << '-I.'
 end
 
+VERSION_FILE = "lib/fluent/version.rb"
+
+file VERSION_FILE => ["VERSION"] do |t|
+  version = File.read("VERSION").strip
+  File.open(VERSION_FILE, "w") {|f|
+    f.write <<EOF
+module Fluent
+
+VERSION = '#{version}'
+
+end
+EOF
+  }
+end
+
 task :test => [:base_test]
 
 Rake::TestTask.new(:base_test) do |t|
@@ -20,5 +59,5 @@ Rake::TestTask.new(:base_test) do |t|
   #t.warning = true
 end
 
-task :default => [:build]
+task :default => [VERSION_FILE, :build]
 

data/VERSION

@@ -1 +1 @@
-0.10.6
+0.10.7

data/bin/fluent-cat

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/cat'

data/bin/fluent-gem

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+require 'rubygems/gem_runner'
+require 'rubygems/exceptions'
+begin
+  Gem::GemRunner.new.run ARGV
+rescue Gem::SystemExitException => e
+  exit e.exit_code
+end

data/bin/fluentd

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'fluent/command/fluentd'

data/lib/fluent/buffer.rb

@@ -114,6 +114,11 @@ class BasicBuffer < Buffer
 
   def initialize
     super
+    @parallel_pop = true
+  end
+
+  def enable_parallel(b=true)
+    @parallel_pop = b
   end
 
   config_param :buffer_chunk_limit, :size, :default => 256*1024*1024
@@ -152,9 +157,10 @@ class BasicBuffer < Buffer
         top << data
         return false
 
-      elsif data.bytesize > @buffer_chunk_limit
-        # TODO
-        raise BufferError, "received data too large"
+      ## FIXME
+      #elsif data.bytesize > @buffer_chunk_limit
+      #  # TODO
+      #  raise BufferError, "received data too large"
 
       elsif @queue.size >= @buffer_queue_limit
         # TODO
@@ -220,7 +226,7 @@ class BasicBuffer < Buffer
   def pop(out)
     chunk = nil
     @queue.synchronize do
-      if @parallel
+      if @parallel_pop
         chunk = @queue.find {|c| c.try_mon_enter }
         return false unless chunk
       else

data/lib/fluent/config.rb

@@ -283,6 +283,8 @@ module Configurable
             Proc.new {|val| val }
           when :integer
             Proc.new {|val| val.to_i }
+          when :float
+            Proc.new {|val| val.to_f }
           when :size
             Proc.new {|val| Config.size_value(val) }
           when :bool

data/lib/fluent/engine.rb

@@ -23,6 +23,8 @@ class EngineClass
     @matches = []
     @sources = []
     @match_cache = {}
+    @started = []
+    @default_loop = nil
   end
 
   def init
@@ -129,23 +131,33 @@ class EngineClass
   end
 
   def run
-    start
+    begin
+      start
 
-    if match?($log.tag)
-      $log.enable_event
-    end
-
-    # for empty loop
-    Coolio::Loop.default.attach Coolio::TimerWatcher.new(1, true)
-    # TODO attach async watch for thread pool
-    Coolio::Loop.default.run
+      if match?($log.tag)
+        $log.enable_event
+      end
 
-    shutdown
+      # for empty loop
+      @default_loop = Coolio::Loop.default
+      @default_loop.attach Coolio::TimerWatcher.new(1, true)
+      # TODO attach async watch for thread pool
+      @default_loop.run
+
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+    ensure
+      shutdown
+    end
   end
 
   def stop
     $log.info "shutting down fluentd"
-    Coolio::Loop.default.stop
+    if @default_loop
+      @default_loop.stop
+      @default_loop = nil
+    end
     nil
   end
 
@@ -153,18 +165,22 @@ class EngineClass
   def start
     @matches.each {|m|
       m.start
+      @started << m
     }
     @sources.each {|s|
       s.start
+      @started << s
     }
   end
 
   def shutdown
-    @matches.each {|m|
-      m.shutdown rescue nil
-    }
-    @sources.each {|s|
-      s.shutdown rescue nil
+    @started.reverse_each {|s|
+      begin
+        s.shutdown
+      rescue
+        $log.warn "unexpected error while shutting down", :error=>$!.to_s
+        $log.warn_backtrace
+      end
     }
   end
 

data/lib/fluent/load.rb

@@ -15,6 +15,7 @@ require 'fluent/log'
 require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/mixin'
+require 'fluent/process'
 require 'fluent/plugin'
 require 'fluent/parser'
 require 'fluent/event'

data/lib/fluent/output.rb

@@ -77,63 +77,18 @@ end
 class OutputThread
   def initialize(output)
     @output = output
-    @error_history = []
     @flush_now = false
     @finish = false
-
-    @flush_interval = 60  # TODO default
-    @retry_limit = 17  # TODO default
-    @retry_wait = 1.0  # TODO default
-
-    @secondary = nil
-    @secondary_limit = 8  # TODO default
+    @next_time = Engine.now + 1.0
   end
 
-  attr_accessor :flush_interval, :retry_limit, :retry_wait, :secondary_limit
-
   def configure(conf)
-    if flush_interval = conf['flush_interval']
-      @flush_interval = Config.time_value(flush_interval).to_i
-    end
-
-    if retry_limit = conf['retry_limit']
-      @retry_limit = retry_limit.to_i
-      if @retry_limit < 0
-        raise ConfigError, "invalid parameter 'retry_limit #{retry_limit}'"
-      end
-    end
-
-    if retry_wait = conf['retry_wait']
-      @retry_wait = Config.time_value(retry_wait)
-    end
-
-    # TODO configure retry_pattern
-
-    if econf = conf.elements.select {|e| e.name == 'secondary' }.first
-      type = econf['type'] || conf['type']
-      @secondary = Plugin.new_output(type)
-      @secondary.configure(econf)
-
-      if secondary_limit = conf['secondary_limit']
-        @secondary_limit = secondary_limit.to_i
-        if @secondary_limit < 0
-          raise ConfigError, "invalid parameter 'secondary_limit #{secondary_limit}'"
-        end
-      end
-
-      @secondary.secondary_init(@output)
-    end
   end
 
   def start
     @mutex = Mutex.new
     @cond = ConditionVariable.new
-    @last_try = Engine.now
-    #calc_next_time()
-    # TODO
-    @next_time = @last_try
     @thread = Thread.new(&method(:run))
-    @secondary.start if @secondary
   end
 
   def shutdown
@@ -141,7 +96,6 @@ class OutputThread
     @mutex.synchronize {
       @cond.signal
     }
-    @secondary.shutdown if @secondary
     Thread.pass
     @thread.join
   end
@@ -159,33 +113,25 @@ class OutputThread
     @mutex.lock
     begin
       until @finish
+        time = Engine.now
 
-        now = Engine.now
-
-        if @next_time <= now ||
-            (@flush_now && @error_history.empty?)
-
+        if @flush_now || @next_time <= time
           @mutex.unlock
           begin
-            try_flush(now)
+            @next_time = @output.try_flush
           ensure
             @mutex.lock
           end
-
-          now = @last_try = Engine.now
-          @flush_now = false
-        end
-
-        next_wait = calc_next_time() - now
-        if next_wait > 0
-          cond_wait(next_wait)
+          next_wait = @next_time - Engine.now
+        else
+          next_wait = @next_time - time
         end
 
+        cond_wait(next_wait) if next_wait > 0
       end
     ensure
       @mutex.unlock
     end
-
   rescue
     $log.error "error on output thread", :error=>$!.to_s
     $log.error_backtrace
@@ -196,17 +142,6 @@ class OutputThread
     }
   end
 
-  def calc_next_time
-    # TODO retry pattern
-    if @error_history.empty?
-      @next_time = @last_try + @flush_interval
-    elsif @error_history.size <= @retry_limit
-      @next_time = @last_try + @retry_wait * (2 ** (@error_history.size-1))
-    else
-      @next_time = @last_try + @retry_wait * (2 ** (@error_history.size-2-@retry_limit))
-    end
-  end
-
   if ConditionVariable.new.method(:wait).arity == 1
     $log.warn "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
     require 'timeout'
@@ -221,77 +156,25 @@ class OutputThread
       @cond.wait(@mutex, sec)
     end
   end
-
-  def try_flush(time)
-    while true
-
-      begin
-        if @error_history.size > @retry_limit
-          has_next = @output.flush_secondary(@secondary)
-        else
-          has_next = @output.try_flush
-        end
-
-        @error_history.clear
-        next if has_next
-
-      rescue
-        e = $!
-        error_count = @error_history.size
-        @error_history << time
-
-        if error_count < @retry_limit
-          $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s
-          $log.warn_backtrace e.backtrace
-
-        elsif @secondary
-          if error_count == @retry_limit
-            $log.warn "failed to flush the buffer.", :error=>e.to_s
-            $log.warn "retry count exceededs limit. falling back to secondary output."
-            $log.warn_backtrace e.backtrace
-            next  # retry immediately
-          elsif error_count <= @retry_limit + @secondary_limit
-            $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s
-            $log.warn_backtrace e.backtrace
-          else
-            $log.warn "failed to flush the buffer.", :error=>e.to_s
-            $log.warn "secondary retry count exceededs limit."
-            $log.warn_backtrace e.backtrace
-            write_abort
-          end
-
-        else
-          $log.warn "failed to flush the buffer.", :error=>e.to_s
-          $log.warn "retry count exceededs limit."
-          $log.warn_backtrace e.backtrace
-          write_abort
-        end
-
-      end
-
-      break
-    end
-  end
-
-  def write_abort
-    $log.error "throwing away old logs."
-    begin
-      @output.write_abort
-    rescue
-      $log.error "unexpected error while aborting", :error=>$!.to_s
-      $log.error_backtrace
-    end
-    @error_history.clear
-  end
 end
 
 
 class BufferedOutput < Output
   def initialize
     super
+    @next_flush_time = 0
+    @last_retry_time = 0
+    @next_retry_time = 0
+    @error_history = []
+    @error_history.extend(MonitorMixin)
+    @secondary_limit = 8
   end
 
   config_param :buffer_type, :string, :default => 'memory'
+  config_param :flush_interval, :time, :default => 60
+  config_param :retry_limit, :integer, :default => 17
+  config_param :retry_wait, :float, :default => 1.0
+  config_param :num_threads, :integer, :default => 1
 
   def configure(conf)
     super
@@ -299,17 +182,46 @@ class BufferedOutput < Output
     @buffer = Plugin.new_buffer(@buffer_type)
     @buffer.configure(conf)
 
-    @writer = OutputThread.new(self)
-    @writer.configure(conf)
+    if @buffer.respond_to?(:enable_parallel)
+      if @num_threads == 1
+        @buffer.enable_parallel(false)
+      else
+        @buffer.enable_parallel(true)
+      end
+    end
+
+    @writers = (1..@num_threads).map {
+      writer = OutputThread.new(self)
+      writer.configure(conf)
+      writer
+    }
+
+    if sconf = conf.elements.select {|e| e.name == 'secondary' }.first
+      type = sconf['type'] || conf['type']
+      @secondary = Plugin.new_output(type)
+      @secondary.configure(sconf)
+
+      if secondary_limit = conf['secondary_limit']
+        @secondary_limit = secondary_limit.to_i
+        if @secondary_limit < 0
+          raise ConfigError, "invalid parameter 'secondary_limit #{secondary_limit}'"
+        end
+      end
+
+      @secondary.secondary_init(self)
+    end
   end
 
   def start
+    @next_flush_time = Engine.now + @flush_interval
     @buffer.start
-    @writer.start
+    @secondary.start if @secondary
+    @writers.each {|writer| writer.start }
   end
 
   def shutdown
-    @writer.shutdown
+    @writers.each {|writer| writer.shutdown }
+    @secondary.shutdown if @secondary
     @buffer.shutdown
   end
 
@@ -321,7 +233,8 @@ class BufferedOutput < Output
   end
 
   def submit_flush
-    @writer.submit_flush
+    # TODO roundrobin?
+    @writers.first.submit_flush
   end
 
   def format_stream(tag, es)
@@ -338,15 +251,113 @@ class BufferedOutput < Output
   #def write(chunk)
   #end
 
+  def enqueue_buffer
+    @buffer.keys.each {|key|
+      @buffer.push(key)
+    }
+  end
+
   def try_flush
-    if @buffer.queue_size == 0
+    time = Engine.now
+
+    empty = @buffer.queue_size == 0
+    if empty && @next_flush_time < (now = Engine.now)
       @buffer.synchronize do
-        @buffer.keys.each {|key|
-          @buffer.push(key)
-        }
+        if @next_flush_time < now
+          enqueue_buffer
+          @next_flush_time = now + @flush_interval
+          empty = @buffer.queue_size == 0
+        end
+      end
+    end
+    if empty
+      return time + 1  # TODO 1
+    end
+
+    begin
+      retrying = !@error_history.empty?
+
+      if retrying
+        @error_history.synchronize do
+          if retrying = !@error_history.empty?  # re-check in synchronize
+            if @next_retry_time >= time
+              # allow retrying for only one thread
+              return time + 1  # TODO 1
+            end
+            # assume next retry failes and
+            # clear them if when it suceeds
+            @last_retry_time = time
+            @error_history << time
+            @next_retry_time += calc_retry_wait
+          end
+        end
+      end
+
+      if @secondary && @error_history.size > @retry_limit
+        has_next = flush_secondary(@secondary)
+      else
+        has_next = @buffer.pop(self)
+      end
+
+      # success
+      if retrying
+        @error_history.clear
+        # Note: don't notify to other threads to prevent
+        #       burst to recovered server
+      end
+
+      if has_next
+        return time  # call try_flush soon
+      else
+        return time + 1  # TODO 1
+      end
+
+    rescue => e
+      if retrying
+        error_count = @error_history.size
+      else
+        # first error
+        error_count = 0
+        @error_history.synchronize do
+          if @error_history.empty?
+            @last_retry_time = time
+            @error_history << time
+            @next_retry_time = time + calc_retry_wait
+          end
+        end
+      end
+
+      if error_count < @retry_limit
+        $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s
+        $log.warn_backtrace e.backtrace
+
+      elsif @secondary
+        if error_count == @retry_limit
+          $log.warn "failed to flush the buffer.", :error=>e.to_s
+          $log.warn "retry count exceededs limit. falling back to secondary output."
+          $log.warn_backtrace e.backtrace
+          retry  # retry immediately
+        elsif error_count <= @retry_limit + @secondary_limit
+          $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s
+          $log.warn_backtrace e.backtrace
+        else
+          $log.warn "failed to flush the buffer.", :error=>e.to_s
+          $log.warn "secondary retry count exceededs limit."
+          $log.warn_backtrace e.backtrace
+          write_abort
+          @error_history.clear
+        end
+
+      else
+        $log.warn "failed to flush the buffer.", :error=>e.to_s
+        $log.warn "retry count exceededs limit."
+        $log.warn_backtrace e.backtrace
+        write_abort
+        @error_history.clear
       end
+
+      return @next_retry_time
     end
-    @buffer.pop(self)
   end
 
   def before_shutdown
@@ -358,8 +369,24 @@ class BufferedOutput < Output
     end
   end
 
+  def calc_retry_wait
+    # TODO retry pattern
+    if @error_history.size <= @retry_limit
+      @retry_wait * (2 ** (@error_history.size-1))
+    else
+      # secondary retry
+      @retry_wait * (2 ** (@error_history.size-2-@retry_limit))
+    end
+  end
+
   def write_abort
-    @buffer.clear!
+    $log.error "throwing away old logs."
+    begin
+      @buffer.clear!
+    rescue
+      $log.error "unexpected error while aborting", :error=>$!.to_s
+      $log.error_backtrace
+    end
   end
 
   def flush_secondary(secondary)
@@ -368,6 +395,42 @@ class BufferedOutput < Output
 end
 
 
+class ObjectBufferedOutput < BufferedOutput
+  def initialize
+    super
+  end
+
+  def emit(tag, es, chain)
+    data = es.to_msgpack_stream
+    key = tag
+    if @buffer.emit(key, data, chain)
+      submit_flush
+    end
+  end
+
+  module BufferedEventStreamMixin
+    include Enumerable
+
+    def repeatable?
+      true
+    end
+
+    def each(&block)
+      msgpack_each(&block)
+    end
+
+    def to_msgpack_stream
+      read
+    end
+  end
+
+  def write(chunk)
+    chunk.extend(BufferedEventStreamMixin)
+    write_objects(chunk.key, chunk)
+  end
+end
+
+
 class TimeSlicedOutput < BufferedOutput
   def initialize
     super
@@ -378,7 +441,6 @@ class TimeSlicedOutput < BufferedOutput
   config_param :time_slice_format, :string, :default => '%Y%m%d'
   config_param :time_slice_wait, :time, :default => 10*60
   config_set_default :buffer_type, 'file'  # overwrite default buffer_type
-  # config_set_default :flush_interval, 60  # TODO overwrite default flush_interval
 
   attr_accessor :localtime
 
@@ -424,16 +486,13 @@ class TimeSlicedOutput < BufferedOutput
     }
   end
 
-  def try_flush
+  def enqueue_buffer
     nowslice = @time_slicer.call(Engine.now.to_i - @time_slice_wait)
-    @buffer.synchronize do
-      @buffer.keys.each {|key|
-        if key < nowslice
-          @buffer.push(key)
-        end
-      }
-    end
-    @buffer.pop(self)
+    @buffer.keys.each {|key|
+      if key < nowslice
+        @buffer.push(key)
+      end
+    }
   end
 
   #def format(tag, event)