fluentd 0.10.7 → 0.10.8

data/ChangeLog

@@ -1,4 +1,16 @@
 
+Release 0.10.8 - 2011/12/03
+
+* Added Supervisor: restart process on SIGHUP or unexpected end of process
+* Added -i commandline option which allows inline config
+* Added TimeSlicedOutputTestDriver
+* BufferedOutput outputs 'retry suceeded' message
+* Use Gem::Specification instead of Gem.searcher which is obsoleted
+* Added BasicBuffer#chunk_limit -> buffer_chunk_limit alias for backward
+  compatibility
+* buf_file: fixed to work with keys which contains '/'
+
+
 Release 0.10.7 - 2011/11/16
 
 * Supports multi-threaded on buffered output plugins ('num_threads')

data/README

@@ -47,6 +47,7 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
   $ fluentd &
   $ echo '{"json":"message"}' | fluent-cat debug.test
 
+== Meta
 
 Web site::  http://fluentd.org/
 Documents:: http://fluentd.org/doc/
@@ -55,3 +56,10 @@ Author:: Sadayuki Furuhashi
 Copyright:: (c) 2011 FURUHASHI Sadayuki
 License:: Apache License, Version 2.0
 
+Contributed by:
+
+* Eiichiro Iwata
+* Masahiro Nakagawa
+* Takashi Nagayasu
+* Yuichi Tateno
+

data/README.rdoc

@@ -44,6 +44,8 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 == Quick Start
 
   $ gem install fluentd
+  $ # install sample configuration file to the directory
+  $ sudo fluentd -s
   $ fluentd &
   $ echo '{"json":"message"}' | fluent-cat debug.test
 
@@ -56,10 +58,17 @@ Author:: Sadayuki Furuhashi
 Copyright:: (c) 2011 FURUHASHI Sadayuki
 License:: Apache License, Version 2.0
 
-Contributed by:
+== Contributors:
 
-* Eiichiro Iwata
-* Masahiro Nakagawa
-* Takashi Nagayasu
-* Yuichi Tateno
+Patches contributed by:
 
+* {Abhishek Parolkar}[https://github.com/parolkar]
+* {Eiichiro Iwata}[http://github.com/eiichiroi]
+* {Francesc Esplugas}[https://github.com/fesplugas]
+* {hirochachacha}[https://github.com/hirochachacha]
+* {Masahiro Nakagawa}[https://github.com/repeatedly]
+* {Sakuro Ozawa}[https://github.com/sakuro]
+* Takashi Nagayasu
+* {Tsuyoshi Ozawa}[https://github.com/oza]
+* {uu59}[https://github.com/uu59]
+* {Yuichi Tateno}[https://github.com/hotchpotch]

data/VERSION

@@ -1 +1 @@
-0.10.7
+0.10.8

data/lib/fluent/buffer.rb

@@ -124,6 +124,11 @@ class BasicBuffer < Buffer
   config_param :buffer_chunk_limit, :size, :default => 256*1024*1024
   config_param :buffer_queue_limit, :integer, :default => 128
 
+  alias chunk_limit buffer_chunk_limit
+  alias chunk_limit= buffer_chunk_limit=
+  alias queue_limit buffer_queue_limit
+  alias queue_limit= buffer_queue_limit=
+
   def configure(conf)
     super
   end

data/lib/fluent/command/fluentd.rb

@@ -25,26 +25,28 @@ op = OptionParser.new
 op.version = Fluent::VERSION
 
 # default values
-config_path = Fluent::DEFAULT_CONFIG_PATH
-plugin_dirs = [Fluent::DEFAULT_PLUGIN_DIR]
-log_level = Fluent::Log::LEVEL_INFO
-log_file = nil
-daemonize = false
-libs = []
-setup_path = nil
-chuser = nil
-chgroup = nil
+opts = {
+  :config_path => Fluent::DEFAULT_CONFIG_PATH,
+  :plugin_dirs => [Fluent::DEFAULT_PLUGIN_DIR],
+  :log_level => Fluent::Log::LEVEL_INFO,
+  :log_path => nil,
+  :daemonize => false,
+  :libs => [],
+  :setup_path => nil,
+  :chuser => nil,
+  :chgroup => nil,
+}
 
 op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
-  setup_path = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
+  opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
 }
 
-op.on('-c', '--config PATH', "config flie path (default: #{config_path})") {|s|
-  config_path = s
+op.on('-c', '--config PATH', "config flie path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
+  opts[:config_path] = s
 }
 
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
-  plugin_dirs << s
+  opts[:plugin_dirs] << s
 }
 
 op.on('-I PATH', "add library path") {|s|
@@ -52,32 +54,37 @@ op.on('-I PATH', "add library path") {|s|
 }
 
 op.on('-r NAME', "load library") {|s|
-  libs << s
+  opts[:libs] << s
 }
 
 op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
-  daemonize = s
+  opts[:daemonize] = s
 }
 
 op.on('--user USER', "change user") {|s|
-  chuser = s
+  opts[:chuser] = s
 }
 
 op.on('--group GROUP', "change group") {|s|
-  chgroup = s
+  opts[:chgroup] = s
 }
 
 op.on('-o', '--log PATH', "log file path") {|s|
-  log_file = s
+  opts[:log_path] = s
+}
+
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-fly") {|s|
+  opts[:inline_config] = s
 }
 
+
 op.on('-v', '--verbose', "increment verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
-    case log_level
+    case opts[:log_level]
     when Fluent::Log::LEVEL_INFO
-      log_level = Fluent::Log::LEVEL_DEBUG
+      opts[:log_level] = Fluent::Log::LEVEL_DEBUG
     when Fluent::Log::LEVEL_DEBUG
-      log_level = Fluent::Log::LEVEL_TRACE
+      opts[:log_level] = Fluent::Log::LEVEL_TRACE
     end
   end
 }
@@ -101,7 +108,7 @@ rescue
 end
 
 
-if setup_path
+if setup_path = opts[:setup_path]
   require 'fileutils'
   FileUtils.mkdir_p File.join(setup_path, "plugin")
   confpath = File.join(setup_path, "fluent.conf")
@@ -117,142 +124,6 @@ if setup_path
   exit 0
 end
 
-
-if log_file && log_file != "-"
-  log_out = File.open(log_file, "a")
-else
-  log_out = STDOUT
-end
-
-$log = Fluent::Log.new(log_out, log_level)
-
-$log.enable_color(false) if log_file
-$log.enable_debug if log_level <= Fluent::Log::LEVEL_DEBUG
-
-
-require 'fluent/load'
-
-begin
-  #
-  # read config file
-  #
-  $log.info "reading config file", :path=>config_path
-  config_fname = File.basename(config_path)
-  config_basedir = File.dirname(config_path)
-  config_data = File.read(config_path)
-
-
-  #
-  # change user
-  #
-  if chgroup
-    chgid = chgroup.to_i
-    if chgid.to_s != chgroup
-      chgid = `id -g #{chgroup}`.to_i
-      if $?.to_i != 0
-        exit 1
-      end
-    end
-    Process::GID.change_privilege(chgid)
-  end
-
-  if chuser
-    chuid = chuser.to_i
-    if chuid.to_s != chuser
-      chuid = `id -u #{chuser}`.to_i
-      if $?.to_i != 0
-        exit 1
-      end
-    end
-    Process::UID.change_privilege(chuid)
-  end
-
-
-  #
-  # run configure
-  #
-  Fluent::Engine.init
-
-  libs.each {|lib|
-    require lib
-  }
-
-  plugin_dirs.each {|dir|
-    if Dir.exist?(dir)
-      dir = File.expand_path(dir)
-      Fluent::Engine.load_plugin_dir(dir)
-    end
-  }
-
-  Fluent::Engine.parse_config(config_data, config_fname, config_basedir)
-
-
-  #
-  # daemonize
-  #
-  trap :INT do
-    Fluent::Engine.stop
-  end
-
-  trap :TERM do
-    Fluent::Engine.stop
-  end
-
-  trap :HUP do
-    if log_file
-      log_out.reopen(log_file, "a")
-    end
-  end
-
-  trap :USR1 do
-    $log.info "force flushing buffered events"
-    Fluent::Engine.flush!
-  end
-
-  if daemonize
-    exit!(0) if fork
-    Process.setsid
-    exit!(0) if fork
-    File.umask(0)
-    STDIN.reopen("/dev/null")
-    STDOUT.reopen("/dev/null", "w")
-    STDERR.reopen("/dev/null", "w")
-    File.open(daemonize, "w") {|f|
-      f.write Process.pid.to_s
-    }
-  end
-
-
-  #
-  # run
-  #
-  $log.info "running fluent-#{Fluent::VERSION}"
-  Fluent::Engine.run
-
-rescue Fluent::ConfigError
-  $log.error "config error", :file=>config_path, :error=>$!.to_s
-  $log.debug_backtrace
-
-  # also STDOUT
-  if log_out != STDOUT
-    console = Fluent::Log.new(STDOUT, log_level).enable_debug
-    console.error "config error", :file=>config_path, :error=>$!.to_s
-    console.debug_backtrace
-  end
-
-  exit 1
-
-rescue
-  $log.error "unexpected error", :error=>$!.to_s
-  $log.error_backtrace
-
-  # also STDOUT
-  if log_out != STDOUT
-    console = Fluent::Log.new(STDOUT, log_level).enable_debug
-    console.error "unexpected error", :error=>$!.to_s
-    console.error_backtrace
-  end
-
-  exit 1
-end
+require 'fluent/supervisor'
+Fluent::Supervisor.new(opts).start
 

data/lib/fluent/mixin.rb

@@ -193,4 +193,154 @@ module SetTagKeyMixin
 end
 
 
+#module PlainTextFormatterMixin
+#  attr_accessor :output_include_time, :output_include_tag, :output_data_type
+#  attr_accessor :output_add_newline, :output_field_separator
+#
+#  def initialize
+#    super
+#    # default values may be overwriten by subclasses
+#    @output_include_time = true
+#    @output_include_tag = true
+#    @output_data_type = 'json'
+#    @output_field_separator = "\t"
+#    @output_add_newline = true
+#  end
+#
+#  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
+#  def configure(conf)
+#    super
+#
+#    if output_include_time = conf['output_include_time']
+#      @output_include_time = Config.bool_value(output_include_time)
+#    end
+#
+#    if output_include_tag = conf['output_include_tag']
+#      @output_include_tag = Config.bool_value(output_include_tag)
+#    end
+#
+#    if output_data_type = conf['output_data_type']
+#      @output_data_type = output_data_type
+#    end
+#
+#    if output_field_separator = conf['output_field_separator']
+#      case output_field_separator
+#      when 'SPACE'
+#        @output_field_separator = ' '
+#      when 'COMMA'
+#        @output_field_separator = ','
+#      else
+#        raise ConfigError, "Unknown output_field_separator option #{output_field_separator.dump}"
+#      end
+#    end
+#
+#    if output_add_newline = conf['output_add_newline']
+#      @output_add_newline = Config.bool_value(output_add_newline)
+#    end
+#
+#    # default timezone: utc (localtime=nil)
+#    if conf['localtime']
+#      @localtime = true
+#    elsif conf['utc']
+#      @localtime = false
+#    end
+#
+#    # mix-in default time formatter (or you can overwrite @time_format/@localtime (or @timef itself) on your own configure)
+#    if @output_include_time
+#      timef = @timef || TimeFormatter.new(@time_format, @localtime)
+#    end
+#
+#    output_field_separator = @output_field_separator
+#
+#    ##
+#    # optimize stringify_record(record) method
+#    #
+#    case @output_data_type
+#    when 'json'
+#      define_singleton_method(:stringify_record) {|record|
+#        record.to_json
+#      }
+#
+#    when /^attr:(.*)$/
+#      out_keys = $1.split(',')
+#      if out_keys.size > 1
+#        define_singleton_method(:stringify_record) {|record|
+#          out_keys.map {|attr|
+#            r = record[attr]
+#            r.respond_to?(:to_str) ? r.to_str : r.to_json
+#          }.join(output_field_separator)
+#        }
+#      elsif out_keys.size == 1
+#        out_key = out_keys[0]
+#        define_singleton_method(:stringify_record) {|record|
+#          r = record[out_key]
+#          r.respond_to?(:to_str) ? r.to_str : r.to_json
+#        }
+#      else
+#        raise ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
+#      end
+#
+#    else
+#      raise ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
+#    end
+#
+#    ##
+#    # optimize format(tag, time, record) method
+#    #
+#    if @output_include_time and @output_include_tag
+#      if @output_add_newline
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{timef.format(time)}#{output_field_separator}#{tag}#{output_field_separator}#{stringify_record(record)}\n"
+#        }
+#      else
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{timef.format(time)}#{output_field_separator}#{tag}#{output_field_separator}#{stringify_record(record)}"
+#        }
+#      end
+#
+#    elsif @output_include_time
+#      if @output_add_newline
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{timef.format(time)}#{output_field_separator}#{stringify_record(record)}\n"
+#        }
+#      else
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{timef.format(time)}#{output_field_separator}#{stringify_record(record)}"
+#        }
+#      end
+#
+#    elsif @output_include_tag
+#      if @output_add_newline
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{tag}#{output_field_separator}#{stringify_record(record)}\n"
+#        }
+#      else
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{tag}#{output_field_separator}#{stringify_record(record)}"
+#        }
+#      end
+#
+#    else # without time, tag
+#      if @output_add_newline
+#        define_singleton_method(:format) {|tag,time,record|
+#          "#{stringify_record(record)}\n"
+#        }
+#      else
+#        define_singleton_method(:format) {|tag,time,record|
+#          stringify_record(record)
+#        }
+#      end
+#    end
+#  end
+#
+#  def stringify_record(record)
+#    # will be overridden in configure
+#  end
+#
+#  def format(tag, time, record)
+#    # will be overridden in configure
+#  end
+#end
+
+
 end