fluentd 0.10.29 → 0.10.30

data/ChangeLog

@@ -1,4 +1,16 @@
 
+Release 0.10.30 - 2012/12/06
+
+* Shows configuration file at start time
+* Changed ruby version dependency from ~> 1.9.2 to => 1.9.2 so that it works
+  with ruby-2.0.
+* TimeSlicedOutput: overwrites default value of buffer_chunk_limit to 256m.
+  fluent-plugin-s3 will be affected.
+* TextParser/in_tail: supports 'nginx' preset formats
+* in/out_forward and in_syslog: support IPv6
+* out_forward: fixed a out of bounds bug
+
+
 Release 0.10.29 - 2012/11/26
 
 * buf_file: added flush_at_shutdown option to force flush before shuttingdown

data/Rakefile

@@ -25,7 +25,7 @@ begin
     gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
       %w[fluent.conf VERSION AUTHORS Rakefile COPYING fluentd.gemspec Gemfile]
     gemspec.executables = ['fluentd', 'fluent-cat', 'fluent-gem', 'fluent-debug']
-    gemspec.required_ruby_version = '~> 1.9.2'
+    gemspec.required_ruby_version = '>= 1.9.2'
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-0.10.29
+0.10.30

data/lib/fluent/engine.rb

@@ -55,6 +55,8 @@ class EngineClass
   end
 
   def configure(conf)
+    $log.info "using configuration file: #{conf.to_s.rstrip}"
+
     conf.elements.select {|e|
       e.name == 'source'
     }.each {|e|

data/lib/fluent/mixin.rb

@@ -217,155 +217,4 @@ module SetTagKeyMixin
   end
 end
 
-
-#module PlainTextFormatterMixin
-#  attr_accessor :output_include_time, :output_include_tag, :output_data_type
-#  attr_accessor :output_add_newline, :output_field_separator
-#
-#  def initialize
-#    super
-#    # default values may be overwriten by subclasses
-#    @output_include_time = true
-#    @output_include_tag = true
-#    @output_data_type = 'json'
-#    @output_field_separator = "\t"
-#    @output_add_newline = true
-#  end
-#
-#  # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
-#  def configure(conf)
-#    super
-#
-#    if output_include_time = conf['output_include_time']
-#      @output_include_time = Config.bool_value(output_include_time)
-#    end
-#
-#    if output_include_tag = conf['output_include_tag']
-#      @output_include_tag = Config.bool_value(output_include_tag)
-#    end
-#
-#    if output_data_type = conf['output_data_type']
-#      @output_data_type = output_data_type
-#    end
-#
-#    if output_field_separator = conf['output_field_separator']
-#      case output_field_separator
-#      when 'SPACE'
-#        @output_field_separator = ' '
-#      when 'COMMA'
-#        @output_field_separator = ','
-#      else
-#        raise ConfigError, "Unknown output_field_separator option #{output_field_separator.dump}"
-#      end
-#    end
-#
-#    if output_add_newline = conf['output_add_newline']
-#      @output_add_newline = Config.bool_value(output_add_newline)
-#    end
-#
-#    # default timezone: utc (localtime=nil)
-#    if conf['localtime']
-#      @localtime = true
-#    elsif conf['utc']
-#      @localtime = false
-#    end
-#
-#    # mix-in default time formatter (or you can overwrite @time_format/@localtime (or @timef itself) on your own configure)
-#    if @output_include_time
-#      timef = @timef || TimeFormatter.new(@time_format, @localtime)
-#    end
-#
-#    output_field_separator = @output_field_separator
-#
-#    ##
-#    # optimize stringify_record(record) method
-#    #
-#    case @output_data_type
-#    when 'json'
-#      define_singleton_method(:stringify_record) {|record|
-#        record.to_json
-#      }
-#
-#    when /^attr:(.*)$/
-#      out_keys = $1.split(',')
-#      if out_keys.size > 1
-#        define_singleton_method(:stringify_record) {|record|
-#          out_keys.map {|attr|
-#            r = record[attr]
-#            r.respond_to?(:to_str) ? r.to_str : r.to_json
-#          }.join(output_field_separator)
-#        }
-#      elsif out_keys.size == 1
-#        out_key = out_keys[0]
-#        define_singleton_method(:stringify_record) {|record|
-#          r = record[out_key]
-#          r.respond_to?(:to_str) ? r.to_str : r.to_json
-#        }
-#      else
-#        raise ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
-#      end
-#
-#    else
-#      raise ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
-#    end
-#
-#    ##
-#    # optimize format(tag, time, record) method
-#    #
-#    if @output_include_time and @output_include_tag
-#      if @output_add_newline
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{timef.format(time)}#{output_field_separator}#{tag}#{output_field_separator}#{stringify_record(record)}\n"
-#        }
-#      else
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{timef.format(time)}#{output_field_separator}#{tag}#{output_field_separator}#{stringify_record(record)}"
-#        }
-#      end
-#
-#    elsif @output_include_time
-#      if @output_add_newline
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{timef.format(time)}#{output_field_separator}#{stringify_record(record)}\n"
-#        }
-#      else
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{timef.format(time)}#{output_field_separator}#{stringify_record(record)}"
-#        }
-#      end
-#
-#    elsif @output_include_tag
-#      if @output_add_newline
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{tag}#{output_field_separator}#{stringify_record(record)}\n"
-#        }
-#      else
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{tag}#{output_field_separator}#{stringify_record(record)}"
-#        }
-#      end
-#
-#    else # without time, tag
-#      if @output_add_newline
-#        define_singleton_method(:format) {|tag,time,record|
-#          "#{stringify_record(record)}\n"
-#        }
-#      else
-#        define_singleton_method(:format) {|tag,time,record|
-#          stringify_record(record)
-#        }
-#      end
-#    end
-#  end
-#
-#  def stringify_record(record)
-#    # will be overridden in configure
-#  end
-#
-#  def format(tag, time, record)
-#    # will be overridden in configure
-#  end
-#end
-
-
 end

data/lib/fluent/output.rb

@@ -451,6 +451,7 @@ class TimeSlicedOutput < BufferedOutput
   config_param :time_slice_format, :string, :default => '%Y%m%d'
   config_param :time_slice_wait, :time, :default => 10*60
   config_set_default :buffer_type, 'file'  # overwrite default buffer_type
+  config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
   config_set_default :flush_interval, nil
 
   attr_accessor :localtime

data/lib/fluent/parser.rb

@@ -205,6 +205,7 @@ class TextParser
     'json' => Proc.new { JSONParser.new },
     'tsv' => Proc.new { TSVParser.new },
     'csv' => Proc.new { CSVParser.new },
+    'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
   }
 
   def self.register_template(name, regexp_or_proc, time_format=nil)

data/lib/fluent/plugin/in_forward.rb

@@ -23,6 +23,7 @@ class ForwardInput < Input
 
   def initialize
     super
+    require 'fluent/plugin/socket_util'
   end
 
   config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
@@ -38,7 +39,7 @@ class ForwardInput < Input
     @lsock = listen
     @loop.attach(@lsock)
 
-    @usock = UDPSocket.new
+    @usock = SocketUtil.create_udp_socket(@bind)
     @usock.bind(@bind, @port)
     @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
     @loop.attach(@hbr)

data/lib/fluent/plugin/in_syslog.rb

@@ -65,6 +65,7 @@ class SyslogInput < Input
 
   def initialize
     super
+    require 'fluent/plugin/socket_util'
   end
 
   config_param :port, :integer, :default => 5140
@@ -93,7 +94,7 @@ class SyslogInput < Input
     @loop = Coolio::Loop.new
 
     $log.debug "listening syslog socket on #{@bind}:#{@port}"
-    @usock = UDPSocket.new
+    @usock = SocketUtil.create_udp_socket(@bind)
     @usock.bind(@bind, @port)
 
     @handler = UdpHandler.new(@usock, callback)

data/lib/fluent/plugin/out_file.rb

@@ -62,9 +62,6 @@ class FileOutput < TimeSlicedOutput
       conf['buffer_path'] ||= "#{@path}.*"
     end
 
-    # override default buffer_chunk_limit
-    conf['buffer_chunk_limit'] ||= '256m'
-
     super
 
     @timef = TimeFormatter.new(@time_format, @localtime)

data/lib/fluent/plugin/out_forward.rb

@@ -25,6 +25,7 @@ class ForwardOutput < ObjectBufferedOutput
     super
     require 'socket'
     require 'fileutils'
+    require 'fluent/plugin/socket_util'
     @nodes = []  #=> [Node]
   end
 
@@ -88,7 +89,8 @@ class ForwardOutput < ObjectBufferedOutput
 
     @loop = Coolio::Loop.new
 
-    @usock = UDPSocket.new
+    # Assume all hosts are same protocol.
+    @usock = SocketUtil.create_udp_socket(@nodes.first.host)
     @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
     @loop.attach(@hb)
 
@@ -126,8 +128,8 @@ class ForwardOutput < ObjectBufferedOutput
 
     wlen = @weight_array.length
     wlen.times do
-      node = @weight_array[@rr]
       @rr = (@rr + 1) % wlen
+      node = @weight_array[@rr]
 
       if node.available?
         begin
@@ -148,6 +150,7 @@ class ForwardOutput < ObjectBufferedOutput
   end
 
   private
+
   def rebuild_weight_array
     standby_nodes, regular_nodes = @nodes.partition {|n|
       n.standby?

data/lib/fluent/plugin/socket_util.rb

@@ -0,0 +1,14 @@
+module Fluent
+  module SocketUtil
+    def create_udp_socket(host)
+      require 'ipaddr'
+
+      if IPAddr.new(IPSocket.getaddress(host)).ipv4?
+        UDPSocket.new
+      else
+        UDPSocket.new(Socket::AF_INET6)
+      end
+    end
+    module_function :create_udp_socket
+  end
+end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.29'
+VERSION = '0.10.30'
 
 end

data/test/plugin/in_syslog.rb

@@ -3,6 +3,7 @@ require 'fluent/test'
 class SyslogInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    require 'fluent/plugin/socket_util'
   end
 
   CONFIG = %[
@@ -11,36 +12,46 @@ class SyslogInputTest < Test::Unit::TestCase
     tag syslog
   ]
 
+  IPv6_CONFIG = %[
+    port 9911
+    bind ::1
+    tag syslog
+  ]
+
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::SyslogInput).configure(conf)
   end
 
   def test_configure
-    d = create_driver
-    assert_equal 9911, d.instance.port
-    assert_equal '127.0.0.1', d.instance.bind
+    {'127.0.0.1' => CONFIG, '::1' => IPv6_CONFIG}.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal 9911, d.instance.port
+      assert_equal k, d.instance.bind
+    }
   end
 
   def test_time_format
-    d = create_driver
+    {'127.0.0.1' => CONFIG, '::1' => IPv6_CONFIG}.each_pair { |k, v|
+      d = create_driver(v)
 
-    tests = [
-      {'msg' => '<6>Sep 11 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-      {'msg' => '<6>Sep  1 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
-    ]
+      tests = [
+        {'msg' => '<6>Sep 11 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
+        {'msg' => '<6>Sep  1 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
+      ]
 
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', 9911)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
+      d.run do
+        u = Fluent::SocketUtil.create_udp_socket(k)
+        u.connect(k, 9911)
+        tests.each {|test|
+          u.send(test['msg'], 0)
+        }
+        sleep 1
+      end
 
-    emits = d.emits
-    emits.each_index {|i|
-      assert_equal(tests[i]['expected'], emits[i][1])
+      emits = d.emits
+      emits.each_index {|i|
+        assert_equal(tests[i]['expected'], emits[i][1])
+      }
     }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.29
+  version: 0.10.30
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-26 00:00:00.000000000 Z
+date: 2012-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -216,6 +216,7 @@ files:
 - lib/fluent/plugin/out_stdout.rb
 - lib/fluent/plugin/out_stream.rb
 - lib/fluent/plugin/out_test.rb
+- lib/fluent/plugin/socket_util.rb
 - lib/fluent/process.rb
 - lib/fluent/status.rb
 - lib/fluent/supervisor.rb
@@ -254,7 +255,7 @@ require_paths:
 required_ruby_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ~>
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement