fluentd 0.10.9 → 0.10.10

data/ChangeLog

@@ -16,7 +16,7 @@ Release 0.10.8 - 2011/12/03
 * Added Supervisor: restart process on SIGHUP or unexpected end of process
 * Added -i commandline option which allows inline config
 * Added TimeSlicedOutputTestDriver
-* BufferedOutput outputs 'retry suceeded' message
+* BufferedOutput outputs 'retry succeeded' message
 * Use Gem::Specification instead of Gem.searcher which is obsoleted
 * Added BasicBuffer#chunk_limit -> buffer_chunk_limit alias for backward
   compatibility

data/ChangeLog.cp

@@ -0,0 +1,265 @@
+
+Release 0.10.10 - 2012/01/19
+
+* in_forward and on_forward don't raise exceptions in callback handlers of
+  cool.io not to stop Loop#run
+* TimeSlicedOutput ignores time_slice_wait if flush_interval is specified
+* in_tail follows symbolic links correctly
+* in_http supports "Content-Type: application/json"
+* TestDriver#run sleeps 0.5 seconds for the out_exec_filter plugin
+
+
+Release 0.10.9 - 2012/01/19
+
+* Fixed TimeSlicedOutputTestDriver
+* Updated cool.io 1.0.0 -> 1.1.0
+* TextParser: fixed regexp of syslog parser to work with rsyslog on CentOS
+* out_exec_filter: improve performance by using buffering
+* out_exec_filter: added num_children parameter
+* out_exec_filter: added remove_prefix/add_prefix parameters
+* out_tail: show warning if pos_file parameter is not set
+* out_copy: fixed problems when event stream is not repeatable
+
+
+Release 0.10.8 - 2011/12/03
+
+* Added Supervisor: restart process on SIGHUP or unexpected end of process
+* Added -i commandline option which allows inline config
+* Added TimeSlicedOutputTestDriver
+* BufferedOutput outputs 'retry succeeded' message
+* Use Gem::Specification instead of Gem.searcher which is obsoleted
+* Added BasicBuffer#chunk_limit -> buffer_chunk_limit alias for backward
+  compatibility
+* buf_file: fixed to work with keys which contains '/'
+
+
+Release 0.10.7 - 2011/11/16
+
+* Supports multi-threaded on buffered output plugins ('num_threads')
+* Supports multi-process on input plugins ('detach_process')
+* Added ObjectBufferedOutput
+* Ensure to call 'shutdown' method of started plugins
+* in_tail supports 'pos_file' option that stores read position to a file
+
+
+Release 0.10.6 - 2011/11/11
+
+* Fixed --group NAME cmdline argument
+* Run configure(conf) after changing user and group
+* Fixed SIGHUP handling
+* BasicBuffer plugin uses Configurable module
+* buf_file uses microseconds + 12 bits random number for unique file name
+
+
+Release 0.10.5 - 2011/11/08
+
+* Added out_forward, in_forward
+* Added out_exec, in_exec
+* Added out_exec_filter
+
+
+Release 0.10.4 - 2011/11/07
+
+* TestDriver uses klass.dup.module_eval instead of inheriting class to
+  override methods
+* fixed encoding problem of String#[] in FileBufferChunk#<<(data)
+
+
+Release 0.10.3 - 2011/10/21
+
+  * Supports 'include' in config file
+    * Supports "http://" schema
+	* Supports wildcards
+
+
+Release 0.10.2 - 2011/10/18
+
+  * Fixed EventStream#to_msgpack_stream (thanks CkReal)
+  * Added gemspec.required_ruby_version = '~> 1.9.2' (thanks sakuro)
+
+
+Release 0.10.1 - 2011/10/16
+
+  * SetTimeKeyMixin accepts include_time_key parameter
+  * SetTagKeyMixin accepts include_tag_key parameter
+  * Fixed Makefile.am
+  * Fixed MemoryBufferChunk#msgpack_each
+
+
+Release 0.10.0 - 2011/10/16
+
+  * Removed Event class
+    * def format(tag, event) -> format(time, tag, record)
+	* def emit(tag, event) -> emit(tag, time, record)
+  * Added plugin APIs
+    * Added Configurable
+    * Added TimeFormatter
+    * Added SetTimeKeyMixin
+    * Added SetTagKeyMixin
+    * Added InputTestDriver
+  * Added Gemfile, gemspec and .rvmrc files
+
+
+Release 0.9.20 - 2011/10/07
+
+  * Config#has_key? sets used flag not to show warning
+  * in_tcp supports JSON
+  * Depends on Yajl
+  * Fixed fluentd -r option
+  * Fixed autogen.sh to create plugins directory
+
+
+Release 0.9.19 - 2011/10/07
+
+  * Secondary outputs work only when error count exceededs retry_limit
+  * Secondary outputs retry upto secondary_limit
+  * Added Output#secondary_init
+  * Added unit test driver for output plugins
+  * Added Fluent::Test.test? method
+  * Added comparision methods (<=>, ==, eql? and hash) to Event class
+  * Added test cases for out_copy, out_roundrobin and out_file
+
+
+Release 0.9.18 - 2011/10/04
+
+  * Performance improvements
+    * in_tcp uses lazy deserialization
+	* Engine caches results of Match#match
+	* out_file caches results of strftime
+  * Fixed TextParser.register_template
+
+
+Release 0.9.17 - 2011/10/03
+
+  * TimeSlicedOutput caches results of strftime() for performance
+  * Fixed TextParser.register_template (thanks eiichiroi)
+  * Fixed fluent-cat --port argument to work
+
+
+Release 0.9.16 - 2011/10/01
+
+  * Fixed it can't use relative path for --plugin command line argument
+  * Fixed out_copy to work with non-repeatable event streams
+
+
+Release 0.9.15 - 2011/09/21
+
+  * Set default encoding to ASCII-8BIT
+  * Fixed thread issues in BasicBuffer
+  * Performance improvements
+
+
+Release 0.9.14 - 2011/09/21
+
+  * Fixed to write unexpected error to log files
+  * Plugin loader uses GemSpec if it is available to load newer plugin
+  * Added Config.bool_value method for plugins
+
+
+Release 0.9.13 - 2011/09/21
+
+  * Updated out_file plugin
+    * Uses localtime by default
+    * Uses \t to separate time, tag and JSON.
+    * Uses ISO-8601 for the time format
+
+
+Release 0.9.12 - 2011/09/20
+
+  * Updated out_file plugin
+    * Supports log compression
+	* Slices log files every day by default
+
+
+Release 0.9.11 - 2011/09/19
+
+  * Added out_null plugin
+  * Added out_roundrobin plugin
+  * in_http supports keep-alive
+  * in_tcp and in_unix detect network failure
+
+
+Release 0.9.10 - 2011/09/07
+
+  * Periodic timer enqueues buffer chunks only when queue is empty
+
+
+Release 0.9.9 - 2011/09/06
+
+  * in_tail strips \n at the end of log line
+  * Uses newer one if multiple plugins are found
+
+
+Release 0.9.8 - 2011/08/16
+
+  * in_http plugin supports 'Expect: 100-continue' header
+  * Added --user and --group command line arguments to change privilege
+  * Changed default retry_limit parameter from 8 to 17
+  * Changed default buffer_chunk_limit parameter from 1m to 16m
+  * Changed default buffer_queue_limit parameter from 100 to 64
+
+
+Release 0.9.7 - 2011/08/06
+
+  * Load plugins named fluent_plugin.rb installed by RubyGems
+  * Load plugins located on fluent/plugin/{buf,in,out}_{type} installed by RubyGems
+  * Removed out_tdlog plugin to fluent-plugin-td gem
+
+
+Release 0.9.6 - 2011/08/06
+
+  * Force flush buffered events on SIGUSR1
+  * out_tdlog uses event tag for the database and table name
+  * Fluent can log Fluent logs
+  * Colorize Fluent logs on console
+  * Changed default unix domain socket path to /var/run/fluent/fluent.sock
+  * Makefile.am and configure.in aware fakeroot
+
+
+Release 0.9.5 - 2011/07/23
+
+  * Added <secondary> tag that fallbacks to secondary output when it can't
+    write to primary output.
+  * <match> pattern supports **; matches any fragments recursively.
+  * <match> pattern supports {x,y,...}; matches any of the inner patterns.
+
+
+Release 0.9.4 - 2011/07/22
+
+  * Switched EventMachine to Cool.io
+  * in_tail continues to follow file if it is deleted on Linux
+  * Added fluentd --setup option that installs sample configuration file
+  * Added out_tdlog plugin
+
+
+Release 0.9.3 - 2011/07/05
+
+  * Disable EventMachine.epoll because it doesn't work correctly on
+    Ruby 1.9 with threads
+  * in_tcp and in_unix uses EventMachine instead of threads
+  * Set BasicSocket.do_not_reverse_lookup = true
+
+
+Release 0.9.2 - 2011/07/03
+
+  * Added TimeSlicedOutput that splits buffer chunk exactly based on the time
+  * Changed structure of buffers from queue to map + queue
+  * Show waring if configuration parameters are not used
+  * Added out_time_file plugin
+
+
+Release 0.9.1 - 2011/06/26
+
+  * Added in_tail plugin
+  * Added in_syslog plugin
+  * Added out_stdout plugin
+  * Added out_tcp plugin
+  * Added out_unix plugin
+  * Improved performance of in_http plugin by replacing WEBrick + thread
+     with EventMachine
+
+
+Release 0.9.0 - 2011/06/20
+
+  * First release
+

data/README.rdoc

@@ -1,19 +1,19 @@
-= Fluent
+= Fluentd
 
-Fluent is an event collector system. It's said that Fluent is generalized version of syslogd, which can deal with JSON object for the log message.
+Fluentd is an event collector system. It is a generalized version of syslogd, which handles JSON objects for its log messages.
 
-== Archtecture
+== Architecture
 
-Fluent collects events from various data sources and write them to files, database or other storages:
+Fluentd collects events from various data sources and writes them to files, database or other types of storages:
 
     
     Web apps  ---+                 +--> file
                  |                 |
                  +-->           ---+
-    /var/log  ------>  fluentd  ------> mail
+    /var/log  ------>  Fluentd  ------> mail
                  +-->           ---+
                  |                 |
-    apache    ----                 +--> fluentd
+    Apache    ----                 +--> Fluentd
     
 
 Fluent also supports log transfer:
@@ -21,24 +21,24 @@ Fluent also supports log transfer:
     
     Web server
     +---------+
-    | fluentd -------
+    | Fluentd -------
     +---------+|     |
      +---------+     |
                      |
     Proxy server     |    Log server, Amazon S3, HDFS, ...
     +---------+      +--> +---------+
-    | fluentd ----------> | fluentd ||
+    | Fluentd ----------> | Fluentd ||
     +---------+|     +--> +---------+|
      +---------+     |     +---------+
                      |
     Database server  |
     +---------+      |
-    | fluentd ---------> mail
+    | Fluentd ---------> mail
     +---------+|
      +---------+
     
 
-An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time when the event occurs. Record is a JSON object.
+An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
 
 
 == Quick Start
@@ -54,6 +54,7 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 Web site::  http://fluentd.org/
 Documents:: http://fluentd.org/doc/
 Source repository:: http://github.com/fluent
+Discussion:: http://groups.google.com/group/fluentd
 Author:: Sadayuki Furuhashi
 Copyright:: (c) 2011 FURUHASHI Sadayuki
 License:: Apache License, Version 2.0
@@ -67,9 +68,13 @@ Patches contributed by:
 * {Eiichiro Iwata}[http://github.com/eiichiroi]
 * {Francesc Esplugas}[https://github.com/fesplugas]
 * {hirochachacha}[https://github.com/hirochachacha]
+* {Hiro Yoshikawa}[https://github.com/hiroyoshikawa]
 * {Masahiro Nakagawa}[https://github.com/repeatedly]
+* {Nathan Parry}[https://github.com/nparry]
 * {Sakuro Ozawa}[https://github.com/sakuro]
-* Takashi Nagayasu
+* {Satoshi Tagomori}[https://github.com/tagomoris]
+* {Takashi Nagayasu}[https://github.com/sumipan]
 * {Tsuyoshi Ozawa}[https://github.com/oza]
 * {uu59}[https://github.com/uu59]
+* {Waldemar Quevedo Salinas}[https://github.com/wallyqs]
 * {Yuichi Tateno}[https://github.com/hotchpotch]

data/VERSION

@@ -1 +1 @@
-0.10.9
+0.10.10

data/lib/fluent/engine.rb

@@ -108,7 +108,7 @@ class EngineClass
     end
     target.emit(tag, es)
   rescue
-    $log.warn "emit transaction faild ", :error=>$!.to_s
+    $log.warn "emit transaction failed ", :error=>$!.to_s
     $log.warn_backtrace
     raise
   end

data/lib/fluent/output.rb

@@ -285,7 +285,7 @@ class BufferedOutput < Output
               return time + 1  # TODO 1
             end
             # assume next retry failes and
-            # clear them if when it suceeds
+            # clear them if when it succeeds
             @last_retry_time = time
             @error_history << time
             @next_retry_time += calc_retry_wait
@@ -304,7 +304,7 @@ class BufferedOutput < Output
         @error_history.clear
         # Note: don't notify to other threads to prevent
         #       burst to recovered server
-        $log.warn "retry suceeded.", :instance=>object_id
+        $log.warn "retry succeeded.", :instance=>object_id
       end
 
       if has_next
@@ -442,6 +442,7 @@ class TimeSlicedOutput < BufferedOutput
   config_param :time_slice_format, :string, :default => '%Y%m%d'
   config_param :time_slice_wait, :time, :default => 10*60
   config_set_default :buffer_type, 'file'  # overwrite default buffer_type
+  config_set_default :flush_interval, nil
 
   attr_accessor :localtime
 
@@ -468,6 +469,28 @@ class TimeSlicedOutput < BufferedOutput
     @time_slice_cache_interval = time_slice_cache_interval
     @before_tc = nil
     @before_key = nil
+
+    if @flush_interval
+      if conf['time_slice_wait']
+        $log.warn "time_slice_wait is ignored if flush_interval is specified: #{conf}"
+      end
+      @enqueue_buffer_proc = Proc.new do
+        @buffer.keys.each {|key|
+          @buffer.push(key)
+        }
+      end
+
+    else
+      @flush_interval = [60, @time_slice_cache_interval].min
+      @enqueue_buffer_proc = Proc.new do
+        nowslice = @time_slicer.call(Engine.now.to_i - @time_slice_wait)
+        @buffer.keys.each {|key|
+          if key < nowslice
+            @buffer.push(key)
+          end
+        }
+      end
+    end
   end
 
   def emit(tag, es, chain)
@@ -488,12 +511,7 @@ class TimeSlicedOutput < BufferedOutput
   end
 
   def enqueue_buffer
-    nowslice = @time_slicer.call(Engine.now.to_i - @time_slice_wait)
-    @buffer.keys.each {|key|
-      if key < nowslice
-        @buffer.push(key)
-      end
-    }
+    @enqueue_buffer_proc.call
   end
 
   #def format(tag, event)

data/lib/fluent/plugin/in_forward.rb

@@ -164,10 +164,18 @@ class ForwardInput < Input
 
     def on_read_json(data)
       @y << data
+    rescue
+      $log.error "forward error: #{$!.to_s}"
+      $log.error_backtrace
+      close
     end
 
     def on_read_msgpack(data)
       @u.feed_each(data, &@on_message)
+    rescue
+      $log.error "forward error: #{$!.to_s}"
+      $log.error_backtrace
+      close
     end
 
     def on_close
@@ -187,6 +195,8 @@ class ForwardInput < Input
       host = addr[3]
       port = addr[1]
       @callback.call(host, port, msg)
+    rescue
+      # TODO log?
     end
   end
 

data/lib/fluent/plugin/in_http.rb

@@ -232,6 +232,8 @@ class HttpInput < Input
       elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
         boundary = WEBrick::HTTPUtils.dequote($1)
         params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
+      elsif @content_type =~ /^application\/json/
+        params['json'] = @body
       end
       path_info = @parser.request_path
 

data/lib/fluent/plugin/in_tail.rb

@@ -109,7 +109,7 @@ class TailInput < Input
 
   class Handler < Coolio::StatWatcher
     def initialize(path, pe, callback)
-      stat = File.lstat(path)
+      stat = File.stat(path)
       @pe = pe
       @inode = stat.ino
       if @inode == @pe.read_inode
@@ -131,7 +131,7 @@ class TailInput < Input
       inode = nil
 
       File.open(path) {|f|
-        stat = f.lstat
+        stat = f.stat
         inode = stat.ino
 
         if @inode != inode || stat.size < @pos

data/lib/fluent/plugin/out_forward.rb

@@ -196,6 +196,8 @@ class ForwardOutput < ObjectBufferedOutput
 
     def on_timer
       @callback.call
+    rescue
+      # TODO log?
     end
   end
 
@@ -225,6 +227,8 @@ class ForwardOutput < ObjectBufferedOutput
       port = addr[1]
       sockaddr = Socket.pack_sockaddr_in(port, host)
       @callback.call(sockaddr, msg)
+    rescue
+      # TODO log?
     end
   end
 

data/lib/fluent/test/base.rb

@@ -51,7 +51,7 @@ class TestDriver
     @instance.start
     begin
       # wait until thread starts
-      10.times { sleep 0.01 }
+      10.times { sleep 0.05 }
       return yield
     ensure
       @instance.shutdown

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.9'
+VERSION = '0.10.10'
 
 end

data/test/plugin/in_http.rb

@@ -58,6 +58,25 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_application_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        http = Net::HTTP.new("127.0.0.1", 9911)
+        req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
+        req.body = record.to_json
+        res = http.request(req)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
   def test_msgpack
     d = create_driver
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.9
+  version: 0.10.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-20 00:00:00.000000000Z
+date: 2012-02-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70352789356340 !ruby/object:Gem::Requirement
+  requirement: &70325962479480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70352789356340
+  version_requirements: *70325962479480
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70352789355760 !ruby/object:Gem::Requirement
+  requirement: &70325962478220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70352789355760
+  version_requirements: *70325962478220
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70352789355160 !ruby/object:Gem::Requirement
+  requirement: &70325962474920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70352789355160
+  version_requirements: *70325962474920
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70352789343800 !ruby/object:Gem::Requirement
+  requirement: &70325962473640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70352789343800
+  version_requirements: *70325962473640
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70352789343300 !ruby/object:Gem::Requirement
+  requirement: &70325962469580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70352789343300
+  version_requirements: *70325962469580
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70352789342760 !ruby/object:Gem::Requirement
+  requirement: &70325962465420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70352789342760
+  version_requirements: *70325962465420
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -86,6 +86,7 @@ executables:
 extensions: []
 extra_rdoc_files:
 - ChangeLog
+- ChangeLog.cp
 - README
 - README.rdoc
 files:
@@ -154,6 +155,7 @@ files:
 - test/plugin/out_roundrobin.rb
 - test/plugin/out_stream.rb
 - ChangeLog
+- ChangeLog.cp
 - README
 - README.rdoc
 homepage: http://fluentd.org/