fluentd 0.10.19 → 0.10.20

data/ChangeLog

@@ -1,5 +1,12 @@
 
-Release 0.10.19 - 2012/04/13
+Release 0.10.20 - 2012/04/27
+
+* Fixed Config#to_s
+* out_exec_filter: supports 'out_format json' option
+* out_exec_filter: supports 'in_format json' option
+
+
+Release 0.10.19 - 2012/04/17
 
 * Fixed in_syslog
 * in_tail: fixed "invalid byte sequence in US-ASCII" error

data/VERSION

@@ -1 +1 @@
-0.10.19
+0.10.20

data/lib/fluent/config.rb

@@ -78,7 +78,7 @@ module Config
       if @arg.empty?
         out << "#{indent}<#{@name}>\n"
       else
-        out << "#{indent}<#{@name} #{@name}>\n"
+        out << "#{indent}<#{@name} #{@arg}>\n"
       end
       each_pair {|k,v|
         out << "#{nindent}#{k} #{v}\n"

data/lib/fluent/plugin/out_exec_filter.rb

@@ -25,15 +25,41 @@ class ExecFilterOutput < BufferedOutput
     super
   end
 
+  SUPPORTED_FORMAT = {
+    'tsv' => :tsv,
+    'json' => :json,
+    'msgpack' => :msgpack,
+  }
+
   config_param :command, :string
-  config_param :in_keys, :string
+
   config_param :remove_prefix, :string, :default => nil
-  config_param :out_keys, :string
   config_param :add_prefix, :string, :default => nil
+
+  config_param :in_format, :default => :tsv do |val|
+    f = SUPPORTED_FORMAT[val]
+    raise ConfigError, "Unsupported in_format '#{val}'" unless f
+    f
+  end
+  config_param :in_keys, :default => [] do |val|
+    val.split(',')
+  end
+
+  config_param :out_format, :default => :tsv do |val|
+    f = SUPPORTED_FORMAT[val]
+    raise ConfigError, "Unsupported out_format '#{val}'" unless f
+    f
+  end
+  config_param :out_keys, :default => [] do |val|  # for tsv format
+    val.split(',')
+  end
+
   config_param :tag, :string, :default => nil
   config_param :tag_key, :string, :default => nil
+
   config_param :time_key, :string, :default => nil
   config_param :time_format, :string, :default => nil
+
   config_param :localtime, :bool, :default => true
   config_param :num_children, :integer, :default => 1
 
@@ -52,9 +78,6 @@ class ExecFilterOutput < BufferedOutput
       raise ConfigError, "'tag' or 'tag_key' option is required on exec_filter output"
     end
 
-    @in_keys = @in_keys.split(',')
-    @out_keys = @out_keys.split(',')
-
     if @time_key
       if @time_format
         f = @time_format
@@ -74,6 +97,30 @@ class ExecFilterOutput < BufferedOutput
     if @add_prefix
       @added_prefix_string = @add_prefix + '.'
     end
+
+    case @in_format
+    when :tsv
+      if @in_keys.empty?
+        raise ConfigError, "in_keys option is required on exec_filter output for tsv in_format"
+      end
+      @formatter = TSVFormatter.new(@in_keys)
+    when :json
+      @formatter = JSONFormatter.new
+    when :msgpack
+      @formatter = MessagePackFormatter.new
+    end
+
+    case @out_format
+    when :tsv
+      if @out_keys.empty?
+        raise ConfigError, "out_keys option is required on exec_filter output for tsv in_format"
+      end
+      @parser = TSVParser.new(@out_keys, method(:on_message))
+    when :json
+      @parser = JSONParser.new(method(:on_message))
+    when :msgpack
+      @parser = MessagePackParser.new(method(:on_message))
+    end
   end
 
   def start
@@ -83,7 +130,7 @@ class ExecFilterOutput < BufferedOutput
     @rr = 0
     begin
       @num_children.times do
-        c = ChildProcess.new(&method(:each_line))
+        c = ChildProcess.new(@parser)
         c.start(@command)
         @children << c
       end
@@ -108,27 +155,22 @@ class ExecFilterOutput < BufferedOutput
   end
 
   def format_stream(tag, es)
-    out = ''
     if @remove_prefix
       if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix
         tag = tag[@removed_length..-1] || ''
       end
     end
 
+    out = ''
+
     es.each {|time,record|
-      last = @in_keys.length-1
-      for i in 0..last
-        key = @in_keys[i]
-        if key == @time_key
-          out << @time_format_proc.call(time)
-        elsif key == @tag_key
-          out << tag
-        else
-          out << record[key].to_s
-        end
-        out << "\t" if i != last
+      if @time_key
+        record[@time_key] = @time_format_proc.call(time)
       end
-      out << "\n"
+      if @tag_key
+        record[@tag_key] = tag
+      end
+      @formatter.call(record, out)
     }
 
     out
@@ -139,43 +181,11 @@ class ExecFilterOutput < BufferedOutput
     @children[r].write chunk
   end
 
-  def each_line(line)
-    line.chomp!
-    vals = line.split("\t")
-
-    tag = @tag
-    time = nil
-    record = {}
-    for i in 0..@out_keys.length-1
-      key = @out_keys[i]
-      val = vals[i]
-      if key == @time_key
-        time = @time_parse_proc.call(val)
-      elsif key == @tag_key
-        tag = if @add_prefix
-                @added_prefix_string + val
-              else
-                val
-              end
-      else
-        record[key] = val
-      end
-    end
-
-    if tag
-      time ||= Engine.now
-      Engine.emit(tag, time, record)
-    end
-  rescue
-    $log.error "exec_filter failed to emit", :error=>$!.to_s, :line=>line
-    $log.warn_backtrace $!.backtrace
-  end
-
   class ChildProcess
-    def initialize(&each_line)
+    def initialize(parser)
       @pid = nil
       @thread = nil
-      @each_line = each_line
+      @parser = parser
     end
 
     def start(command)
@@ -215,7 +225,7 @@ class ExecFilterOutput < BufferedOutput
     end
 
     def run
-      @io.each_line(&@each_line)
+      @parser.call(@io)
     rescue
       $log.error "exec_filter process exited", :error=>$!.to_s
       $log.warn_backtrace $!.backtrace
@@ -223,6 +233,106 @@ class ExecFilterOutput < BufferedOutput
       Process.waitpid(@pid)
     end
   end
+
+  class Formatter
+  end
+
+  class TSVFormatter < Formatter
+    def initialize(in_keys)
+      @in_keys = in_keys
+      super()
+    end
+
+    def call(record, out)
+      last = @in_keys.length-1
+      for i in 0..last
+        key = @in_keys[i]
+        out << record[key].to_s
+        out << "\t" if i != last
+      end
+      out << "\n"
+    end
+  end
+
+  class JSONFormatter < Formatter
+    def call(record, out)
+      out << Yajl.dump(record)
+    end
+  end
+
+  class MessagePackFormatter < Formatter
+    def call(record, out)
+      record.to_msgpack(out)
+    end
+  end
+
+  def on_message(record)
+    if val = record.delete(@time_key)
+      time = @time_parse_proc.call(val)
+    else
+      time = Engine.now
+    end
+
+    if val = record.delete(@tag_key)
+      tag = if @add_prefix
+              @added_prefix_string + val
+            else
+              val
+            end
+    else
+      tag = @tag
+    end
+
+    Engine.emit(tag, time, record)
+
+  rescue
+    $log.error "exec_filter failed to emit", :error=>$!.to_s, :record=>Yajl.dump(record)
+    $log.warn_backtrace $!.backtrace
+  end
+
+  class Parser
+    def initialize(on_message)
+      @on_message = on_message
+    end
+  end
+
+  class TSVParser < Parser
+    def initialize(out_keys, on_message)
+      @out_keys = out_keys
+      super(on_message)
+    end
+
+    def call(io)
+      io.each_line(&method(:each_line))
+    end
+
+    def each_line(line)
+      line.chomp!
+      vals = line.split("\t")
+
+      record = Hash[@out_keys.zip(vals)]
+
+      @on_message.call(record)
+    end
+  end
+
+  class JSONParser < Parser
+    def call(io)
+      y = Yajl::Parser.new
+      y.on_parse_complete = @on_message
+      y.parse(io)
+    end
+  end
+
+  class MessagePackParser < Parser
+    def call(io)
+      @u = MessagePack::Unpacker.new(io)
+      begin
+        @u.each(&@on_message)
+      rescue EOFError
+      end
+    end
+  end
 end
 
 

data/lib/fluent/plugin/out_file.rb

@@ -22,8 +22,8 @@ class FileOutput < TimeSlicedOutput
   Plugin.register_output('file', self)
 
   SUPPORTED_COMPRESS = {
-    :gz => :gz,
-    :gzip => :gz,
+    'gz' => :gz,
+    'gzip' => :gz,
   }
 
   config_param :path, :string
@@ -31,9 +31,9 @@ class FileOutput < TimeSlicedOutput
   config_param :time_format, :string, :default => nil
 
   config_param :compress, :default => nil do |val|
-    c = SUPPORTED_COMPRESS[val.to_sym]
+    c = SUPPORTED_COMPRESS[val]
     unless c
-      raise ConfigError, "Unsupported compression algorithm '#{compress}'"
+      raise ConfigError, "Unsupported compression algorithm '#{val}'"
     end
     c
   end

data/lib/fluent/plugin/out_stdout.rb

@@ -21,10 +21,14 @@ module Fluent
 class StdoutOutput < Output
   Plugin.register_output('stdout', self)
 
+  config_param :autoflush, :bool, :default => false
+
   def emit(tag, es, chain)
     es.each {|time,record|
       puts "#{Time.at(time).localtime} #{tag}: #{Yajl.dump(record)}"
     }
+    STDOUT.flush if @autoflush
+
     chain.next
   end
 end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.19'
+VERSION = '0.10.20'
 
 end

data/test/plugin/out_exec_filter.rb

@@ -156,5 +156,25 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
     assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
   end
+
+  def test_json_1
+    d = create_driver(%[
+      command cat
+      in_keys message
+      out_format json
+      time_key time
+      tag_key tag
+    ], 'input.test')
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]}, time+10)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
+  end
 end
 

data/test/plugin/out_stdout.rb

@@ -0,0 +1,28 @@
+require 'fluent/test'
+
+class StdoutOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::StdoutOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert (not d.instance.autoflush)
+
+    d = create_driver %[
+      autoflush true
+    ]
+    assert d.instance.autoflush
+  end
+
+  # def test_emit
+  # end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.19
+  version: 0.10.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-17 00:00:00.000000000Z
+date: 2012-04-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70342941850580 !ruby/object:Gem::Requirement
+  requirement: &70174884555560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70342941850580
+  version_requirements: *70174884555560
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70342941849920 !ruby/object:Gem::Requirement
+  requirement: &70174884536260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70342941849920
+  version_requirements: *70174884536260
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70342941849140 !ruby/object:Gem::Requirement
+  requirement: &70174884535660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342941849140
+  version_requirements: *70174884535660
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70342941848200 !ruby/object:Gem::Requirement
+  requirement: &70174884535020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70342941848200
+  version_requirements: *70174884535020
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70342941847500 !ruby/object:Gem::Requirement
+  requirement: &70174884534420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70342941847500
+  version_requirements: *70174884534420
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70342941846860 !ruby/object:Gem::Requirement
+  requirement: &70174884533820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70342941846860
+  version_requirements: *70174884533820
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70342941846100 !ruby/object:Gem::Requirement
+  requirement: &70174884533220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70342941846100
+  version_requirements: *70174884533220
 - !ruby/object:Gem::Dependency
   name: timecop
-  requirement: &70342941845440 !ruby/object:Gem::Requirement
+  requirement: &70174884532660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 0.3.0
   type: :development
   prerelease: false
-  version_requirements: *70342941845440
+  version_requirements: *70174884532660
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70342941844840 !ruby/object:Gem::Requirement
+  requirement: &70174884532120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70342941844840
+  version_requirements: *70174884532120
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -192,6 +192,7 @@ files:
 - test/plugin/out_file.rb
 - test/plugin/out_forward.rb
 - test/plugin/out_roundrobin.rb
+- test/plugin/out_stdout.rb
 - test/plugin/out_stream.rb
 - ChangeLog
 - README
@@ -237,4 +238,5 @@ test_files:
 - test/plugin/out_file.rb
 - test/plugin/out_forward.rb
 - test/plugin/out_roundrobin.rb
+- test/plugin/out_stdout.rb
 - test/plugin/out_stream.rb