fluentd 0.10.4 → 0.10.5

data/lib/fluent/plugin/out_stream.rb

@@ -18,6 +18,7 @@
 module Fluent
 
 
+# obsolete
 class StreamOutput < BufferedOutput
   def initialize
     require 'socket'
@@ -84,11 +85,14 @@ class StreamOutput < BufferedOutput
 end
 
 
+# obsolete
 class TcpOutput < StreamOutput
   Plugin.register_output('tcp', self)
 
   def initialize
     super
+    ## TODO
+    #$log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
   end
 
   config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
@@ -104,11 +108,14 @@ class TcpOutput < StreamOutput
 end
 
 
+# obsolete
 class UnixOutput < StreamOutput
   Plugin.register_output('unix', self)
 
   def initialize
     super
+    ## TODO
+    #$log.warn "'unix' output is obsoleted and will be removed."
   end
 
   config_param :path, :string

data/lib/fluent/plugin/out_test.rb

@@ -22,15 +22,25 @@ class TestOutput < Output
   Plugin.register_output('test', self)
 
   def initialize
-    @emits = []
+    @emit_streams = []
     @name = nil
   end
 
-  attr_reader :emits, :name
+  attr_reader :emit_streams, :name
+
+  def emits
+    all = []
+    @emit_streams.each {|tag,events|
+      events.each {|time,record|
+        all << [tag, time, record]
+      }
+    }
+    all
+  end
 
   def events
     all = []
-    @emits.each {|tag,events|
+    @emit_streams.each {|tag,events|
       all.concat events
     }
     all
@@ -38,7 +48,7 @@ class TestOutput < Output
 
   def records
     all = []
-    @emits.each {|tag,events|
+    @emit_streams.each {|tag,events|
       events.each {|time,record|
         all << record
       }
@@ -60,7 +70,7 @@ class TestOutput < Output
 
   def emit(tag, es, chain)
     chain.next
-    @emits << [tag, es.to_a]
+    @emit_streams << [tag, es.to_a]
   end
 end
 

data/lib/fluent/test/input_test.rb

@@ -22,7 +22,7 @@ module Test
 class InputTestDriver < TestDriver
   def initialize(klass, &block)
     super(klass, &block)
-    @emits = []
+    @emit_streams = []
     @expects = nil
   end
 
@@ -35,11 +35,21 @@ class InputTestDriver < TestDriver
     @expects ||= []
   end
 
-  attr_reader :emits
+  attr_reader :emit_streams
+
+  def emits
+    all = []
+    @emit_streams.each {|tag,events|
+      events.each {|time,record|
+        all << [tag, time, record]
+      }
+    }
+    all
+  end
 
   def events
     all = []
-    @emits.each {|tag,events|
+    @emit_streams.each {|tag,events|
       all.concat events
     }
     all
@@ -47,7 +57,7 @@ class InputTestDriver < TestDriver
 
   def records
     all = []
-    @emits.each {|tag,events|
+    @emit_streams.each {|tag,events|
       events.each {|time,record|
         all << record
       }
@@ -66,7 +76,7 @@ class InputTestDriver < TestDriver
 
       if @expects
         i = 0
-        @emits.each {|tag,events|
+        @emit_streams.each {|tag,events|
           events.each {|time,record|
             assert_equal(@expects[i], [tag, time, record])
             i += 1
@@ -80,7 +90,7 @@ class InputTestDriver < TestDriver
 
   private
   def emit_stream(tag, es)
-    @emits << [tag, es.to_a]
+    @emit_streams << [tag, es.to_a]
   end
 end
 

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.4'
+VERSION = '0.10.5'
 
 end

data/test/plugin/in_exec.rb

@@ -0,0 +1,44 @@
+require 'fluent/test'
+require 'net/http'
+
+class ExecInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    command bash -c "echo -e '2011-01-02 13:14:15\ttag1\tok'"
+    keys time,tag,k1
+    time_key time
+    tag_key tag
+    time_format %Y-%m-%d %H:%M:%S
+    run_interval 1s
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ExecInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal ["time","tag","k1"], d.instance.keys
+    assert_equal "tag", d.instance.tag_key
+    assert_equal "time", d.instance.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
+  end
+
+  def test_emit
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert_equal true, emits.length > 0
+    assert_equal ["tag1", time, {"k1"=>"ok"}], emits[0]
+  end
+end
+

data/test/plugin/in_forward.rb

@@ -0,0 +1,123 @@
+require 'fluent/test'
+
+class ForwardInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    port 13998
+    bind 127.0.0.1
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 13998, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+  end
+
+  def connect
+    TCPSocket.new('127.0.0.1', 13998)
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_data [tag, 0, record].to_msgpack
+      }
+      sleep 0.5
+    end
+  end
+
+  def test_message
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_data [tag, time, record].to_msgpack
+      }
+      sleep 0.5
+    end
+  end
+
+  def test_forward
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = []
+      d.expected_emits.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ["tag1", entries].to_msgpack
+      sleep 0.5
+    end
+  end
+
+  def test_packed_forward
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = ''
+      d.expected_emits.each {|tag,time,record|
+        [time, record].to_msgpack(entries)
+      }
+      send_data ["tag1", entries].to_msgpack
+      sleep 0.5
+    end
+  end
+
+  def test_message_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_data [tag, time, record].to_json
+      }
+      sleep 0.5
+    end
+  end
+
+  def send_data(data)
+    io = connect
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  # TODO heartbeat
+end
+

data/test/plugin/out_exec.rb

@@ -0,0 +1,69 @@
+require 'fluent/test'
+require 'fileutils'
+
+class ExecOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
+
+  CONFIG = %[
+    buffer_path #{TMP_DIR}/buffer
+    command cat >#{TMP_DIR}/out
+    keys time,tag,k1
+    tag_key tag
+    time_key time
+    time_format %Y-%m-%d %H:%M:%S
+    localtime
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ExecOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal ["time","tag","k1"], d.instance.keys
+    assert_equal "tag", d.instance.tag_key
+    assert_equal "time", d.instance.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
+    assert_equal true, d.instance.localtime
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+    d.emit({"k1"=>"v1","kx"=>"vx"}, time)
+    d.emit({"k1"=>"v2","kx"=>"vx"}, time)
+
+    d.expect_format %[2011-01-02 13:14:15\ttest\tv1\n]
+    d.expect_format %[2011-01-02 13:14:15\ttest\tv2\n]
+
+    d.run
+  end
+
+  def test_write
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+    d.emit({"k1"=>"v1","kx"=>"vx"}, time)
+    d.emit({"k1"=>"v2","kx"=>"vx"}, time)
+
+    d.run
+
+    expect_path = "#{TMP_DIR}/out"
+    assert_equal true, File.exist?(expect_path)
+
+    data = File.read(expect_path)
+    expect_data =
+      %[2011-01-02 13:14:15\ttest\tv1\n] +
+      %[2011-01-02 13:14:15\ttest\tv2\n]
+    assert_equal expect_data, data
+  end
+end
+

data/test/plugin/out_exec_filter.rb

@@ -0,0 +1,74 @@
+require 'fluent/test'
+require 'fileutils'
+
+class ExecFilterOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    command cat
+    in_keys time,tag,k1
+    out_keys time,tag,k2
+    tag_key tag
+    time_key time
+    time_format %Y-%m-%d %H:%M:%S
+    localtime
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal ["time","tag","k1"], d.instance.in_keys
+    assert_equal ["time","tag","k2"], d.instance.out_keys
+    assert_equal "tag", d.instance.tag_key
+    assert_equal "time", d.instance.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
+    assert_equal true, d.instance.localtime
+  end
+
+  def test_emit_1
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"k1"=>1}, time)
+      d.emit({"k1"=>2}, time)
+      sleep 0.5
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+    assert_equal ["test", time, {"k2"=>"1"}], emits[0]
+    assert_equal ["test", time, {"k2"=>"2"}], emits[1]
+  end
+
+  def test_emit_2
+    d = create_driver %[
+      command cat
+      in_keys time,k1
+      out_keys time,k2
+      tag xxx
+      time_key time
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"k1"=>1}, time)
+      d.emit({"k1"=>2}, time)
+      sleep 0.5
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+    assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
+    assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
+  end
+end
+