fluentd 0.10.50 → 0.10.51

data/lib/fluent/test/base.rb

@@ -39,11 +39,11 @@ module Fluent
 
       attr_reader :instance, :config
 
-      def configure(str)
+      def configure(str, use_v1 = false)
         if str.is_a?(Fluent::Config::Element)
           @config = str
         else
-          @config = Config.parse(str, "(test)")
+          @config = Config.parse(str, "(test)", "(test_dir)", use_v1)
         end
         @instance.configure(@config)
         self

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.50'
+  VERSION = '0.10.51'
 
 end

data/spec/config/config_parser_spec.rb

@@ -8,6 +8,12 @@ require "fluent/config/v1_parser"
 describe Fluent::Config::V1Parser do
   include_context 'config_helper'
 
+  def read_config(path)
+    path = File.expand_path(path)
+    data = File.read(path)
+    Fluent::Config::V1Parser.parse(data, File.basename(path), File.dirname(path))
+  end
+
   def parse_text(text)
     basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
     Fluent::Config::V1Parser.parse(text, '(test)', basepath, nil)
@@ -238,7 +244,7 @@ describe Fluent::Config::V1Parser do
 
     it 'parses @include / include correctly' do
       prepare_config
-      c = Fluent::Config.read("#{TMP_DIR}/config_test_1.conf", true)
+      c = read_config("#{TMP_DIR}/config_test_1.conf")
       expect(c['k1']).to eq('root_config')
       expect(c['k2']).to eq('relative_path_include')
       expect(c['k3']).to eq('relative_include_in_included_file')

data/test/plugin/test_in_http.rb

@@ -10,13 +10,13 @@ class HttpInputTest < Test::Unit::TestCase
   PORT = unused_port
   CONFIG = %[
     port #{PORT}
-    bind 127.0.0.1
+    bind "127.0.0.1"
     body_size_limit 10m
     keepalive_timeout 5
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::HttpInput).configure(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::HttpInput).configure(conf, true)
   end
 
   def test_configure
@@ -157,7 +157,7 @@ class HttpInputTest < Test::Unit::TestCase
 
   def test_with_regexp
     d = create_driver(CONFIG + %[
-      format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
+      format /^(?<field_1>\\\\d+):(?<field_2>\\\\w+)$/
       types field_1:integer
     ])
 

data/test/plugin/test_in_stream.rb

@@ -105,30 +105,6 @@ module StreamInputTest
   end
 end
 
-class TcpInputTest < Test::Unit::TestCase
-  include StreamInputTest
-
-  PORT = unused_port
-  CONFIG = %[
-    port #{PORT}
-    bind 127.0.0.1
-  ]
-
-  def create_driver(conf=CONFIG)
-    super(Fluent::TcpInput, conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal PORT, d.instance.port
-    assert_equal '127.0.0.1', d.instance.bind
-  end
-
-  def connect
-    TCPSocket.new('127.0.0.1', PORT)
-  end
-end
-
 class UnixInputTest < Test::Unit::TestCase
   include StreamInputTest
 

data/test/plugin/test_in_syslog.rb

@@ -111,6 +111,26 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests)
   end
 
+  def test_msg_size_with_json_format
+    d = create_driver([CONFIG, 'format json'].join("\n"))
+    time = Time.parse('2013-09-18 12:00:00 +0900').to_i
+    tests = ['Hello!', 'Syslog!'].map { |msg|
+      event = {'time' => time, 'message' => msg}
+      {'msg' => '<6>' + event.to_json + "\n", 'expected' => msg}
+    }
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
   def create_test_case
     # actual syslog message has "\n"
     [
@@ -120,7 +140,8 @@ class SyslogInputTest < Test::Unit::TestCase
   end
 
   def compare_test_result(emits, tests)
-    emits.each_index {|i|
+    emits.each_index { |i|
+      assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
     }
   end

data/test/plugin/test_in_tail.rb

@@ -95,7 +95,7 @@ class TailInputTest < Test::Unit::TestCase
 
   def test_rotate_file
     emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
-    assert(emits.length > 0)
+    assert_equal(4, emits.length)
     assert_equal({"message" => "test3"}, emits[0][2])
     assert_equal({"message" => "test4"}, emits[1][2])
     assert_equal({"message" => "test5"}, emits[2][2])
@@ -104,7 +104,7 @@ class TailInputTest < Test::Unit::TestCase
 
   def test_rotate_file_with_read_from_head
     emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
-    assert(emits.length > 0)
+    assert_equal(6, emits.length)
     assert_equal({"message" => "test1"}, emits[0][2])
     assert_equal({"message" => "test2"}, emits[1][2])
     assert_equal({"message" => "test3"}, emits[2][2])
@@ -113,32 +113,71 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal({"message" => "test6"}, emits[5][2])
   end
 
-  def sub_test_rotate_file(config = nil)
-    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
-      f.puts "test1"
-      f.puts "test2"
+  def test_rotate_file_with_write_old
+    emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
+      File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+      rotated_file.puts "test7"
+      rotated_file.puts "test8"
+      rotated_file.flush
+
+      sleep 1
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "test5"
+        f.puts "test6"
+      }
     }
+    assert_equal(6, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test7"}, emits[2][2])
+    assert_equal({"message" => "test8"}, emits[3][2])
+    assert_equal({"message" => "test5"}, emits[4][2])
+    assert_equal({"message" => "test6"}, emits[5][2])
+  end
+
+  def test_rotate_file_with_write_old_and_no_new_file
+    emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
+      rotated_file.puts "test7"
+      rotated_file.puts "test8"
+      rotated_file.flush
+    }
+    assert_equal(4, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test7"}, emits[2][2])
+    assert_equal({"message" => "test8"}, emits[3][2])
+  end
+
+  def sub_test_rotate_file(config = nil)
+    file = File.open("#{TMP_DIR}/tail.txt", "w")
+    file.puts "test1"
+    file.puts "test2"
+    file.flush
+
     d = create_driver(config)
     d.run do
       sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
-        f.puts "test3"
-        f.puts "test4"
-      }
+      file.puts "test3"
+      file.puts "test4"
+      file.flush
       sleep 1
 
       FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
-      sleep 1
-
-      File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
-      sleep 1
+      if block_given?
+        yield file
+        sleep 1
+      else
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+        sleep 1
 
-      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
-        f.puts "test5"
-        f.puts "test6"
-      }
-      sleep 1
+        File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+          f.puts "test5"
+          f.puts "test6"
+        }
+        sleep 1
+      end
     end
 
     d.run do
@@ -146,6 +185,8 @@ class TailInputTest < Test::Unit::TestCase
     end
 
     d.emits
+  ensure
+    file.close
   end
 
   def test_lf

data/test/plugin/test_in_tcp.rb

@@ -0,0 +1,88 @@
+require 'fluent/test'
+require 'helper'
+
+class TcpInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  BASE_CONFIG = %[
+    port #{PORT}
+    tag tcp
+  ]
+  CONFIG = BASE_CONFIG + %[
+    bind 127.0.0.1
+    format none
+  ]
+  IPv6_CONFIG = BASE_CONFIG + %[
+    bind ::1
+    format none
+  ]
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::TcpInput).configure(conf)
+  end
+
+  def test_configure
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal PORT, d.instance.port
+      assert_equal k, d.instance.bind
+      assert_equal "\n", d.instance.delimiter
+    }
+  end
+
+  {
+    'none' => [
+      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
+      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
+    ],
+    'json' => [
+      {'msg' => {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n", 'expected' => 'tcptest1'},
+      {'msg' => {'k' => 'tcptest2', 'message' => 456}.to_json + "\n", 'expected' => 456},
+    ]
+  }.each { |format, test_cases|
+    define_method("test_msg_size_#{format}") do
+      d = create_driver(BASE_CONFIG + "format #{format}")
+      tests = test_cases
+
+      d.run do
+        tests.each {|test|
+          TCPSocket.open('127.0.0.1', PORT) do |s|
+            s.send(test['msg'], 0)
+          end
+        }
+        sleep 1
+      end
+
+      compare_test_result(d.emits, tests)
+    end
+
+    define_method("test_msg_size_with_same_connection_#{format}") do
+      d = create_driver(BASE_CONFIG + "format #{format}")
+      tests = test_cases
+
+      d.run do
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          tests.each {|test|
+            s.send(test['msg'], 0)
+          }
+        end
+        sleep 1
+      end
+
+      compare_test_result(d.emits, tests)
+    end
+  }
+
+  def compare_test_result(emits, tests)
+    assert_equal(2, emits.size)
+    emits.each_index {|i|
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+    }
+  end
+end

data/test/plugin/test_in_udp.rb

@@ -0,0 +1,104 @@
+require 'fluent/test'
+require 'helper'
+
+class UdpInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  BASE_CONFIG = %[
+    port #{PORT}
+    tag udp
+  ]
+  CONFIG = BASE_CONFIG + %!
+    bind 127.0.0.1
+    format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
+  !
+  IPv6_CONFIG = BASE_CONFIG + %!
+    bind ::1
+    format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
+  !
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::UdpInput).configure(conf)
+  end
+
+  def test_configure
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal PORT, d.instance.port
+      assert_equal k, d.instance.bind
+      assert_equal 4096, d.instance.body_size_limit
+    }
+  end
+
+  def test_time_format
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+
+      tests = [
+        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
+        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
+      ]
+
+      d.run do
+        u = Fluent::SocketUtil.create_udp_socket(k)
+        u.connect(k, PORT)
+        tests.each {|test|
+          u.send(test['msg'], 0)
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      emits.each_index {|i|
+        assert_equal(tests[i]['expected'], emits[i][1])
+      }
+    }
+  end
+
+  {
+    'none' => [
+      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
+      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
+    ],
+    'json' => [
+      {'msg' => {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n", 'expected' => 'tcptest1'},
+      {'msg' => {'k' => 'tcptest2', 'message' => 456}.to_json + "\n", 'expected' => 456},
+    ],
+    '/^\\[(?<time>[^\\]]*)\\] (?<message>.*)/' => [
+      {'msg' => '[Sep 10 00:00:00] localhost: ' + 'x' * 100 + "\n", 'expected' => 'localhost: ' + 'x' * 100},
+      {'msg' => '[Sep 10 00:00:00] localhost: ' + 'x' * 1024 + "\n", 'expected' => 'localhost: ' + 'x' * 1024},
+    ]
+  }.each { |format, test_cases|
+    define_method("test_msg_size_#{format[0] == '/' ? 'regexp' : format}") do
+      d = create_driver(BASE_CONFIG + "format #{format}")
+      tests = test_cases
+
+      d.run do
+        u = UDPSocket.new
+        u.connect('127.0.0.1', PORT)
+        tests.each { |test|
+          u.send(test['msg'], 0)
+        }
+        sleep 1
+      end
+
+      compare_test_result(d.emits, tests)
+    end
+  }
+
+  def compare_test_result(emits, tests)
+    assert_equal(2, emits.size)
+    emits.each_index {|i|
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+    }
+  end
+end

data/test/plugin/test_out_exec.rb

@@ -16,15 +16,15 @@ class ExecOutputTest < Test::Unit::TestCase
     localtime
   ]
   TSV_CONFIG = %[
-    keys time,tag,k1
-    tag_key tag
-    time_key time
+    keys "time,tag,k1"
+    tag_key "tag"
+    time_key "time"
     time_format %Y-%m-%d %H:%M:%S
   ]
 
   def create_driver(conf = TSV_CONFIG)
     config = CONFIG + conf
-    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::ExecOutput).configure(config)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::ExecOutput).configure(config, true)
   end
 
   def create_test_case