fluentd 0.10.50 → 0.10.51

data/test/plugin/test_out_file.rb

@@ -22,6 +22,14 @@ class FileOutputTest < Test::Unit::TestCase
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
   end
 
+  def with_timezone(timezone = 'UTC', &block)
+    old = ENV['TZ']
+    ENV['TZ'] = timezone
+    output = yield
+    ENV['TZ'] = old
+    output
+  end
+
   def test_configure
     d = create_driver %[
       path test_path
@@ -31,6 +39,17 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal :gz, d.instance.compress
   end
 
+  def test_default_localtime
+    d = create_driver(%[path #{TMP_DIR}/out_file_test])
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    with_timezone('Asia/Taipei') do
+      d.emit({"a"=>1}, time)
+      d.expect_format %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n]
+      d.run
+    end
+  end
+
   def test_format
     d = create_driver
 

data/test/test_config.rb

@@ -10,6 +10,13 @@ class ConfigTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
 
+  def read_config(path)
+    path = File.expand_path(path)
+    File.open(path) { |io|
+      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+    }
+  end
+
   def prepare_config
     write_config "#{TMP_DIR}/config_test_1.conf", %[
       k1 root_config
@@ -63,7 +70,7 @@ class ConfigTest < Test::Unit::TestCase
 
   def test_include
     prepare_config
-    c = Config.read("#{TMP_DIR}/config_test_1.conf")
+    c = read_config("#{TMP_DIR}/config_test_1.conf")
     assert_equal 'root_config', c['k1']
     assert_equal 'relative_path_include', c['k2']
     assert_equal 'relative_include_in_included_file', c['k3']
@@ -112,7 +119,7 @@ class ConfigTest < Test::Unit::TestCase
        </rule>
      </match>
     ]
-    root_conf  = Config.read("#{TMP_DIR}/config_test_not_fetched.conf")
+    root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.conf")
     match_conf = root_conf.elements.first
     rule_conf  = match_conf.elements.first
 

data/test/test_formatter.rb

@@ -37,29 +37,33 @@ module FormatterTest
       @time = Engine.now
     end
 
+    def configure(conf)
+      @formatter.configure({'utc' => true}.merge(conf))
+    end
+
     def test_format
-      @formatter.configure({})
+      configure({})
       formatted = @formatter.format(tag, @time, record)
 
       assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
     end
 
     def test_format_without_time
-      @formatter.configure('output_time' => 'false')
+      configure('output_time' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
       assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
     end
 
     def test_format_without_tag
-      @formatter.configure('output_tag' => 'false')
+      configure('output_tag' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
       assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
     end
 
     def test_format_without_time_and_tag
-      @formatter.configure('output_tag' => 'false', 'output_time' => 'false')
+      configure('output_tag' => 'false', 'output_time' => 'false')
       formatted = @formatter.format('tag', @time, record)
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)

data/test/test_parser.rb

@@ -67,14 +67,45 @@ module ParserTest
 
     def test_call_with_configure
       # Specify conf by configure method instaed of intializer
-      parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!))
+      regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
+      parser = TextParser::RegexpParser.new(regexp)
       parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
       internal_test_case(parser)
+      assert_equal(regexp, parser.patterns['format'])
+      assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.patterns['time_format'])
     end
 
     def test_call_with_typed_and_name_separator
       internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
     end
+
+    def test_call_without_time
+      time_at_start = Time.now.to_i
+      text = "tagomori_satoshi tagomoris 34\n"
+
+      parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
+      parser.configure('types'=>'name:string,user:string,age:bool')
+
+      [parser.call(text), parser.call(text) { |time, record| return time, record}].each { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
+      }
+
+      parser2 = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
+      parser2.configure('types'=>'name:string,user:string,age:bool')
+      parser2.time_default_current = false
+
+      [parser2.call(text), parser2.call(text) { |time, record| return time, record}].each { |time, record|
+        assert_equal "tagomori_satoshi", record["name"]
+        assert_equal "tagomoris", record["user"]
+        assert_equal 34, record["age"]
+
+        assert_nil time, "parser returns nil if configured so"
+      }
+
+    end
   end
 
   class ApacheParserTest < ::Test::Unit::TestCase
@@ -120,6 +151,8 @@ module ParserTest
           'agent'   => 'Opera/12.0'
         }, record)
       }
+      assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
+      assert_equal(TextParser::ApacheParser::TIME_FORMAT, @parser.patterns['time_format'])
     end
   end
 
@@ -131,6 +164,7 @@ module ParserTest
     end
 
     def test_call
+      @parser.configure({})
       @parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal({
@@ -140,6 +174,24 @@ module ParserTest
           'message' => '[error] Syslog test'
         }, record)
       }
+      assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
+      assert_equal(TextParser::SyslogParser::TIME_FORMAT, @parser.patterns['time_format'])
+    end
+
+    def test_call_with_priority
+      @parser.configure('with_priority' => true)
+      @parser.call('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
+        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+        assert_equal({
+          'pri'     => 6,
+          'host'    => '192.168.0.1',
+          'ident'   => 'fluentd',
+          'pid'     => '11111',
+          'message' => '[error] Syslog test'
+        }, record)
+      }
+      assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.patterns['format'])
+      assert_equal(TextParser::SyslogParser::TIME_FORMAT, @parser.patterns['time_format'])
     end
   end
 
@@ -160,6 +212,31 @@ module ParserTest
         }, record)
       }
     end
+
+    def test_call_without_time
+      time_at_start = Time.now.to_i
+
+      @parser.call('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal({
+          'host'   => '192.168.0.1',
+          'size'   => 777,
+          'method' => 'PUT',
+        }, record)
+      }
+
+      parser = TextParser::JSONParser.new
+      parser.estimate_current_event = false
+      parser.configure({})
+      parser.call('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
+        assert_equal({
+          'host'   => '192.168.0.1',
+          'size'   => 777,
+          'method' => 'PUT',
+        }, record)
+        assert_nil time, "parser return nil w/o time and if specified so"
+      }
+    end
   end
 
   class NginxParserTest < ::Test::Unit::TestCase
@@ -222,6 +299,31 @@ module ParserTest
         }, record)
       }
     end
+
+    def test_call_with_time
+      time_at_start = Time.now.to_i
+
+      parser = TextParser::TSVParser.new
+      parser.configure('keys' => 'a,b')
+      parser.call("192.168.0.1\t111") { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal({
+          'a' => '192.168.0.1',
+          'b' => '111',
+        }, record)
+      }
+
+      parser = TextParser::TSVParser.new
+      parser.estimate_current_event = false
+      parser.configure('keys' => 'a,b', 'time_key' => 'time')
+      parser.call("192.168.0.1\t111") { |time, record|
+        assert_equal({
+          'a' => '192.168.0.1',
+          'b' => '111',
+        }, record)
+        assert_nil time, "parser returns nil w/o time and if configured so"
+      }
+    end
   end
 
   class CSVParserTest < ::Test::Unit::TestCase
@@ -238,6 +340,31 @@ module ParserTest
         }, record)
       }
     end
+
+    def test_call_without_time
+      time_at_start = Time.now.to_i
+
+      parser = TextParser::CSVParser.new
+      parser.configure('keys' => 'c,d')
+      parser.call("192.168.0.1,111") { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal({
+          'c' => '192.168.0.1',
+          'd' => '111',
+        }, record)
+      }
+
+      parser = TextParser::CSVParser.new
+      parser.estimate_current_event = false
+      parser.configure('keys' => 'c,d', 'time_key' => 'time')
+      parser.call("192.168.0.1,111") { |time, record|
+        assert_equal({
+          'c' => '192.168.0.1',
+          'd' => '111',
+        }, record)
+        assert_nil time, "parser returns nil w/o time and if configured so"
+      }
+    end
   end
 
   class LabeledTSVParserTest < ::Test::Unit::TestCase
@@ -299,13 +426,39 @@ module ParserTest
         }, record)
       }
     end
-  end
+
+    def test_call_without_time
+      time_at_start = Time.now.to_i
+
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure({})
+      parser.call("host:192.168.0.1\treq_id:111") { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal({
+          'host'   => '192.168.0.1',
+          'req_id' => '111',
+        }, record)
+      }
+
+      parser = TextParser::LabeledTSVParser.new
+      parser.estimate_current_event = false
+      parser.configure({})
+      parser.call("host:192.168.0.1\treq_id:111") { |time, record|
+        assert_equal({
+          'host'   => '192.168.0.1',
+          'req_id' => '111',
+        }, record)
+        assert_nil time, "parser returns nil w/o time and if configured so"
+      }
+    end
+ end
 
   class NoneParserTest < ::Test::Unit::TestCase
     include ParserTest
 
     def test_config_params
       parser = TextParser::NoneParser.new
+      parser.configure({})
       assert_equal "message", parser.message_key
 
       parser.configure('message_key' => 'foobar')
@@ -314,6 +467,7 @@ module ParserTest
 
     def test_call
       parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser.configure({})
       parser.call('log message!') { |time, record|
         assert_equal({'message' => 'log message!'}, record)
       }
@@ -326,6 +480,25 @@ module ParserTest
         assert_equal({'foobar' => 'log message!'}, record)
       }
     end
+
+    def test_call_without_default_time
+      time_at_start = Time.now.to_i
+
+      parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser.configure({})
+      parser.call('log message!') { |time, record|
+        assert time && time >= time_at_start, "parser puts current time without time input"
+        assert_equal({'message' => 'log message!'}, record)
+      }
+
+      parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser.estimate_current_event = false
+      parser.configure({})
+      parser.call('log message!') { |time, record|
+        assert_equal({'message' => 'log message!'}, record)
+        assert_nil time, "parser returns nil w/o time if configured so"
+      }
+    end
   end
 
   class MultilineParserTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.50
+  version: 0.10.51
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-17 00:00:00.000000000 Z
+date: 2014-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -255,6 +255,7 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - AUTHORS
+- CONTRIBUTING.md
 - COPYING
 - ChangeLog
 - Gemfile
@@ -311,6 +312,8 @@ files:
 - lib/fluent/plugin/in_stream.rb
 - lib/fluent/plugin/in_syslog.rb
 - lib/fluent/plugin/in_tail.rb
+- lib/fluent/plugin/in_tcp.rb
+- lib/fluent/plugin/in_udp.rb
 - lib/fluent/plugin/out_copy.rb
 - lib/fluent/plugin/out_exec.rb
 - lib/fluent/plugin/out_exec_filter.rb
@@ -355,6 +358,8 @@ files:
 - test/plugin/test_in_stream.rb
 - test/plugin/test_in_syslog.rb
 - test/plugin/test_in_tail.rb
+- test/plugin/test_in_tcp.rb
+- test/plugin/test_in_udp.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
 - test/plugin/test_out_exec_filter.rb
@@ -421,6 +426,8 @@ test_files:
 - test/plugin/test_in_stream.rb
 - test/plugin/test_in_syslog.rb
 - test/plugin/test_in_tail.rb
+- test/plugin/test_in_tcp.rb
+- test/plugin/test_in_udp.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
 - test/plugin/test_out_exec_filter.rb