fluentd 0.12.17 → 0.12.18

data/test/plugin/test_in_exec.rb

@@ -44,9 +44,18 @@ class ExecInputTest < Test::Unit::TestCase
     ]
   end
 
+  def regexp_config
+    %[
+      command ruby #{@script} "2011-01-02 13:14:15" 3
+      format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
+      tag regex_tag
+      run_interval 1s
+    ]
+  end
+
   def test_configure
     d = create_driver
-    assert_equal :tsv, d.instance.format
+    assert_equal 'tsv', d.instance.format
     assert_equal ["time","tag","k1"], d.instance.keys
     assert_equal "tag", d.instance.tag_key
     assert_equal "time", d.instance.time_key
@@ -55,16 +64,22 @@ class ExecInputTest < Test::Unit::TestCase
 
   def test_configure_with_json
     d = create_driver json_config
-    assert_equal :json, d.instance.format
+    assert_equal 'json', d.instance.format
     assert_equal [], d.instance.keys
   end
 
   def test_configure_with_msgpack
     d = create_driver msgpack_config
-    assert_equal :msgpack, d.instance.format
+    assert_equal 'msgpack', d.instance.format
     assert_equal [], d.instance.keys
   end
 
+  def test_configure_with_regexp
+    d = create_driver regexp_config
+    assert_equal '/(?<time>[^\]]*) (?<message>[^ ]*)/', d.instance.format
+    assert_equal 'regex_tag', d.instance.tag
+  end
+
   # TODO: Merge following tests into one case with parameters
 
   def test_emit
@@ -102,4 +117,17 @@ class ExecInputTest < Test::Unit::TestCase
     assert_equal true, emits.length > 0
     assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
   end
+
+  def test_emit_regexp
+    d = create_driver regexp_config
+
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert_equal true, emits.length > 0
+    assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
+    assert_equal @test_time, emits[0][1]
+  end
 end

data/test/plugin/test_out_forward.rb

@@ -78,6 +78,12 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.nodes.first.conf.dns_round_robin
   end
 
+  def test_configure_no_server
+    assert_raise(Fluent::ConfigError, 'forward output plugin requires at least one <server> is required') do
+      create_driver('')
+    end
+  end
+
   def test_phi_failure_detector
     d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first

data/test/scripts/exec_script.rb

@@ -13,6 +13,10 @@ def gen_msgpack(time)
   {'tagger' => 'tag1', 'datetime' => time, 'k1' => 'ok'}.to_msgpack
 end
 
+def gen_raw_string(time)
+  "#{time} hello"
+end
+
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -23,4 +27,6 @@ when 1
   puts gen_json(time)
 when 2
   print gen_msgpack(time)
+when 3
+  print gen_raw_string(time)
 end

data/test/test_formatter.rb

@@ -48,11 +48,33 @@ module FormatterTest
     end
   end
 
+  class BaseFormatterTestWithTestDriver < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def create_driver(conf={})
+      formatter = Fluent::Test::FormatterTestDriver.new(Formatter).configure(conf)
+    end
+
+    def test_call
+      d = create_driver
+      assert_raise NotImplementedError do
+        d.format('tag', Engine.now, {})
+      end
+    end
+
+    def test_call_with_string_literal_configure
+      d = create_driver('')
+      assert_raise NotImplementedError do
+        d.format('tag', Engine.now, {})
+      end
+    end
+  end
+
   class OutFileFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('out_file').call
+      @formatter = Fluent::Test::FormatterTestDriver.new('out_file')
       @time = Engine.now
     end
 
@@ -87,13 +109,24 @@ module FormatterTest
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
+
+    def test_format_without_time_and_tag_against_string_literal_configure
+      @formatter.configure(%[
+        utc         true
+        output_tag  false
+        output_time false
+      ])
+      formatted = @formatter.format('tag', @time, record)
+
+      assert_equal("#{Yajl.dump(record)}\n", formatted)
+    end
   end
 
   class JsonFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::JSONFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::JSONFormatter)
       @time = Engine.now
     end
 

data/test/test_parser.rb

@@ -39,6 +39,31 @@ module ParserTest
     end
   end
 
+  class BaseParserTestWithTestDriver < ::Test::Unit::TestCase
+    include ParserTest
+
+    def create_driver(conf={})
+      Fluent::Test::ParserTestDriver.new(Fluent::Parser).configure(conf)
+    end
+
+    def test_init
+      d = create_driver
+      assert_true d.instance.estimate_current_event
+    end
+
+    def test_configure_against_string_literal
+      d = create_driver('keep_time_key true')
+      assert_true d.instance.keep_time_key
+    end
+
+    def test_parse
+      d = create_driver
+      assert_raise NotImplementedError do
+        d.parse('')
+      end
+    end
+  end
+
   class TimeParserTest < ::Test::Unit::TestCase
     include ParserTest
 
@@ -94,19 +119,19 @@ module ParserTest
     def test_parse_with_configure
       # Specify conf by configure method instaed of intializer
       regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
-      parser = TextParser::RegexpParser.new(regexp)
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, regexp)
       parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
       internal_test_case(parser)
-      assert_equal(regexp, parser.patterns['format'])
-      assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.patterns['time_format'])
+      assert_equal(regexp, parser.instance.patterns['format'])
+      assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_typed_and_name_separator
-      internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
+      internal_test_case(Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
     end
 
     def test_parse_with_time_key
-      parser = TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/)
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, /(?<logtime>[^\]]*)/)
       parser.configure(
         'time_format'=>"%Y-%m-%d %H:%M:%S %z",
         'time_key'=>'logtime',
@@ -131,10 +156,9 @@ module ParserTest
         assert_equal 34, record["age"]
       }
 
-      parser2 = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
-      parser2.configure('types'=>'name:string,user:string,age:bool')
-      parser2.time_default_current = false
-
+      parser2 = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
+      parser2.configure('types'=>'name:string,user:string,age:integer')
+      parser2.instance.estimate_current_event = false
       [parser2.parse(text), parser2.parse(text) { |time, record| return time, record}].each { |time, record|
         assert_equal "tagomori_satoshi", record["name"]
         assert_equal "tagomoris", record["user"]
@@ -145,7 +169,7 @@ module ParserTest
     end
 
     def test_parse_with_keep_time_key
-      parser = TextParser::RegexpParser.new(
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
         Regexp.new(%q!(?<time>.*)!),
         'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
         'keep_time_key'=>'true',
@@ -157,7 +181,7 @@ module ParserTest
     end
 
     def test_parse_with_keep_time_key_with_typecast
-      parser = TextParser::RegexpParser.new(
+      parser = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser,
         Regexp.new(%q!(?<time>.*)!),
         'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
         'keep_time_key'=>'true',
@@ -281,7 +305,7 @@ module ParserTest
     include ParserTest
 
     def setup
-      @parser = TextParser::TEMPLATE_REGISTRY.lookup('syslog').call
+      @parser = Fluent::Test::ParserTestDriver.new('syslog')
       @expected = {
         'host'    => '192.168.0.1',
         'ident'   => 'fluentd',
@@ -296,8 +320,8 @@ module ParserTest
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_time_format
@@ -306,7 +330,7 @@ module ParserTest
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal('%b %d %M:%S:%H', @parser.patterns['time_format'])
+      assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_priority
@@ -315,8 +339,8 @@ module ParserTest
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_without_colon
@@ -325,8 +349,8 @@ module ParserTest
         assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
-      assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
-      assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
+      assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
+      assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
     end
 
     def test_parse_with_keep_time_key

data/test/test_process.rb

@@ -0,0 +1,47 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/event'
+require 'stringio'
+require 'msgpack'
+
+module FluentProcessTest
+  class DelayedForwarderTest < Test::Unit::TestCase
+    include Fluent
+
+    test 'run and emit' do
+      io = StringIO.new
+      fwd = Fluent::DetachProcessManager::DelayedForwarder.new(io, 0.001)
+
+      num_events_per_tag = 5000
+      num_tags = 20
+
+      now = Time.now.to_i
+      record = {'key' => 'value'}
+      (num_tags * num_events_per_tag).times do |i|
+        tag = "foo.bar#{i % num_tags}"
+        fwd.emit(tag, OneEventStream.new(now, record))
+      end
+      sleep 0.5
+
+      io.pos = 0
+
+      tags = {}
+      MessagePack::Unpacker.new(io).each do |tag_and_msgpacks|
+        tag, ms = *tag_and_msgpacks
+        tags[tag] ||= ''
+        tags[tag] << ms
+      end
+
+      assert_equal(num_tags, tags.size)
+      num_tags.times do |i|
+        tag = "foo.bar#{i % num_tags}"
+        ms = tags[tag]
+        count = 0
+        MessagePack::Unpacker.new(StringIO.new(ms)).each do |x|
+          count += 1
+        end
+        assert_equal(num_events_per_tag, count)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.17
+  version: 0.12.18
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-04 00:00:00.000000000 Z
+date: 2015-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -380,8 +380,10 @@ files:
 - lib/fluent/test.rb
 - lib/fluent/test/base.rb
 - lib/fluent/test/filter_test.rb
+- lib/fluent/test/formatter_test.rb
 - lib/fluent/test/input_test.rb
 - lib/fluent/test/output_test.rb
+- lib/fluent/test/parser_test.rb
 - lib/fluent/timezone.rb
 - lib/fluent/version.rb
 - test/config/assertions.rb
@@ -444,6 +446,7 @@ files:
 - test/test_output.rb
 - test/test_parser.rb
 - test/test_plugin_classes.rb
+- test/test_process.rb
 - test/test_root_agent.rb
 homepage: http://fluentd.org/
 licenses:
@@ -465,7 +468,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector
@@ -530,4 +533,5 @@ test_files:
 - test/test_output.rb
 - test/test_parser.rb
 - test/test_plugin_classes.rb
+- test/test_process.rb
 - test/test_root_agent.rb