fluentd 0.10.44 → 0.10.45

data/lib/fluent/plugin/out_exec.rb

@@ -15,6 +15,9 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'fluent/plugin/exec_util'
+
 module Fluent
   class ExecOutput < TimeSlicedOutput
     Plugin.register_output('exec', self)
@@ -26,41 +29,52 @@ module Fluent
     end
 
     config_param :command, :string
-    config_param :keys, :string
+    config_param :keys, :default => [] do |val|
+      val.split(',')
+    end
     config_param :tag_key, :string, :default => nil
     config_param :time_key, :string, :default => nil
     config_param :time_format, :string, :default => nil
+    config_param :format, :default => :tsv do |val|
+      f = ExecUtil::SUPPORTED_FORMAT[val]
+      raise ConfigError, "Unsupported format '#{val}'" unless f
+      f
+    end
 
     def configure(conf)
       super
 
-      @keys = @keys.split(',')
+      @formatter = case @format
+                   when :tsv
+                     if @keys.empty?
+                       raise ConfigError, "keys option is required on exec output for tsv format"
+                     end
+                     ExecUtil::TSVFormatter.new(@keys)
+                   when :json
+                     ExecUtil::JSONFormatter.new
+                   when :msgpack
+                     ExecUtil::MessagePackFormatter.new
+                   end
 
       if @time_key
         if @time_format
           tf = TimeFormatter.new(@time_format, @localtime)
           @time_format_proc = tf.method(:format)
         else
-          @time_format_proc = Proc.new {|time| time.to_s }
+          @time_format_proc = Proc.new { |time| time.to_s }
         end
       end
     end
 
     def format(tag, time, record)
       out = ''
-      last = @keys.length-1
-      for i in 0..last
-        key = @keys[i]
-        if key == @time_key
-          out << @time_format_proc.call(time)
-        elsif key == @tag_key
-          out << tag
-        else
-          out << record[key].to_s
-        end
-        out << "\t" if i != last
+      if @time_key
+        record[@time_key] = @time_format_proc.call(time)
+      end
+      if @tag_key
+        record[@tag_key] = tag
       end
-      out << "\n"
+      @formatter.call(record, out)
       out
     end
 

data/lib/fluent/plugin/out_exec_filter.rb

@@ -15,28 +15,24 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'fluent/plugin/exec_util'
+
 module Fluent
   class ExecFilterOutput < BufferedOutput
     Plugin.register_output('exec_filter', self)
 
     def initialize
       super
-      require 'fluent/plugin/exec_util'
     end
 
-    SUPPORTED_FORMAT = {
-      'tsv' => :tsv,
-      'json' => :json,
-      'msgpack' => :msgpack,
-    }
-
     config_param :command, :string
 
     config_param :remove_prefix, :string, :default => nil
     config_param :add_prefix, :string, :default => nil
 
     config_param :in_format, :default => :tsv do |val|
-      f = SUPPORTED_FORMAT[val]
+      f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported in_format '#{val}'" unless f
       f
     end
@@ -48,7 +44,7 @@ module Fluent
     config_param :in_time_format, :default => nil
 
     config_param :out_format, :default => :tsv do |val|
-      f = SUPPORTED_FORMAT[val]
+      f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported out_format '#{val}'" unless f
       f
     end
@@ -140,11 +136,11 @@ module Fluent
         if @in_keys.empty?
           raise ConfigError, "in_keys option is required on exec_filter output for tsv in_format"
         end
-        @formatter = TSVFormatter.new(@in_keys)
+        @formatter = ExecUtil::TSVFormatter.new(@in_keys)
       when :json
-        @formatter = JSONFormatter.new
+        @formatter = ExecUtil::JSONFormatter.new
       when :msgpack
-        @formatter = MessagePackFormatter.new
+        @formatter = ExecUtil::MessagePackFormatter.new
       end
 
       case @out_format
@@ -333,38 +329,6 @@ module Fluent
       end
     end
 
-    class Formatter
-    end
-
-    class TSVFormatter < Formatter
-      def initialize(in_keys)
-        @in_keys = in_keys
-        super()
-      end
-
-      def call(record, out)
-        last = @in_keys.length-1
-        for i in 0..last
-          key = @in_keys[i]
-          out << record[key].to_s
-          out << "\t" if i != last
-        end
-        out << "\n"
-      end
-    end
-
-    class JSONFormatter < Formatter
-      def call(record, out)
-        out << Yajl.dump(record) << "\n"
-      end
-    end
-
-    class MessagePackFormatter < Formatter
-      def call(record, out)
-        record.to_msgpack(out)
-      end
-    end
-
     def on_message(record)
       if val = record.delete(@out_time_key)
         time = @time_parse_proc.call(val)

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.44'
+  VERSION = '0.10.45'
 
 end

data/test/parser.rb

@@ -282,4 +282,85 @@ module ParserTest
     end
   end
 
+  class MultilineParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def create_parser(conf)
+      parser = TextParser::TEMPLATE_FACTORIES['multiline'].call
+      parser.configure(conf)
+      parser
+    end
+
+    def test_configure_with_invalid_params
+      [{'format100' => '/(?<msg>.*)/'}, {'format1' => '/(?<msg>.*)/', 'format3' => '/(?<msg>.*)/'}, 'format1' => '/(?<msg>.*)'].each { |config|
+        assert_raise(ConfigError) {
+          create_parser(config)
+        }
+      }
+    end
+
+    def test_call
+      parser = create_parser('format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/')
+      time, record = parser.call(<<EOS.chomp)
+2013-3-03 14:27:33 [main] ERROR Main - Exception
+javax.management.RuntimeErrorException: null
+\tat Main.main(Main.java:16) ~[bin/:na]
+EOS
+
+      assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
+      assert_equal({
+        "thread"  => "main",
+        "level"   => "ERROR",
+        "message" => " Main - Exception\njavax.management.RuntimeErrorException: null\n\tat Main.main(Main.java:16) ~[bin/:na]"
+      }, record)
+    end
+
+    def test_call_with_firstline
+      parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/')
+      time, record = parser.call(<<EOS.chomp)
+----
+time=2013-3-03 14:27:33 
+message=test1
+EOS
+
+      assert(parser.firstline?('----'))
+      assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
+      assert_equal({"message" => "test1"}, record)
+    end
+
+    def test_call_with_multiple_formats
+      parser = create_parser('format_firstline' => '/^Started/',
+        'format1' => '/Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/',
+        'format2' => '/Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/',
+        'format3' => '/(  Parameters: (?<parameters>[^ ]+)\n)?/',
+        'format4' => '/  Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/',
+        'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/'
+        )
+      time, record = parser.call(<<EOS.chomp)
+Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
+Processing by UsersController#show as HTML
+  Parameters: {"user_id"=>"123"}
+  Rendered users/show.html.erb within layouts/application (0.3ms)
+Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
+EOS
+
+      assert(parser.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
+      assert_equal(str2time('2013-06-14 12:00:11 +0900').to_i, time)
+      assert_equal({
+        "method" => "GET",
+        "path" => "/users/123/",
+        "host" => "127.0.0.1",
+        "controller" => "UsersController",
+        "controller_method" => "show",
+        "format" => "HTML",
+        "parameters" => "{\"user_id\"=>\"123\"}",
+        "template" => "users/show.html.erb",
+        "layout" => "layouts/application",
+        "code" => "200",
+        "runtime" => "4",
+        "view_runtime" => "3.2",
+        "ar_runtime" => "0.0"
+      }, record)
+    end
+  end
 end

data/test/plugin/in_tail.rb

@@ -1,7 +1,10 @@
 require 'fluent/test'
 require 'net/http'
+require 'flexmock'
 
 class TailInputTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   def setup
     Fluent::Test.setup
     FileUtils.rm_rf(TMP_DIR)
@@ -10,16 +13,19 @@ class TailInputTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
-  CONFIG = %[
+  COMMON_CONFIG = %[
     path #{TMP_DIR}/tail.txt
     tag t1
     rotate_wait 2s
     pos_file #{TMP_DIR}/tail.pos
+  ]
+  SINGLE_LINE_CONFIG = %[
     format /(?<message>.*)/
   ]
 
-  def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::TailInput).configure(conf)
+  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
+    config = use_common_conf ? COMMON_CONFIG + conf : conf
+    Fluent::Test::InputTestDriver.new(Fluent::NewTailInput).configure(config)
   end
 
   def test_configure
@@ -104,4 +110,231 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal({"message"=>"	tab"}, emits[4][2])
     assert_equal({"message"=>"tab	"}, emits[5][2])
   end
+
+  # multiline mode test
+
+  def test_multiline
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+    assert_equal({"message1" => "test5"}, emits[1][2])
+    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert_equal({"message1" => "test8"}, emits[3][2])
+  end
+
+  def test_multiline_with_multiple_formats
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)\\n?/
+      format2 /(f (?<message2>[^\\n]+)\\n?)?/
+      format3 /(f (?<message3>.*))?/
+      format_firstline /^[s]/
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+    assert_equal({"message1" => "test5"}, emits[1][2])
+    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert_equal({"message1" => "test8"}, emits[3][2])
+  end
+
+  def test_multilinelog_with_multiple_paths
+    files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
+    files.each { |file| File.open(file, "w") { |f| } }
+
+    d = create_driver(%[
+      path #{files[0]},#{files[1]}
+      tag t1
+      format multiline
+      format1 /^[s|f] (?<message>.*)/
+      format_firstline /^[s]/
+    ], false)
+    d.run do
+      files.each do |file|
+        File.open(file, 'a') { |f|
+          f.puts "f #{file} line should be ignored"
+          f.puts "s test1"
+          f.puts "f test2"
+          f.puts "f test3"
+          f.puts "s test4"
+        }
+      end
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
+    assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
+    # "test4" events are here because these events are flushed at shutdown phase
+    assert_equal({"message" => "test4"}, emits[2][2])
+    assert_equal({"message" => "test4"}, emits[3][2])
+  end
+
+  # * path test
+  # TODO: Clean up tests
+  EX_RORATE_WAIT = 0
+
+  EX_CONFIG = %[
+    tag tail
+    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+    format none
+    pos_file #{TMP_DIR}/tail.pos
+    read_from_head true
+    refresh_interval 30
+    rotate_wait #{EX_RORATE_WAIT}s
+  ]
+  EX_PATHS = [
+    'test/plugin/data/2010/01/20100102-030405.log',
+    'test/plugin/data/log/foo/bar.log',
+    'test/plugin/data/log/test.log'
+  ]
+
+  def test_expand_paths
+    plugin = create_driver(EX_CONFIG, false).instance
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
+      assert_equal EX_PATHS, plugin.expand_paths.sort
+    end
+  end
+
+  def test_refresh_watchers
+    plugin = create_driver(EX_CONFIG, false).instance
+    sio = StringIO.new
+    plugin.instance_eval do
+      @pf = Fluent::NewTailInput::PositionFile.parse(sio)
+      @loop = Coolio::Loop.new
+   end
+
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
+
+      flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
+        EX_PATHS.each do |path|
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, any, any).once.and_return do
+            flexmock('TailWatcher') { |watcher|
+              watcher.should_receive(:attach).once
+              watcher.should_receive(:unwatched=).zero_or_more_times
+              watcher.should_receive(:line_buffer).zero_or_more_times
+            }
+          end
+        end
+        plugin.refresh_watchers
+      end
+
+      plugin.instance_eval do
+        @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
+      end
+
+      flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, any, any).once.and_return do
+          flexmock('TailWatcher') do |watcher|
+            watcher.should_receive(:attach).once
+            watcher.should_receive(:unwatched=).zero_or_more_times
+            watcher.should_receive(:line_buffer).zero_or_more_times
+          end
+        end
+        plugin.refresh_watchers
+      end
+
+      flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
+        watcherclass.should_receive(:new).never
+        plugin.refresh_watchers
+      end
+    end
+  end
+
+  DummyWatcher = Struct.new("DummyWatcher", :tag)
+
+  def test_receive_lines
+    plugin = create_driver(EX_CONFIG, false).instance
+    flexstub(Fluent::Engine) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('tail', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(Fluent::Engine) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag *.post
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(Fluent::Engine) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*.post
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(Fluent::Engine) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*.post*ignore
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(Fluent::Engine) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+  end
 end