fluent-plugin-tail-multiline 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 30ae0c4b7c3d24c02a04d5fef674ca5e24f133ae
+  data.tar.gz: 10ac8354faf5e40d599d9dd9f8753c4ce46ecc4d
+SHA512:
+  metadata.gz: 352fdfcb3089d9553db4930a7514cb0a843b47c73e8f0e3c7da5617b7faf090357340e807d2cc65fd4ef12acb6ad1b4c218f8627d33637ae10d124c1afcc9ec7
+  data.tar.gz: 663c365b32692660f80eb0c4b4c22985ed3733d73bf4020689d2b8c864fd37c9da1908265020e2de7c219b45f26aa0ace0152a93ac67a08aee8410675f1a98a6

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-tail-multiline.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2013 - Tomohisa Ota
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,83 @@
+# Fluent::Plugin::Tail-Multiline
+
+Tail-Multiline plugin extends built-in tail plugin with following features
++ Support log with multiple line output such as stacktrace
++ RegEx parameter to detect first line
++ Save raw log data
+
+**built-in templates are not supported. It does not support multiple line log anyway**
+
+## Installation
+
+Use ruby gem as :
+
+    gem 'fluent-plugin-tail-multiline'
+
+Or, if you're using td-client, you can call td-client's gem
+
+    $ /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-tail-multiline
+
+## Base Configuration
+Tail-Multiline extends [tail plugin](http://docs.fluentd.org/categories/in_tail).  
+
+## Configuration
+### Additional Parameters
+ name                 | type                            | description
+----------------------|---------------------------------|---------------------------
+format_firstline      | string(default = format)        | RegEx to detect first line of multiple line log, no name capture required
+rawdata_key           | string(default = null)          | Store raw data with given key
+
+## Examples
+### Java log with exception
+#### Input
+```
+2013-3-03 14:27:33 [main] INFO  Main - Start
+2013-3-03 14:27:33 [main] ERROR Main - Exception
+javax.management.RuntimeErrorException: null
+    at Main.main(Main.java:16) ~[bin/:na]
+2013-3-03 14:27:33 [main] INFO  Main - End
+```
+#### Parameters
+```
+tag test
+format /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
+```
+#### Output
+```
+2013-03-03 14:27:33 +0900 test: {"thread":"main","level":"INFO","message":"  Main - Start"}
+2013-03-03 14:27:33 +0900 test: {"thread":"main","level":"ERROR","message":" Main - Exception\njavax.management.RuntimeErrorException: null\n\tat Main.main(Main.java:16) ~[bin/:na]"}
+2013-03-03 14:27:33 +0900 test: {"thread":"main","level":"INFO","message":"  Main - End\n"}
+```
+
+### Case where first line does not have any name capture
+#### Input
+```
+----
+time=2013-3-03 14:27:33 
+message=test1
+----
+time=2013-3-03 14:27:34
+message=test2
+```
+
+#### Parameters
+```
+tag test
+format /time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/
+format_firstline /----/
+```
+
+#### Output
+```
+2013-03-03 14:27:33 +0900 test: {"message":"test1"}
+2013-03-03 14:27:34 +0900 test: {"message":"test2"}
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-tail-multiline.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-tail-multiline"
+  gem.version       = "0.1.0"
+  gem.authors       = ["Tomohisa Ota"]
+  gem.email         = ["tomohisa.ota+github@gmail.com"]
+  gem.description   = ""
+  gem.summary       = gem.description
+  gem.homepage      = "http://github.com/tomohisaota/fluent-plugin-tail-multiline"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.files.reject! { |fn| fn.include? "doc/" }
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_tail_multiline.rb

@@ -0,0 +1,192 @@
+module Fluent
+  require 'fluent/plugin/in_tail'
+  class TailMultilineInput < TailInput
+    
+    class MultilineTextParser < TextParser
+      def configure(conf, required=true)
+        format = conf['format']
+        if format == nil
+          raise ConfigError, "'format' parameter is required"
+        elsif format[0] != ?/ || format[format.length-1] != ?/ 
+          raise ConfigError, "'format' should be RegEx. Template is not supported in multiline mode"
+        end
+        
+        begin
+          @regex = Regexp.new(format[1..-2],Regexp::MULTILINE)
+          if @regex.named_captures.empty?
+            raise "No named captures"
+          end
+        rescue
+          raise ConfigError, "Invalid regexp in format '#{format[1..-2]}': #{$!}"
+        end
+
+        @parser = RegexpParser.new(@regex)
+
+        if @parser.respond_to?(:configure)
+          @parser.configure(conf)
+        end
+
+        format_firstline = conf['format_firstline']
+        if format_firstline
+          # Use custom matcher for 1st line
+          if format_firstline[0] == '/' && format_firstline[format_firstline.length-1] == '/'
+            @regex = Regexp.new(format_firstline[1..-2])
+          else
+            raise ConfigError, "Invalid regexp in format_firstline '#{format_firstline[1..-2]}': #{$!}"
+          end
+        end
+
+        return true
+      end
+      
+      def match_firstline(text)
+        @regex.match(text)
+      end
+    end
+    
+    Plugin.register_input('tail_multiline', self)
+    
+    config_param :format, :string
+    config_param :format_firstline, :string, :default => nil
+    config_param :rawdata_key, :string, :default => nil
+    config_param :auto_flush_sec, :integer, :default => 1
+    
+    def initialize
+      super
+      @locker = Monitor.new
+      @logbuf = nil
+      @logbuf_flusher = CallLater::new
+    end
+    
+    def configure_parser(conf)
+      @parser = MultilineTextParser.new
+      @parser.configure(conf)
+    end
+    
+    def receive_lines(lines)
+      @logbuf_flusher.cancel()
+      es = MultiEventStream.new
+      @locker.synchronize do
+        lines.each {|line|
+            if @parser.match_firstline(line)
+              time, record = parse_logbuf(@logbuf)
+              if time && record
+                es.add(time, record)
+              end
+              @logbuf = line
+            else
+              @logbuf += line if(@logbuf)
+            end
+        }
+      end
+      unless es.empty?
+        begin
+          Engine.emit_stream(@tag, es)
+        rescue
+          # ignore errors. Engine shows logs and backtraces.
+        end
+      end
+      @logbuf_flusher.call_later(@auto_flush_sec) do
+        flush_logbuf()
+      end
+    end
+    
+    def shutdown
+      super
+      flush_logbuf()
+      @logbuf_flusher.shutdown()
+    end   
+
+    def flush_logbuf
+      time, record = nil,nil
+      @locker.synchronize do
+        time, record = parse_logbuf(@logbuf)
+        @logbuf = nil
+      end
+      if time && record
+        Engine.emit(@tag, time, record)
+      end
+    end
+
+    def parse_logbuf(buf)
+      return nil,nil unless buf
+      buf.chomp!
+      begin
+        time, record = @parser.parse(buf)
+      rescue
+        $log.warn line.dump, :error=>$!.to_s
+        $log.debug_backtrace
+      end
+      return nil,nil unless time && record
+      record[@rawdata_key] = buf if @rawdata_key
+      return time, record
+    end
+    
+  end
+  
+  class CallLater 
+    def initialize
+      @locker = Monitor::new
+      @thread = Thread.new(&method(:run))
+      initExecBlock()
+    end
+    
+    def call_later(delay,&block)
+      @locker.synchronize do
+        @exec_time = Engine.now + delay
+        @exec_block = block
+      end
+      @thread.run
+    end
+    
+    def run
+      @running = true
+      while true
+        sleepSec = -1
+        @locker.synchronize do
+          now = Engine.now
+          if @exec_block && @exec_time <= now
+            @exec_block.call()
+            initExecBlock()
+          end          
+          return unless @running
+          unless(@exec_time == -1)
+            sleepSec = @exec_time - now 
+          end
+        end
+        if (sleepSec == -1)
+          sleep()
+        else
+          sleep(sleepSec)
+        end
+      end
+    rescue => e
+      puts e
+    end
+    
+    def cancel()
+      initExecBlock()
+    end
+
+    def shutdown()
+      @locker.synchronize do
+        @running = false
+      end
+      if(@thread)
+        @thread.run
+        @thread.join
+        @thread = nil
+      end
+    end    
+    
+    private 
+
+    def initExecBlock()
+      @locker.synchronize do
+        @exec_time = -1
+        @exec_block = nil
+      end
+    end
+    
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_tail_multiline'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_tail_multiline.rb

@@ -0,0 +1,203 @@
+require 'helper'
+
+require 'tempfile'
+
+class TailMultilineInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+  # CONFIG = %[
+  #   path #{TMP_DIR}/out_file_test
+  #   compress gz
+  #   utc
+  # ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::TailMultilineInput).configure(conf)
+  end
+  
+  def test_emit_no_additional_option
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format /^[s|f] (?<message>.*)/
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      assert_equal({"message"=>"test1"}, emits[0][2])
+      assert_equal({"message"=>"test2"}, emits[1][2])
+      assert_equal({"message"=>"test3"}, emits[2][2])
+      assert_equal({"message"=>"test4"}, emits[3][2])
+      assert_equal({"message"=>"test5"}, emits[4][2])
+      assert_equal({"message"=>"test6"}, emits[5][2])
+      assert_equal({"message"=>"test7"}, emits[6][2])
+      assert_equal({"message"=>"test8"}, emits[7][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+  
+  def test_emit_with_rawdata
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format /^[s|f] (?<message>.*)/
+        rawdata_key rawdata
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      assert_equal({"message"=>"test1","rawdata"=>"f test1"}, emits[0][2])
+      assert_equal({"message"=>"test2","rawdata"=>"s test2"}, emits[1][2])
+      assert_equal({"message"=>"test3","rawdata"=>"f test3"}, emits[2][2])
+      assert_equal({"message"=>"test4","rawdata"=>"f test4"}, emits[3][2])
+      assert_equal({"message"=>"test5","rawdata"=>"s test5"}, emits[4][2])
+      assert_equal({"message"=>"test6","rawdata"=>"s test6"}, emits[5][2])
+      assert_equal({"message"=>"test7","rawdata"=>"f test7"}, emits[6][2])
+      assert_equal({"message"=>"test8","rawdata"=>"s test8"}, emits[7][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+  def test_emit_with_format_firstline
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format /^[s|f] (?<message>.*)/
+        format_firstline /^[s]/
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      n = -1
+      assert_equal({"message"=>"test2\nf test3\nf test4"}, emits[0][2])
+      assert_equal({"message"=>"test5"}, emits[1][2])
+      assert_equal({"message"=>"test6\nf test7"}, emits[2][2])
+      assert_equal({"message"=>"test8"}, emits[3][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+  
+  def test_emit_with_format_firstline_with_rawdata
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format /^[s|f] (?<message>.*)/
+        format_firstline /^[s]/
+        rawdata_key rawdata
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      n = -1
+      assert_equal({"message"=>"test2\nf test3\nf test4","rawdata"=>"s test2\nf test3\nf test4"}, emits[0][2])
+      assert_equal({"message"=>"test5","rawdata"=>"s test5"}, emits[1][2])
+      assert_equal({"message"=>"test6\nf test7","rawdata"=>"s test6\nf test7"}, emits[2][2])
+      assert_equal({"message"=>"test8","rawdata"=>"s test8"}, emits[3][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+  
+  def test_multilinelog
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+        format_firstline /^[s]/
+        rawdata_key rawdata
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      n = -1
+      assert_equal({"message1"=>"test2","message2"=>"test3","message3"=>"test4","rawdata"=>"s test2\nf test3\nf test4"}, emits[0][2])
+      assert_equal({"message1"=>"test5","rawdata"=>"s test5"}, emits[1][2])
+      assert_equal({"message1"=>"test6","message2"=>"test7","rawdata"=>"s test6\nf test7"}, emits[2][2])
+      assert_equal({"message1"=>"test8","rawdata"=>"s test8"}, emits[3][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+  
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-tail-multiline
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tomohisa Ota
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- tomohisa.ota+github@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-tail-multiline.gemspec
+- lib/fluent/plugin/in_tail_multiline.rb
+- test/helper.rb
+- test/plugin/test_in_tail_multiline.rb
+homepage: http://github.com/tomohisaota/fluent-plugin-tail-multiline
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: ''
+test_files:
+- test/helper.rb
+- test/plugin/test_in_tail_multiline.rb