fluent-plugin-tail-multiline 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 434a384c57025afcc2757f27b7dbf78d4b7bd14d
-  data.tar.gz: 886b7794644309392a25503215961f715cf85730
+  metadata.gz: dff2631e00656f109380d6b27c4f145d8149dc45
+  data.tar.gz: 1a14e17bae8ef5f85754e766321354e9d6620351
 SHA512:
-  metadata.gz: d03dfcad48c6b49095c7dd565e9424069d0b5d47576776e6a4d5abd0015a83a2f945bf9245ae72ff7fe74394799d2dd9caafde2c0b67ce127ea811de3b3bcf6d
-  data.tar.gz: 6ebbb94e246e3e203f458a2fd2708f9280956f0a31d4046a1d39478f62b5deb7256033b97f4b517c7703255f00d5362e1f93fb4800aea3254e1e1d9e368fc726
+  metadata.gz: f0205218e761d286422953179e61bcce121b89af151a5e277ae7a4ae9c84198bf9d127ed24d6942a830e0b9b2229a65c587298c2cac8ac37e59b5af18dbe38de
+  data.tar.gz: fa0c1f06d70ad1a08bf25644f8804085ecb07e33a186f2db9e575706f4a66af69696d72c4c549e1ca581d0afdc7ca888c910e31520f5599f5a9dc1d55e98056b

data/README.md

@@ -26,6 +26,7 @@ Tail-Multiline extends [tail plugin](http://docs.fluentd.org/categories/in_tail)
 ----------------------|---------------------------------|---------------------------
 format_firstline      | string(default = format)        | RegEx to detect first line of multiple line log, no name capture required
 rawdata_key           | string(default = null)          | Store raw data with given key
+format{1..20}         | string(default = null)          | The substitute for too long `format`
 
 ## Examples
 ### Java log with exception
@@ -73,6 +74,47 @@ format_firstline /----/
 2013-03-03 14:27:34 +0900 test: {"message":"test2"}
 ```
 
+### Case where too long regexp is required
+#### Input
+```
+Started GET "/users/" for 127.0.0.1 at 2013-06-14 12:00:04 +0900
+Processing by UsersController#index as HTML
+  Rendered users/index.html.erb within layouts/application (0.5ms)
+Completed 200 OK in 821ms (Views: 819.5ms | ActiveRecord: 0.0ms)
+
+
+Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
+Processing by UsersController#show as HTML
+  Parameters: {"user_id"=>"123"}
+  Rendered users/show.html.erb within layouts/application (0.3ms)
+Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
+```
+
+#### Parameters
+```
+tag test
+format_firstline /^Started/
+format /Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\nProcessing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n(  Parameters: (?<parameters>[^ ]+)\n)?  Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\nCompleted (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/
+```
+
+It's too long format. You can rewrite above parameters with `format{1..20}`.
+
+```
+tag test
+format_firstline /^Started/
+format1 /Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/
+format2 /Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/
+format3 /(  Parameters: (?<parameters>[^ ]+)\n)?/
+format4 /  Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/
+format5 /Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/
+```
+
+#### Output
+```
+2013-06-14 12:00:04 +0900 test: {"method":"GET","path":"/users/","host":"127.0.0.1","controller":"UsersController","controller_method":"index","format":"HTML","template":"users/index.html.erb","layout":"layouts/application","code":"200","runtime":"821","view_runtime":"819.5","ar_runtime":"0.0"}
+2013-06-14 12:00:11 +0900 test: {"method":"GET","path":"/users/123/","host":"127.0.0.1","controller":"UsersController","controller_method":"show","format":"HTML","parameters":"{\"user_id\"=>\"123\"}","template":"users/show.html.erb","layout":"layouts/application","code":"200","runtime":"4","view_runtime":"3.2","ar_runtime":"0.0"}
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-tail-multiline.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-tail-multiline"
-  gem.version       = "0.1.1"
+  gem.version       = "0.1.2"
   gem.authors       = ["Tomohisa Ota"]
   gem.email         = ["tomohisa.ota+github@gmail.com"]
   gem.description   = "Extend tail plugin to support log with multiple line"

data/lib/fluent/plugin/in_tail_multiline.rb

@@ -45,11 +45,16 @@ module Fluent
     end
     
     Plugin.register_input('tail_multiline', self)
+
+    FORMAT_MAX_NUM = 20
     
     config_param :format, :string
     config_param :format_firstline, :string, :default => nil
     config_param :rawdata_key, :string, :default => nil
     config_param :auto_flush_sec, :integer, :default => 1
+    (1..FORMAT_MAX_NUM).each do |i|
+      config_param "format#{i}".to_sym, :string, :default => nil
+    end
     
     def initialize
       super
@@ -57,6 +62,28 @@ module Fluent
       @logbuf = nil
       @logbuf_flusher = CallLater::new
     end
+
+    def configure(conf)
+      if conf['format'].nil?
+        invalids = conf.keys.select{|k| k =~ /^format(\d+)$/ and not (1..FORMAT_MAX_NUM).include?($1.to_i)}
+        if invalids.size > 0
+          raise ConfigError, "invalid number formats (valid format number:1-#{FORMAT_MAX_NUM}):" + invalids.join(",")
+        end
+        format_index_list = conf.keys.select{|s| s =~ /^format\d+$/}.map{|v| (/^format(\d+)$/.match(v))[1].to_i}
+        if (1..format_index_list.max).map{|i| conf["format#{i}"]}.include?(nil)
+          raise Fluent::ConfigError, "jump of format index found"
+        end
+        formats = (1..FORMAT_MAX_NUM).map {|i|
+          conf["format#{i}"]
+        }.delete_if {|format|
+          format.nil?
+        }.map {|format|
+          format[1..-2]
+        }.join
+        conf['format'] = '/' + formats + '/'
+      end
+      super
+    end
     
     def configure_parser(conf)
       @parser = MultilineTextParser.new
@@ -127,8 +154,8 @@ module Fluent
   class CallLater 
     def initialize
       @locker = Monitor::new
-      @thread = Thread.new(&method(:run))
       initExecBlock()
+      @thread = Thread.new(&method(:run))
     end
     
     def call_later(delay,&block)

data/test/plugin/test_in_tail_multiline.rb

@@ -200,4 +200,76 @@ class TailMultilineInputTest < Test::Unit::TestCase
     end
   end
   
-end
+  def test_multilinelog_with_serial_number_format
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      d = create_driver %[
+        path #{tmpFile.path}
+        tag test
+        format1 /^s (?<message1>[^\\n]+)\\n?/
+        format2 /(f (?<message2>[^\\n]+)\\n?)?/
+        format3 /(f (?<message3>.*))?/
+        format_firstline /^[s]/
+        rawdata_key rawdata
+      ]
+      d.run do
+        File.open(tmpFile.path, "w") {|f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      n = -1
+      assert_equal({"message1"=>"test2","message2"=>"test3","message3"=>"test4","rawdata"=>"s test2\nf test3\nf test4"}, emits[0][2])
+      assert_equal({"message1"=>"test5","rawdata"=>"s test5"}, emits[1][2])
+      assert_equal({"message1"=>"test6","message2"=>"test7","rawdata"=>"s test6\nf test7"}, emits[2][2])
+      assert_equal({"message1"=>"test8","rawdata"=>"s test8"}, emits[3][2])
+    ensure
+      tmpFile.close(true)
+    end
+  end
+
+  def test_multilinelog_with_invalid_number_format
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      e = assert_raise(Fluent::ConfigError) {
+        create_driver %[
+          path #{tmpFile.path}
+          tag test
+          format21 /(?<message1>.*)/
+          format22 /(?<message2>.*)/
+        ]
+      }
+      assert_equal(e.message, "invalid number formats (valid format number:1-20):format21,format22")
+    ensure
+      tmpFile.close(true)
+    end
+  end
+
+  def test_multilinelog_with_jump_of_number_format
+    tmpFile = Tempfile.new("in_tail_multiline-")
+    begin
+      e = assert_raise(Fluent::ConfigError) {
+        create_driver %[
+          path #{tmpFile.path}
+          tag test
+          format1 /(?<message1>.*)/
+          format3 /(?<message2>.*)/
+        ]
+      }
+      assert_equal(e.message, "jump of format index found")
+    ensure
+      tmpFile.close(true)
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-tail-multiline
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Tomohisa Ota
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-04 00:00:00.000000000 Z
+date: 2013-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd