fluent-plugin-parser 0.1.0 → 0.1.1

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.1.0"
+  gem.version       = "0.1.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}
@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_development_dependency "rake"
   gem.add_development_dependency "fluentd"
   gem.add_runtime_dependency "fluentd"
 end

data/lib/fluent/plugin/fixed_parser.rb

@@ -0,0 +1,136 @@
+#
+# This module is copied from fluentd/lib/fluent/parser.rb and
+# fixed not to overwrite 'time' (reserve nil) when time not found in parsed string.
+module FluentExt; end
+
+class FluentExt::TextParser
+  class RegexpParser
+    include Fluent::Configurable
+
+    config_param :time_format, :string, :default => nil
+
+    def initialize(regexp, conf={})
+      super()
+      @regexp = regexp
+      unless conf.empty?
+        configure(conf)
+      end
+    end
+
+    def call(text)
+      m = @regexp.match(text)
+      unless m
+        $log.debug "pattern not match: #{text}"
+        # TODO?
+        return nil, nil
+      end
+
+      time = nil
+      record = {}
+
+      m.names.each {|name|
+        if value = m[name]
+          case name
+          when "time"
+            if @time_format
+              time = Time.strptime(value, @time_format).to_i
+            else
+              time = Time.parse(value).to_i
+            end
+          else
+            record[name] = value
+          end
+        end
+      }
+
+      return time, record
+    end
+  end
+
+  class JSONParser
+    include Fluent::Configurable
+
+    config_param :time_key, :string, :default => 'time'
+    config_param :time_format, :string, :default => nil
+
+    def call(text)
+      record = Yajl.load(text)
+
+      time = nil
+      
+      if value = record.delete(@time_key)
+        if @time_format
+          time = Time.strptime(value, @time_format).to_i
+        else
+          time = value.to_i
+        end
+      end
+
+      return time, record
+    end
+  end
+
+  TEMPLATES = {
+    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
+    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
+    'json' => JSONParser.new,
+  }
+
+  def self.register_template(name, regexp_or_proc, time_format=nil)
+    if regexp_or_proc.is_a?(Regexp)
+      pr = regexp_or_proc
+    else
+      regexp = regexp_or_proc
+      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
+    end
+
+    TEMPLATES[name] = pr
+  end
+
+  def initialize
+    @parser = nil
+  end
+
+  def configure(conf, required=true)
+    format = conf['format']
+
+    if format == nil
+      if required
+        raise Fluent::ConfigError, "'format' parameter is required"
+      else
+        return nil
+      end
+    end
+
+    if format[0] == ?/ && format[format.length-1] == ?/
+      # regexp
+      begin
+        regexp = Regexp.new(format[1..-2])
+        if regexp.named_captures.empty?
+          raise "No named captures"
+        end
+      rescue
+        raise Fluent::ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+      end
+
+      @parser = RegexpParser.new(regexp)
+
+    else
+      # built-in template
+      @parser = TEMPLATES[format]
+      unless @parser
+        raise ConfigError, "Unknown format template '#{format}'"
+      end
+    end
+
+    if @parser.respond_to?(:configure)
+      @parser.configure(conf)
+    end
+
+    return true
+  end
+
+  def parse(text)
+    return @parser.call(text)
+  end
+end

data/lib/fluent/plugin/out_parser.rb

@@ -1,3 +1,5 @@
+require_relative './fixed_parser'
+
 class Fluent::ParserOutput < Fluent::Output
   Fluent::Plugin.register_output('parser', self)
 
@@ -29,26 +31,8 @@ class Fluent::ParserOutput < Fluent::Output
       @added_prefix_string = @add_prefix + '.'
     end
 
-    @parser = Fluent::TextParser.new
+    @parser = FluentExt::TextParser.new
     @parser.configure(conf)
-
-    m = if @parser.regexp.named_captures['time']
-          method(:parse_with_time)
-        else
-          method(:parse_without_time)
-        end
-    (class << self; self; end).module_eval do
-      define_method(:parse, m)
-    end
-  end
-
-  def parse_with_time(value)
-    @parser.parse(value)
-  end
-
-  def parse_without_time(value)
-    t,r = @parser.parse(value)
-    return [nil, r]
   end
 
   def emit(tag, es, chain)
@@ -72,7 +56,7 @@ class Fluent::ParserOutput < Fluent::Output
       es.each {|time,record|
         value = record[@key_name]
         t,values = if value
-                     parse(value)
+                     @parser.parse(value)
                    else
                      [nil, {}]
                    end
@@ -84,7 +68,7 @@ class Fluent::ParserOutput < Fluent::Output
       es.each {|time,record|
         value = record[@key_name]
         t,values = if value
-                     parse(value)
+                     @parser.parse(value)
                    else
                      [nil, {}]
                    end

data/test/plugin/test_out_parser.rb

@@ -51,16 +51,6 @@ class ParserOutputTest < Test::Unit::TestCase
         key_name foo
       ]
     }
-    assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
-        remove_prefix foo.baz
-        add_prefix foo.bar
-        format /(?<x>.)/
-        key_name foo
-        time_format %Y%m%d
-      ]
-    }
-
     d = create_driver %[
       tag foo.bar
       key_name foo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-02 00:00:00.000000000 Z
+date: 2012-06-12 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -53,10 +69,11 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
-- LICENSE
+- LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-parser.gemspec
+- lib/fluent/plugin/fixed_parser.rb
 - lib/fluent/plugin/out_deparser.rb
 - lib/fluent/plugin/out_parser.rb
 - test/helper.rb

data/LICENSE

@@ -1,22 +0,0 @@
-Copyright (c) 2012 TAGOMORI Satoshi
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.