fluent-plugin-fields-parser 0.1.0 → 0.1.1

data/README.md

@@ -17,6 +17,8 @@ Use RubyGems:
 
         remove_tag_prefix   raw
         add_tag_prefix      parsed
+
+        strict_key_value     false
     </match>
 
 If following record is passed:
@@ -25,7 +27,7 @@ If following record is passed:
 {"message": "Audit log user=Johny action='add-user' result=success" }
 ```
 
-then you got new record:
+then you will get a new record:
 
 ```
 {
@@ -92,7 +94,7 @@ Configuration
 
 For input like:
 ```
-{ "message": "data black:54 white=55 red=10"}
+{ "message": "data black:54 white:55 red:10"}
 ```
 
 it returns:
@@ -114,6 +116,41 @@ You cat add and/or remove tag prefix using Configuration parameters
         add_tag_prefix      parsed
     </match>
 
-It it matched tag "raw.some.record", then it emits tag "parsed.some.record".
+If it matched tag "raw.some.record", then it emits tag "parsed.some.record".
+
+### Parameter strict_key_value
+
+```
+    <match pattern>
+        type                fields_parser
+        strict_key_value   true
+    </match>
+```
+
+If `strict_key_value` is set to `true`, the parser will use the [ruby logfmt
+parser](https://github.com/cyberdelia/logfmt-ruby) which will parse the log
+message based on the popular [logfmt](https://brandur.org/logfmt) key/value
+format.  Do note that this parser will create Fixnum and Float type values
+when it parses integer and float values.
 
+All information provided in the log message must be in a strict key=value
+format.  For example, if following record is passed:
 
+```
+{"message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true" }
+```
+
+then you will get a new record:
+
+```
+{
+    "message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true",
+    "msg": "Audit log",
+    "user": "Johnny",
+    "action": "add-user",
+    "result": "success",
+    "iVal": 23,
+    "fVal": 1.02,
+    "bVal": "true"
+}
+```

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/{fluent-plugin-tokenizer.gemspec → fluent-plugin-fields-parser.gemspec}

@@ -18,5 +18,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd"
+  gem.add_dependency "logfmt"
   gem.add_development_dependency "rake"
 end
+

data/lib/fluent/plugin/out_fields_parser.rb

@@ -1,3 +1,5 @@
+require "logfmt"
+
 module Fluent
   class OutputFieldsParser < Fluent::Output
     Fluent::Plugin.register_output('fields_parser', self)
@@ -8,6 +10,7 @@ module Fluent
     config_param :fields_key,         :string, :default => nil
     config_param :pattern,            :string,
                  :default => %{([a-zA-Z_]\\w*)=((['"]).*?(\\3)|[\\w.@$%/+-]*)}
+    config_param :strict_key_value,  :bool, :default => false
 
     def compiled_pattern
       @compiled_pattern ||= Regexp.new(pattern)
@@ -39,15 +42,22 @@ module Fluent
       source = record[parse_key].to_s
       target = fields_key ? (record[fields_key] ||= {}) : record
 
-      source.scan(compiled_pattern) do |match|
-        (key, value, begining_quote, ending_quote) = match
-        next if key.nil?
-        next if target.has_key?(key)
-        value = value.to_s
-        from_pos = begining_quote.to_s.length
-        to_pos = value.length - ending_quote.to_s.length - 1
-        target[key] = value[from_pos..to_pos]
+      if strict_key_value
+        # Use logfmt to parse it (key=value)
+        parsed = Logfmt.parse(source)
+        target.merge!(parsed)
+      else
+        source.scan(compiled_pattern) do |match|
+          (key, value, begining_quote, ending_quote) = match
+          next if key.nil?
+          next if target.has_key?(key)
+          value = value.to_s
+          from_pos = begining_quote.to_s.length
+          to_pos = value.length - ending_quote.to_s.length - 1
+          target[key] = value[from_pos..to_pos]
+        end
       end
+
       return record
     end
   end

data/test/out_fields_parser.rb

@@ -231,4 +231,40 @@ class FieldsParserOutputTest < Test::Unit::TestCase
       emits[1][2]
     )
   end
+
+def test_strict_key_value
+  d = create_driver("strict_key_value true")
+
+  orig_message = %{msg="Audit log" user=Johnny action="add-user" dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
+  d.run do
+    d.emit({'message' => orig_message})
+    d.emit({'message' => 'a'})
+  end
+
+  emits = d.emits
+  assert_equal 2, emits.size
+  assert_equal "orig.test.tag", emits[0][0]
+  assert_equal(
+    {
+      'message' => orig_message,
+      "msg"=>"Audit log",
+      'user' => "Johnny",
+      'action' => 'add-user',
+      'dontignore' => "don't-ignore-this",
+      'result' => 'success',
+      'iVal' => 23,
+      'fVal' => 1.02,
+      'bVal' => "true"
+    },
+    emits[0][2]
+  )
+  assert_equal(
+    {
+      'message' => 'a',
+    },
+    emits[1][2]
+  )
+end
+
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-fields-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-07 00:00:00.000000000 Z
+date: 2014-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: logfmt
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -54,7 +70,7 @@ files:
 - README.md
 - Rakefile
 - VERSION
-- fluent-plugin-tokenizer.gemspec
+- fluent-plugin-fields-parser.gemspec
 - lib/fluent/plugin/out_fields_parser.rb
 - test/out_fields_parser.rb
 homepage: https://github.com/tomas-zemres/fluent-plugin-fields-parser