fluent-plugin-fields-autotype 0.1.1

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/README.md

@@ -0,0 +1,119 @@
+fluent-plugin-fields-autotype
+===========================
+
+Fluent output filter plugin for parsing key/value fields in records
+based on <key>=<value> pattern. Automatically determines the data type of the value (string, integer or float).
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-fields-autotype
+
+## Configuration
+
+    <match pattern>
+        type                fields_autotype
+
+        remove_tag_prefix   raw
+        add_tag_prefix      parsed
+        pattern             (\S+)=(\S+)
+    </match>
+
+If following record is passed:
+
+```
+{"message": "Audit log user=Johny action=add-user result=success" }
+```
+
+then you will get a new record:
+
+```
+{
+    "message": "Audit log username=Johny action=add-user result=success",
+    "username": "Johny",
+    "action": "add-user",
+    "result": "success"
+}
+```
+
+### Parameter parse_key
+
+For configuration
+
+    <match pattern>
+        type        fields_autotype
+
+        parse_key   log_message
+    </match>
+
+it parses key "log_message" instead of default key `message`.
+
+### Parameter fields_key
+
+Configuration
+
+    <match pattern>
+        type        fields_autotype
+
+        parse_key   log_message
+        fields_key  fields
+    </match>
+
+For input like:
+
+```
+{
+    "log_message": "Audit log username=Johny action=add-user result=success",
+}
+```
+
+it adds parsed fields into defined key.
+
+```
+{
+    "log_message": "Audit log username=Johny action='add-user' result=success",
+    "fields": {"user": "Johny", "action": "add-user", "result": "success"}
+}
+```
+
+(It adds new keys into top-level record by default.)
+
+### Parameter pattern
+
+You can define custom pattern (regexp) for seaching keys/values. Data type like float and int are automatically determined.
+
+Configuration
+
+    <match pattern>
+        type        fields_autotype
+
+        pattern     (\w+):(\S+)
+    </match>
+
+For input like:
+```
+{ "message": "black:54 white:55 red:10.1"}
+```
+
+it returns:
+
+```
+{ "message": "black:54 white=55 red=10.1",
+  "black": 54, "white": 55, "red": 10.1
+}
+```
+
+### Tag prefix
+
+You cat add and/or remove tag prefix using Configuration parameters
+
+    <match pattern>
+        type                fields_autotype
+
+        remove_tag_prefix   raw
+        add_tag_prefix      parsed
+    </match>
+
+If it matched tag "raw.some.record", then it emits tag "parsed.some.record".
+

data/Rakefile

@@ -0,0 +1,14 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/fluent-plugin-fields-autotype.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-fields-autotype"
+  gem.description   = "Fluent output filter plugin for parsing key/value fields in records. Automatically determines type of the value as integer, float or string"
+  gem.homepage      = "https://github.com/CiscoZeus/fluent-plugin-fields-autotype"
+  gem.summary       = gem.description
+  gem.version       = File.read("VERSION").strip
+  gem.authors       = ["Manoj Sharma"]
+  gem.email         = ["vigyanik@gmail.com"]
+  gem.has_rdoc      = false
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd"
+  gem.add_development_dependency "rake"
+end
+

data/lib/fluent/plugin/out_fields_autotype.rb

@@ -0,0 +1,76 @@
+class String
+  def is_i?
+    /\A[-+]?\d+\z/ === self
+  end 
+  def nan?
+    self !~ /^\s*[+-]?((\d+_?)*\d+(\.(\d+_?)*\d+)?|\.(\d+_?)*\d+)(\s*|([eE][+-]?(\d+_?)*\d+)\s*)$/
+  end    
+end
+
+module Fluent
+  class OutputFieldsAutotype < Fluent::Output
+    Fluent::Plugin.register_output('fields_autotype', self)
+
+    config_param :remove_tag_prefix,  :string, :default => nil
+    config_param :add_tag_prefix,     :string, :default => nil
+    config_param :parse_key,          :string, :default => 'message'
+    config_param :fields_key,         :string, :default => nil
+    config_param :pattern,            :string,
+                 :default => %{(\\S+)=(\\S+)}
+
+
+    def compiled_pattern
+      @compiled_pattern ||= Regexp.new(pattern)
+    end
+
+    def emit(tag, es, chain)
+      tag = update_tag(tag)
+      es.each { |time, record|
+        Engine.emit(tag, time, parse_fields(record))
+      }
+      chain.next
+    end
+
+    def update_tag(tag)
+      if remove_tag_prefix
+        if remove_tag_prefix == tag
+          tag = ''
+        elsif tag.to_s.start_with?(remove_tag_prefix+'.')
+          tag = tag[remove_tag_prefix.length+1 .. -1]
+        end
+      end
+      if add_tag_prefix
+        tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
+      end
+      return tag
+    end
+
+    def parse_fields(record)
+      source = record[parse_key].to_s
+      target = fields_key ? (record[fields_key] ||= {}) : record
+
+      reduced_source = source.dup
+      until reduced_source.length == 0 do
+        match1 = reduced_source.match pattern
+        if !(match1.nil?)
+          key = match1[1].to_s
+          val = match1[2].to_s
+          if !(target.has_key?(key))
+            if val.is_i?
+              target[key] = val.to_i
+            elsif val.nan?
+              target[key] = val
+            else
+              target[key] = val.to_f
+            end
+          end
+          reduced_source = reduced_source[match1.offset(2)[1]..-1]
+        else
+          reduced_source = ""
+        end
+      end
+      return record
+    end
+  end
+end
+

data/test/out_fields_autotype.rb

@@ -0,0 +1,269 @@
+require 'fluent/test'
+require 'fluent/plugin/out_fields_autotype'
+
+class FieldsAutotypeOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='', tag='orig.test.tag')
+    Fluent::Test::OutputTestDriver.new(Fluent::OutputFieldsAutotype, tag).configure(conf)
+  end
+
+  def test_config_defaults
+    d = create_driver()
+
+    orig_message = %{parse this num=-56.7 tok=abc%25 null=}
+    d.run do
+      d.emit({
+        'message' => orig_message,
+        'other_key' => %{ test2 a=b },
+      })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'other_key' => %{ test2 a=b },
+        'num' => -56.7,
+        'tok' => 'abc%25',
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_quoted_values
+    d = create_driver()
+
+    orig_message = %{blax dq="asd ' asd +3" sq='as " s " 4' s=yu 6}
+    d.run do
+      d.emit({
+        'message' => orig_message,
+      })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'dq' => "\"asd",
+        'sq' => "'as",
+        's' => 'yu'
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_parsed_key_is_missing
+    d = create_driver()
+
+    d.run do
+      d.emit({})
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {},
+      emits[0][2]
+    )
+  end
+
+  def test_existing_keys_are_not_overriden
+    d = create_driver()
+
+    orig_message = %{mock a=77 message=blax a=999 e=5}
+    d.run do
+      d.emit({'message' => orig_message, 'e' => nil })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'a' => 77,
+        'e' => nil,
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_tag_prefixes
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    })
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new.test.tag", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag=nil)
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag='original')
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new.original", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag='orig')
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new", emits[0][0]
+  end
+
+  def test_parse_key
+    d = create_driver('parse_key  custom_key')
+
+    d.run do
+      d.emit({
+        'message' => %{ test2 c=d },
+        'custom_key' => %{ test2 a=b },
+      })
+      d.emit({})
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => %{ test2 c=d },
+        'custom_key' => %{ test2 a=b },
+        'a' => 'b'
+      },
+      emits[0][2]
+    )
+    assert_equal(
+      {
+      },
+      emits[1][2]
+    )
+  end
+
+  def test_fields_key
+    d = create_driver("fields_key output-key")
+
+    orig_message = %{parse this num=-56.7 tok=abc%25 message=a+b}
+    d.run do
+      d.emit({'message' => orig_message})
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'output-key' => {
+          'num' => -56.7,
+          'tok' => 'abc%25',
+          'message' => 'a+b',
+        }
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_custom_pattern
+    d = create_driver("pattern (\\w+):(\\d+)")
+
+    orig_message = %{parse this a:44 b:ignore-this h=7 bbb:999}
+    d.run do
+      d.emit({'message' => orig_message})
+      d.emit({'message' => 'a'})
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'a' => 44,
+        'bbb' => 999,
+      },
+      emits[0][2]
+    )
+    assert_equal(
+      {
+        'message' => 'a',
+      },
+      emits[1][2]
+    )
+  end
+
+def test_strict_key_value
+  d = create_driver()
+
+  orig_message = %{msg=Auditlog user=Johnny action=add-user dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
+  d.run do
+    d.emit({'message' => orig_message})
+    d.emit({'message' => 'a'})
+  end
+
+  emits = d.emits
+  assert_equal 2, emits.size
+  assert_equal "orig.test.tag", emits[0][0]
+  assert_equal(
+    {
+      'message' => orig_message,
+      "msg"=>"Auditlog",
+      'user' => "Johnny",
+      'action' => 'add-user',
+      'dontignore' => "don't-ignore-this",
+      'result' => 'success',
+      'iVal' => 23,
+      'fVal' => 1.02,
+      'bVal' => "true"
+    },
+    emits[0][2]
+  )
+  assert_equal(
+    {
+      'message' => 'a',
+    },
+    emits[1][2]
+  )
+end
+
+
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-fields-autotype
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
+platform: ruby
+authors:
+- Manoj Sharma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent output filter plugin for parsing key/value fields in records.
+  Automatically determines type of the value as integer, float or string
+email:
+- vigyanik@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-fields-autotype.gemspec
+- lib/fluent/plugin/out_fields_autotype.rb
+- test/out_fields_autotype.rb
+homepage: https://github.com/CiscoZeus/fluent-plugin-fields-autotype
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluent output filter plugin for parsing key/value fields in records. Automatically
+  determines type of the value as integer, float or string
+test_files: []