fluent-plugin-fields-parser 0.1.0

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/README.md

@@ -0,0 +1,119 @@
+fluent-plugin-fields-parser
+===========================
+
+Fluent output filter plugin for parsing key/value fields in records
+based on <key>=<value> pattern.
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-fields-parser
+
+## Configuration
+
+    <match pattern>
+        type                fields_parser
+
+        remove_tag_prefix   raw
+        add_tag_prefix      parsed
+    </match>
+
+If following record is passed:
+
+```
+{"message": "Audit log user=Johny action='add-user' result=success" }
+```
+
+then you got new record:
+
+```
+{
+    "message": "Audit log username=Johny action='add-user' result=success",
+    "user": "Johny",
+    "action": "add-user",
+    "result": "success"
+}
+```
+
+### Parameter parse_key
+
+For configuration
+
+    <match pattern>
+        type        fields_parser
+
+        parse_key   log_message
+    </match>
+
+it parses key "log_message" instead of default key `message`.
+
+### Parameter fields_key
+
+Configuration
+
+    <match pattern>
+        type        fields_parser
+
+        parse_key   log_message
+        fields_key  fields
+    </match>
+
+For input like:
+
+```
+{
+    "log_message": "Audit log username=Johny action='add-user' result=success",
+}
+```
+
+it adds parsed fields into defined key.
+
+```
+{
+    "log_message": "Audit log username=Johny action='add-user' result=success",
+    "fields": {"user": "Johny", "action": "add-user", "result": "success"}
+}
+```
+
+(It adds new keys into top-level record by default.)
+
+### Parameter pattern
+
+You can define custom pattern (regexp) for seaching keys/values.
+
+Configuration
+
+    <match pattern>
+        type        fields_parser
+
+        pattern     (\w+):(\d+)
+    </match>
+
+For input like:
+```
+{ "message": "data black:54 white=55 red=10"}
+```
+
+it returns:
+
+```
+{ "message": "data black:54 white=55 red=10",
+  "black": "54", "white": "55", "red": "10"
+}
+```
+
+### Tag prefix
+
+You cat add and/or remove tag prefix using Configuration parameters
+
+    <match pattern>
+        type                fields_parser
+
+        remove_tag_prefix   raw
+        add_tag_prefix      parsed
+    </match>
+
+It it matched tag "raw.some.record", then it emits tag "parsed.some.record".
+
+

data/Rakefile

@@ -0,0 +1,14 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-tokenizer.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-fields-parser"
+  gem.description   = "Fluent output filter plugin for parsing key/value fields in records"
+  gem.homepage      = "https://github.com/tomas-zemres/fluent-plugin-fields-parser"
+  gem.summary       = gem.description
+  gem.version       = File.read("VERSION").strip
+  gem.authors       = ["Tomas Pokorny"]
+  gem.email         = ["tomas.zemres@gmail.com"]
+  gem.has_rdoc      = false
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd"
+  gem.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_fields_parser.rb

@@ -0,0 +1,54 @@
+module Fluent
+  class OutputFieldsParser < Fluent::Output
+    Fluent::Plugin.register_output('fields_parser', self)
+
+    config_param :remove_tag_prefix,  :string, :default => nil
+    config_param :add_tag_prefix,     :string, :default => nil
+    config_param :parse_key,          :string, :default => 'message'
+    config_param :fields_key,         :string, :default => nil
+    config_param :pattern,            :string,
+                 :default => %{([a-zA-Z_]\\w*)=((['"]).*?(\\3)|[\\w.@$%/+-]*)}
+
+    def compiled_pattern
+      @compiled_pattern ||= Regexp.new(pattern)
+    end
+
+    def emit(tag, es, chain)
+      tag = update_tag(tag)
+      es.each { |time, record|
+        Engine.emit(tag, time, parse_fields(record))
+      }
+      chain.next
+    end
+
+    def update_tag(tag)
+      if remove_tag_prefix
+        if remove_tag_prefix == tag
+          tag = ''
+        elsif tag.to_s.start_with?(remove_tag_prefix+'.')
+          tag = tag[remove_tag_prefix.length+1 .. -1]
+        end
+      end
+      if add_tag_prefix
+        tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
+      end
+      return tag
+    end
+
+    def parse_fields(record)
+      source = record[parse_key].to_s
+      target = fields_key ? (record[fields_key] ||= {}) : record
+
+      source.scan(compiled_pattern) do |match|
+        (key, value, begining_quote, ending_quote) = match
+        next if key.nil?
+        next if target.has_key?(key)
+        value = value.to_s
+        from_pos = begining_quote.to_s.length
+        to_pos = value.length - ending_quote.to_s.length - 1
+        target[key] = value[from_pos..to_pos]
+      end
+      return record
+    end
+  end
+end

data/test/out_fields_parser.rb

@@ -0,0 +1,234 @@
+require 'fluent/test'
+require 'fluent/plugin/out_fields_parser'
+
+class FieldsParserOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='', tag='orig.test.tag')
+    Fluent::Test::OutputTestDriver.new(Fluent::OutputFieldsParser, tag).configure(conf)
+  end
+
+  def test_config_defaults
+    d = create_driver()
+
+    orig_message = %{parse this num=-56.7 tok=abc%25 null=}
+    d.run do
+      d.emit({
+        'message' => orig_message,
+        'other_key' => %{ test2 a=b },
+      })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'other_key' => %{ test2 a=b },
+        'num' => '-56.7',
+        'tok' => 'abc%25',
+        'null' => '',
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_quoted_values
+    d = create_driver()
+
+    orig_message = %{blax dq="asd ' asd +3" sq='as " s " 4' s=yu 6}
+    d.run do
+      d.emit({
+        'message' => orig_message,
+      })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'dq' => "asd ' asd +3",
+        'sq' => 'as " s " 4',
+        's' => 'yu'
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_parsed_key_is_missing
+    d = create_driver()
+
+    d.run do
+      d.emit({})
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {},
+      emits[0][2]
+    )
+  end
+
+  def test_existing_keys_are_not_overriden
+    d = create_driver()
+
+    orig_message = %{mock a=77 message=blax a=999 e=5}
+    d.run do
+      d.emit({'message' => orig_message, 'e' => nil })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'a' => '77',
+        'e' => nil,
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_tag_prefixes
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    })
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new.test.tag", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag=nil)
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag='original')
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new.original", emits[0][0]
+
+    d = create_driver(%{
+      remove_tag_prefix   orig
+      add_tag_prefix      new
+    }, tag='orig')
+
+    d.run do
+      d.emit({ message => 'abc' })
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "new", emits[0][0]
+  end
+
+  def test_parse_key
+    d = create_driver('parse_key  custom_key')
+
+    d.run do
+      d.emit({
+        'message' => %{ test2 c=d },
+        'custom_key' => %{ test2 a=b },
+      })
+      d.emit({})
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => %{ test2 c=d },
+        'custom_key' => %{ test2 a=b },
+        'a' => 'b'
+      },
+      emits[0][2]
+    )
+    assert_equal(
+      {
+      },
+      emits[1][2]
+    )
+  end
+
+  def test_fields_key
+    d = create_driver("fields_key output-key")
+
+    orig_message = %{parse this num=-56.7 tok=abc%25 message=a+b}
+    d.run do
+      d.emit({'message' => orig_message})
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'output-key' => {
+          'num' => '-56.7',
+          'tok' => 'abc%25',
+          'message' => 'a+b',
+        }
+      },
+      emits[0][2]
+    )
+  end
+
+  def test_custom_pattern
+    d = create_driver("pattern (\\w+):(\\d+)")
+
+    orig_message = %{parse this a:44 b:ignore-this h=7 bbb:999}
+    d.run do
+      d.emit({'message' => orig_message})
+      d.emit({'message' => 'a'})
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.size
+    assert_equal "orig.test.tag", emits[0][0]
+    assert_equal(
+      {
+        'message' => orig_message,
+        'a' => '44',
+        'bbb' => '999',
+      },
+      emits[0][2]
+    )
+    assert_equal(
+      {
+        'message' => 'a',
+      },
+      emits[1][2]
+    )
+  end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-fields-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Tomas Pokorny
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent output filter plugin for parsing key/value fields in records
+email:
+- tomas.zemres@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-tokenizer.gemspec
+- lib/fluent/plugin/out_fields_parser.rb
+- test/out_fields_parser.rb
+homepage: https://github.com/tomas-zemres/fluent-plugin-fields-parser
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluent output filter plugin for parsing key/value fields in records
+test_files: []