fluent-plugin-kvp-filter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7b76c39c7009d841d6a82c0391080c7f70d1bbfd
+  data.tar.gz: 179eb3364b8e6e9c066dfb6338d7f7fcd7ff850a
+SHA512:
+  metadata.gz: 429df975faf7af7521634fd9ad6c4ef667c5075b31617de1df8c5598a924146f2a9c6c46b9aecc275ddf13e0783f9d2dea3e57bb69dfa1177f42bc8029acef36
+  data.tar.gz: 49885293083ece34914496c1b62770dc9c97045abc79fed75660b2c4c425d7d93d4e2013976a3632b6cbd9d61aaebf941534fd4eaaa7c2664220fb507c7e14a3

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+rvm:
+- 2.2
+- 2.1
+deploy:
+  provider: script
+  script: .travis/deploy.sh
+  # provider: rubygems
+  # api_key:
+  #   secure: Nl+oGHjLKpqh6xizIS6KKD4NofFDx5nvCI1psCXg5KCBl1cZQ6mZwMeqJYonrW3NgCpDHgTG0uImiRg6bAmzE4KC5zja6aDZ3jFzWjOa84vnnvwkzRKxbVwuN+Au5+wUGZQUS6T8WROA+AHO/WNYmfHEOvP9qcb56AGzIM7qFL88hoTDGnnOUOs6WEIc5TLmAu6h+/YccEwWrYw+6DYJ9zgZpl7HeOZsMO9vLrq8ud/s3dub1syTlMx9xZ4cqNEvTpMIPNWGx1W35EDbXMcWLbE6KLPxdlfHwk9rY6Kv9qHMH0FROvuiW9uSf/OQiIrOb1+ym+TOWO5sKAGsbgWOdp7rakQ1n6dP1nt3qJJW67Omldv518u+2t2uIyK8I6N4A4+Uk8jMZOUOBdyA6chpa/0btUM0KqG9qeUYS/Z4P0KSo1nabVaLgrAeUbdhTUo+jl2bU0kXmOtnMxrJ3N1f6iTt3aDKjJvV7tC1qbGtsk0IMq9wGTccCbBiBMQnnHguQ/MXEmvSA6l93e9sr8RjWUuJF70xr3n4OYgnfcvekhNKoznukMI7XascM2iQYw6hlKE7sD3b9tyfbdlM45Ak2XaIjzqQlM4296K2/0Q4lzwM2ijqeN2UhNUJAol9/o5xNwxwd/kmPgvqRkD5CCdtRlfQfAPFp8X00rkSEjpTcuk=
+  # gemspec: fluent-plugin-kvp-filter.gemspec
+  on:
+    tags: true
+    repo: matt-deboer/fluent-plugin-kvp-filter
+env:
+  global:
+  - secure: bDv6wW3RpvXJJlEkMXPpqRBdmU9lOhXMf+x9znF2jXu7BEJQAb8ZsVJCJCbz2iTGan5/GdThhW3kKDjZz8bGlQP6J8j3lJmeQyGp1CW3DQBvHnoYruoqFdeycfgrf1E32ce97yQtw9/GNkb8dLAl309M+gUwk6UPZNgC62qohzXhSDU1eAuPZ5ad/uZ3uYtfgTY7IWpTycMP0dSm8kVr9lJADshRCP2iKf7op/5r9EiL/VU/XhXMT+2FqhD9t/YfdQsJlXo4JWvSoStPWO+YC6+rZ+vO6ovWMMxD6HYBPZUZqF4/KQXtggduHjQs9ogidG6rkEvBGKg/EfqnmeAHw/auEr38H0PGDwkuT8go332w2TwxLWdonpdYdJ77xG8Z4Sq+pQNCQX0papHgg4vGuWUjcRFIPufUBcTTwUnbRj4Dum92Q1NkBy4bgF/ZE4W1tEmjWZxxXhz790q05tWQv7Ng4mbCsnboV0K0JmBDl8tNZ7OAzQFadmorBea/dLQQbQbB383vPpbSAEABXyIlIpzoZmWAqvjWTngyTGuOIFW1lNG9G0rf+XDkus1+j6sZZQ5c3BZOm/C2fmaf4rPbe3qAiSUM2zpantiCsdn4zzS7LznWYADIgPZ2BJ/6bwtQYH45VhR+R6aG87oxTNO7wm6Iw1pSePZnPKGyWF0NEuM=

data/.travis/deploy.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+yes y | gem uninstall dpl
+yes y | gem uninstall gems
+gem install gems -v 0.8.3
+gem install dpl -v 1.8.27
+dpl --provider=rubygems --api-key=$RUBY_GEMS_API_KEY --gemspec=./fluent-plugin-kvp-filter.gemspec

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,155 @@
+# fluent-plugin-kvp-filter [![Build Status](https://travis-ci.org/matt-deboer/fluent-plugin-kvp-filter.png)](https://travis-ci.org/matt-deboer/fluent-plugin-kvp-filter)
+
+Fluent filter plugin for parsing key/value fields in records
+based on <key>=<value> pattern.
+
+_Forked from [fluent-plugin-fields-parser](https://github.com/tomas-zemres/fluent-plugin-fields-parser)
+and converted to a fluentd filter type._
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-kvp-filter
+
+## Configuration
+
+    <filter pattern>
+        type                fields_parser
+
+        strict_key_value     false
+    </filter>
+
+If following record is passed:
+
+```
+{"message": "Audit log user=Johny action='add-user' result=success" }
+```
+
+then you will get a new record:
+
+```
+{
+    "message": "Audit log username=Johny action='add-user' result=success",
+    "user": "Johny",
+    "action": "add-user",
+    "result": "success"
+}
+```
+
+### Parameter parse_key
+
+For configuration
+
+    <filter pattern>
+        type        fields_parser
+
+        parse_key   log_message
+    </filter>
+
+it parses key "log_message" instead of default key `message`.
+
+### Parameter remove_parse_key
+
+For configuration
+
+    <filter pattern>
+        type                fields_parser
+
+        parse_key           log_message
+        remove_parse_key    true
+    </filter>
+
+it will remove the key "log_message" after parsing/extracting key/value pairs from it.
+
+### Parameter fields_key
+
+Configuration
+
+    <filter pattern>
+        type        fields_parser
+
+        parse_key   log_message
+        fields_key  fields
+    </filter>
+
+For input like:
+
+```
+{
+    "log_message": "Audit log username=Johny action='add-user' result=success",
+}
+```
+
+it adds parsed fields into defined key.
+
+```
+{
+    "log_message": "Audit log username=Johny action='add-user' result=success",
+    "fields": {"user": "Johny", "action": "add-user", "result": "success"}
+}
+```
+
+(It adds new keys into top-level record by default.)
+
+### Parameter pattern
+
+You can define custom pattern (regexp) for seaching keys/values.
+
+Configuration
+
+    <filter pattern>
+        type        fields_parser
+
+        pattern     (\w+):(\d+)
+    </filter>
+
+For input like:
+```
+{ "message": "data black:54 white:55 red:10"}
+```
+
+it returns:
+
+```
+{ "message": "data black:54 white=55 red=10",
+  "black": "54", "white": "55", "red": "10"
+}
+```
+
+### Parameter strict_key_value
+
+```
+    <filter pattern>
+        type                fields_parser
+        strict_key_value   true
+    </filter>
+```
+
+If `strict_key_value` is set to `true`, the parser will use the [ruby logfmt
+parser](https://github.com/cyberdelia/logfmt-ruby) which will parse the log
+message based on the popular [logfmt](https://brandur.org/logfmt) key/value
+format.  Do note that this parser will create Fixnum and Float type values
+when it parses integer and float values.
+
+All information provided in the log message must be in a strict key=value
+format.  For example, if following record is passed:
+
+```
+{"message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true" }
+```
+
+then you will get a new record:
+
+```
+{
+    "message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true",
+    "msg": "Audit log",
+    "user": "Johnny",
+    "action": "add-user",
+    "result": "success",
+    "iVal": 23,
+    "fVal": 1.02,
+    "bVal": "true"
+}
+```

data/Rakefile

@@ -0,0 +1,14 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-kvp-filter.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-kvp-filter"
+  gem.description   = "Fluent filter plugin for parsing key/value fields in records"
+  gem.homepage      = "https://github.com/matt-deboer/fluent-plugin-kvp-filter"
+  gem.summary       = gem.description
+  gem.version       = %x[git describe --tags --always]
+  gem.authors       = ["Matt DeBoer"]
+  gem.email         = ["matt.deboer@gmail.com"]
+  gem.has_rdoc      = false
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", '>= 0.14.0'
+  gem.add_dependency "logfmt"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency 'test-unit'
+end
+

data/lib/fluent/plugin/filter_kvp.rb

@@ -0,0 +1,52 @@
+require 'fluent/time'
+require 'fluent/config/error'
+require 'fluent/plugin/filter'
+require 'logfmt'
+
+module Fluent::Plugin
+  class KVPFilter < Filter
+    Fluent::Plugin.register_filter('kvp', self)
+
+    config_param :parse_key,          :string, :default => 'message'
+    config_param :fields_key,         :string, :default => nil
+    config_param :pattern,            :string,
+                 :default => %{([a-zA-Z_]\\w*)=((['"]).*?(\\3)|[\\w.@$%/+-]*)}
+    config_param :strict_key_value,  :bool, :default => false
+    config_param :remove_parse_key,  :bool, :default => false
+
+    def configure(conf)
+      super
+    end
+
+    def compiled_pattern
+      @compiled_pattern ||= Regexp.new(pattern)
+    end
+
+    def filter(tag, time, record)
+      source = record[parse_key].to_s
+      target = fields_key ? (record[fields_key] ||= {}) : record
+
+      if strict_key_value
+        # Use logfmt to parse it (key=value)
+        parsed = Logfmt.parse(source)
+        target.merge!(parsed)
+      else
+        source.scan(compiled_pattern) do |match|
+          (key, value, begining_quote, ending_quote) = match
+          next if key.nil?
+          next if target.has_key?(key)
+          value = value.to_s
+          from_pos = begining_quote.to_s.length
+          to_pos = value.length - ending_quote.to_s.length - 1
+          target[key] = value[from_pos..to_pos]
+        end
+      end
+
+      if remove_parse_key
+        record.delete parse_key
+      end
+
+      return record
+    end
+  end
+end

data/test/filter_kvp.rb

@@ -0,0 +1,269 @@
+# require 'fluent/test'
+# require 'fluent/plugin/filter_kvp'
+
+# class KVPFilterTest < Test::Unit::TestCase
+#   def setup
+#     Fluent::Test.setup
+#   end
+
+#   def create_driver(conf='', tag='orig.test.tag')
+#     Fluent::Test::OutputTestDriver.new(Fluent::OutputFieldsParser, tag).configure(conf)
+#   end
+
+#   def test_config_defaults
+#     d = create_driver()
+
+#     orig_message = %{parse this num=-56.7 tok=abc%25 null=}
+#     d.run do
+#       d.emit({
+#         'message' => orig_message,
+#         'other_key' => %{ test2 a=b },
+#       })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         'other_key' => %{ test2 a=b },
+#         'num' => '-56.7',
+#         'tok' => 'abc%25',
+#         'null' => '',
+#       },
+#       emits[0][2]
+#     )
+#   end
+
+#   def test_quoted_values
+#     d = create_driver()
+
+#     orig_message = %{blax dq="asd ' asd +3" sq='as " s " 4' s=yu 6}
+#     d.run do
+#       d.emit({
+#         'message' => orig_message,
+#       })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         'dq' => "asd ' asd +3",
+#         'sq' => 'as " s " 4',
+#         's' => 'yu'
+#       },
+#       emits[0][2]
+#     )
+#   end
+
+#   def test_parsed_key_is_missing
+#     d = create_driver()
+
+#     d.run do
+#       d.emit({})
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {},
+#       emits[0][2]
+#     )
+#   end
+
+#   def test_existing_keys_are_not_overriden
+#     d = create_driver()
+
+#     orig_message = %{mock a=77 message=blax a=999 e=5}
+#     d.run do
+#       d.emit({'message' => orig_message, 'e' => nil })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         'a' => '77',
+#         'e' => nil,
+#       },
+#       emits[0][2]
+#     )
+#   end
+
+#   def test_tag_prefixes
+#     d = create_driver(%{
+#       remove_tag_prefix   orig
+#       add_tag_prefix      new
+#     })
+
+#     d.run do
+#       d.emit({ 'message' => 'abc' })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "new.test.tag", emits[0][0]
+
+#     d = create_driver(%{
+#       remove_tag_prefix   orig
+#       add_tag_prefix      new
+#     }, tag=nil)
+
+#     d.run do
+#       d.emit({ 'message' => 'abc' })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "new", emits[0][0]
+
+#     d = create_driver(%{
+#       remove_tag_prefix   orig
+#       add_tag_prefix      new
+#     }, tag='original')
+
+#     d.run do
+#       d.emit({ 'message' => 'abc' })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "new.original", emits[0][0]
+
+#     d = create_driver(%{
+#       remove_tag_prefix   orig
+#       add_tag_prefix      new
+#     }, tag='orig')
+
+#     d.run do
+#       d.emit({ 'message' => 'abc' })
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "new", emits[0][0]
+#   end
+
+#   def test_parse_key
+#     d = create_driver('parse_key  custom_key')
+
+#     d.run do
+#       d.emit({
+#         'message' => %{ test2 c=d },
+#         'custom_key' => %{ test2 a=b },
+#       })
+#       d.emit({})
+#     end
+
+#     emits = d.emits
+#     assert_equal 2, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => %{ test2 c=d },
+#         'custom_key' => %{ test2 a=b },
+#         'a' => 'b'
+#       },
+#       emits[0][2]
+#     )
+#     assert_equal(
+#       {
+#       },
+#       emits[1][2]
+#     )
+#   end
+
+#   def test_fields_key
+#     d = create_driver("fields_key output-key")
+
+#     orig_message = %{parse this num=-56.7 tok=abc%25 message=a+b}
+#     d.run do
+#       d.emit({'message' => orig_message})
+#     end
+
+#     emits = d.emits
+#     assert_equal 1, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         'output-key' => {
+#           'num' => '-56.7',
+#           'tok' => 'abc%25',
+#           'message' => 'a+b',
+#         }
+#       },
+#       emits[0][2]
+#     )
+#   end
+
+#   def test_custom_pattern
+#     d = create_driver("pattern (\\w+):(\\d+)")
+
+#     orig_message = %{parse this a:44 b:ignore-this h=7 bbb:999}
+#     d.run do
+#       d.emit({'message' => orig_message})
+#       d.emit({'message' => 'a'})
+#     end
+
+#     emits = d.emits
+#     assert_equal 2, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         'a' => '44',
+#         'bbb' => '999',
+#       },
+#       emits[0][2]
+#     )
+#     assert_equal(
+#       {
+#         'message' => 'a',
+#       },
+#       emits[1][2]
+#     )
+#   end
+
+#   def test_strict_key_value
+#     d = create_driver("strict_key_value true")
+
+#     orig_message = %{msg="Audit log" user=Johnny action="add-user" dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
+#     d.run do
+#       d.emit({'message' => orig_message})
+#       d.emit({'message' => 'a'})
+#     end
+
+#     emits = d.emits
+#     assert_equal 2, emits.size
+#     assert_equal "orig.test.tag", emits[0][0]
+#     assert_equal(
+#       {
+#         'message' => orig_message,
+#         "msg"=>"Audit log",
+#         'user' => "Johnny",
+#         'action' => 'add-user',
+#         'dontignore' => "don't-ignore-this",
+#         'result' => 'success',
+#         'iVal' => 23,
+#         'fVal' => 1.02,
+#         'bVal' => "true"
+#       },
+#       emits[0][2]
+#     )
+#     assert_equal(
+#       {
+#         'message' => 'a',
+#       },
+#       emits[1][2]
+#     )
+#   end
+
+# end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kvp-filter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matt DeBoer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: logfmt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent filter plugin for parsing key/value fields in records
+email:
+- matt.deboer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- ".travis/deploy.sh"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-kvp-filter.gemspec
+- lib/fluent/plugin/filter_kvp.rb
+- test/filter_kvp.rb
+homepage: https://github.com/matt-deboer/fluent-plugin-kvp-filter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Fluent filter plugin for parsing key/value fields in records
+test_files: []