fluent-plugin-kvp-filter 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +34 -0
- data/.travis.yml +17 -0
- data/.travis/deploy.sh +7 -0
- data/Gemfile +2 -0
- data/README.md +155 -0
- data/Rakefile +14 -0
- data/VERSION +1 -0
- data/fluent-plugin-kvp-filter.gemspec +25 -0
- data/lib/fluent/plugin/filter_kvp.rb +52 -0
- data/test/filter_kvp.rb +269 -0
- metadata +110 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7b76c39c7009d841d6a82c0391080c7f70d1bbfd
|
4
|
+
data.tar.gz: 179eb3364b8e6e9c066dfb6338d7f7fcd7ff850a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 429df975faf7af7521634fd9ad6c4ef667c5075b31617de1df8c5598a924146f2a9c6c46b9aecc275ddf13e0783f9d2dea3e57bb69dfa1177f42bc8029acef36
|
7
|
+
data.tar.gz: 49885293083ece34914496c1b62770dc9c97045abc79fed75660b2c4c425d7d93d4e2013976a3632b6cbd9d61aaebf941534fd4eaaa7c2664220fb507c7e14a3
|
data/.gitignore
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/lib/bundler/man/
|
26
|
+
|
27
|
+
# for a library or gem, you might want to ignore these files since the code is
|
28
|
+
# intended to run in multiple environments; otherwise, check them in:
|
29
|
+
# Gemfile.lock
|
30
|
+
# .ruby-version
|
31
|
+
# .ruby-gemset
|
32
|
+
|
33
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
34
|
+
.rvmrc
|
data/.travis.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 2.2
|
4
|
+
- 2.1
|
5
|
+
deploy:
|
6
|
+
provider: script
|
7
|
+
script: .travis/deploy.sh
|
8
|
+
# provider: rubygems
|
9
|
+
# api_key:
|
10
|
+
# secure: Nl+oGHjLKpqh6xizIS6KKD4NofFDx5nvCI1psCXg5KCBl1cZQ6mZwMeqJYonrW3NgCpDHgTG0uImiRg6bAmzE4KC5zja6aDZ3jFzWjOa84vnnvwkzRKxbVwuN+Au5+wUGZQUS6T8WROA+AHO/WNYmfHEOvP9qcb56AGzIM7qFL88hoTDGnnOUOs6WEIc5TLmAu6h+/YccEwWrYw+6DYJ9zgZpl7HeOZsMO9vLrq8ud/s3dub1syTlMx9xZ4cqNEvTpMIPNWGx1W35EDbXMcWLbE6KLPxdlfHwk9rY6Kv9qHMH0FROvuiW9uSf/OQiIrOb1+ym+TOWO5sKAGsbgWOdp7rakQ1n6dP1nt3qJJW67Omldv518u+2t2uIyK8I6N4A4+Uk8jMZOUOBdyA6chpa/0btUM0KqG9qeUYS/Z4P0KSo1nabVaLgrAeUbdhTUo+jl2bU0kXmOtnMxrJ3N1f6iTt3aDKjJvV7tC1qbGtsk0IMq9wGTccCbBiBMQnnHguQ/MXEmvSA6l93e9sr8RjWUuJF70xr3n4OYgnfcvekhNKoznukMI7XascM2iQYw6hlKE7sD3b9tyfbdlM45Ak2XaIjzqQlM4296K2/0Q4lzwM2ijqeN2UhNUJAol9/o5xNwxwd/kmPgvqRkD5CCdtRlfQfAPFp8X00rkSEjpTcuk=
|
11
|
+
# gemspec: fluent-plugin-kvp-filter.gemspec
|
12
|
+
on:
|
13
|
+
tags: true
|
14
|
+
repo: matt-deboer/fluent-plugin-kvp-filter
|
15
|
+
env:
|
16
|
+
global:
|
17
|
+
- secure: bDv6wW3RpvXJJlEkMXPpqRBdmU9lOhXMf+x9znF2jXu7BEJQAb8ZsVJCJCbz2iTGan5/GdThhW3kKDjZz8bGlQP6J8j3lJmeQyGp1CW3DQBvHnoYruoqFdeycfgrf1E32ce97yQtw9/GNkb8dLAl309M+gUwk6UPZNgC62qohzXhSDU1eAuPZ5ad/uZ3uYtfgTY7IWpTycMP0dSm8kVr9lJADshRCP2iKf7op/5r9EiL/VU/XhXMT+2FqhD9t/YfdQsJlXo4JWvSoStPWO+YC6+rZ+vO6ovWMMxD6HYBPZUZqF4/KQXtggduHjQs9ogidG6rkEvBGKg/EfqnmeAHw/auEr38H0PGDwkuT8go332w2TwxLWdonpdYdJ77xG8Z4Sq+pQNCQX0papHgg4vGuWUjcRFIPufUBcTTwUnbRj4Dum92Q1NkBy4bgF/ZE4W1tEmjWZxxXhz790q05tWQv7Ng4mbCsnboV0K0JmBDl8tNZ7OAzQFadmorBea/dLQQbQbB383vPpbSAEABXyIlIpzoZmWAqvjWTngyTGuOIFW1lNG9G0rf+XDkus1+j6sZZQ5c3BZOm/C2fmaf4rPbe3qAiSUM2zpantiCsdn4zzS7LznWYADIgPZ2BJ/6bwtQYH45VhR+R6aG87oxTNO7wm6Iw1pSePZnPKGyWF0NEuM=
|
data/.travis/deploy.sh
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
# fluent-plugin-kvp-filter [![Build Status](https://travis-ci.org/matt-deboer/fluent-plugin-kvp-filter.png)](https://travis-ci.org/matt-deboer/fluent-plugin-kvp-filter)
|
2
|
+
|
3
|
+
Fluent filter plugin for parsing key/value fields in records
|
4
|
+
based on <key>=<value> pattern.
|
5
|
+
|
6
|
+
_Forked from [fluent-plugin-fields-parser](https://github.com/tomas-zemres/fluent-plugin-fields-parser)
|
7
|
+
and converted to a fluentd filter type._
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Use RubyGems:
|
12
|
+
|
13
|
+
gem install fluent-plugin-kvp-filter
|
14
|
+
|
15
|
+
## Configuration
|
16
|
+
|
17
|
+
<filter pattern>
|
18
|
+
type fields_parser
|
19
|
+
|
20
|
+
strict_key_value false
|
21
|
+
</filter>
|
22
|
+
|
23
|
+
If following record is passed:
|
24
|
+
|
25
|
+
```
|
26
|
+
{"message": "Audit log user=Johny action='add-user' result=success" }
|
27
|
+
```
|
28
|
+
|
29
|
+
then you will get a new record:
|
30
|
+
|
31
|
+
```
|
32
|
+
{
|
33
|
+
"message": "Audit log username=Johny action='add-user' result=success",
|
34
|
+
"user": "Johny",
|
35
|
+
"action": "add-user",
|
36
|
+
"result": "success"
|
37
|
+
}
|
38
|
+
```
|
39
|
+
|
40
|
+
### Parameter parse_key
|
41
|
+
|
42
|
+
For configuration
|
43
|
+
|
44
|
+
<filter pattern>
|
45
|
+
type fields_parser
|
46
|
+
|
47
|
+
parse_key log_message
|
48
|
+
</filter>
|
49
|
+
|
50
|
+
it parses key "log_message" instead of default key `message`.
|
51
|
+
|
52
|
+
### Parameter remove_parse_key
|
53
|
+
|
54
|
+
For configuration
|
55
|
+
|
56
|
+
<filter pattern>
|
57
|
+
type fields_parser
|
58
|
+
|
59
|
+
parse_key log_message
|
60
|
+
remove_parse_key true
|
61
|
+
</filter>
|
62
|
+
|
63
|
+
it will remove the key "log_message" after parsing/extracting key/value pairs from it.
|
64
|
+
|
65
|
+
### Parameter fields_key
|
66
|
+
|
67
|
+
Configuration
|
68
|
+
|
69
|
+
<filter pattern>
|
70
|
+
type fields_parser
|
71
|
+
|
72
|
+
parse_key log_message
|
73
|
+
fields_key fields
|
74
|
+
</filter>
|
75
|
+
|
76
|
+
For input like:
|
77
|
+
|
78
|
+
```
|
79
|
+
{
|
80
|
+
"log_message": "Audit log username=Johny action='add-user' result=success",
|
81
|
+
}
|
82
|
+
```
|
83
|
+
|
84
|
+
it adds parsed fields into defined key.
|
85
|
+
|
86
|
+
```
|
87
|
+
{
|
88
|
+
"log_message": "Audit log username=Johny action='add-user' result=success",
|
89
|
+
"fields": {"user": "Johny", "action": "add-user", "result": "success"}
|
90
|
+
}
|
91
|
+
```
|
92
|
+
|
93
|
+
(It adds new keys into top-level record by default.)
|
94
|
+
|
95
|
+
### Parameter pattern
|
96
|
+
|
97
|
+
You can define custom pattern (regexp) for seaching keys/values.
|
98
|
+
|
99
|
+
Configuration
|
100
|
+
|
101
|
+
<filter pattern>
|
102
|
+
type fields_parser
|
103
|
+
|
104
|
+
pattern (\w+):(\d+)
|
105
|
+
</filter>
|
106
|
+
|
107
|
+
For input like:
|
108
|
+
```
|
109
|
+
{ "message": "data black:54 white:55 red:10"}
|
110
|
+
```
|
111
|
+
|
112
|
+
it returns:
|
113
|
+
|
114
|
+
```
|
115
|
+
{ "message": "data black:54 white=55 red=10",
|
116
|
+
"black": "54", "white": "55", "red": "10"
|
117
|
+
}
|
118
|
+
```
|
119
|
+
|
120
|
+
### Parameter strict_key_value
|
121
|
+
|
122
|
+
```
|
123
|
+
<filter pattern>
|
124
|
+
type fields_parser
|
125
|
+
strict_key_value true
|
126
|
+
</filter>
|
127
|
+
```
|
128
|
+
|
129
|
+
If `strict_key_value` is set to `true`, the parser will use the [ruby logfmt
|
130
|
+
parser](https://github.com/cyberdelia/logfmt-ruby) which will parse the log
|
131
|
+
message based on the popular [logfmt](https://brandur.org/logfmt) key/value
|
132
|
+
format. Do note that this parser will create Fixnum and Float type values
|
133
|
+
when it parses integer and float values.
|
134
|
+
|
135
|
+
All information provided in the log message must be in a strict key=value
|
136
|
+
format. For example, if following record is passed:
|
137
|
+
|
138
|
+
```
|
139
|
+
{"message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true" }
|
140
|
+
```
|
141
|
+
|
142
|
+
then you will get a new record:
|
143
|
+
|
144
|
+
```
|
145
|
+
{
|
146
|
+
"message": "msg=\"Audit log\" user=Johnny action=\"add-user\" result=success iVal=23 fVal=1.02 bVal=true",
|
147
|
+
"msg": "Audit log",
|
148
|
+
"user": "Johnny",
|
149
|
+
"action": "add-user",
|
150
|
+
"result": "success",
|
151
|
+
"iVal": 23,
|
152
|
+
"fVal": 1.02,
|
153
|
+
"bVal": "true"
|
154
|
+
}
|
155
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require "bundler/gem_tasks"
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require 'rake/testtask'
|
7
|
+
|
8
|
+
Rake::TestTask.new(:test) do |test|
|
9
|
+
test.libs << 'lib' << 'test'
|
10
|
+
test.test_files = FileList['test/*.rb']
|
11
|
+
test.verbose = true
|
12
|
+
end
|
13
|
+
|
14
|
+
task :default => :test
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-kvp-filter"
|
6
|
+
gem.description = "Fluent filter plugin for parsing key/value fields in records"
|
7
|
+
gem.homepage = "https://github.com/matt-deboer/fluent-plugin-kvp-filter"
|
8
|
+
gem.summary = gem.description
|
9
|
+
gem.version = %x[git describe --tags --always]
|
10
|
+
gem.authors = ["Matt DeBoer"]
|
11
|
+
gem.email = ["matt.deboer@gmail.com"]
|
12
|
+
gem.has_rdoc = false
|
13
|
+
gem.license = 'MIT'
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
|
20
|
+
gem.add_dependency "fluentd", '>= 0.14.0'
|
21
|
+
gem.add_dependency "logfmt"
|
22
|
+
gem.add_development_dependency "rake"
|
23
|
+
gem.add_development_dependency 'test-unit'
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'fluent/time'
|
2
|
+
require 'fluent/config/error'
|
3
|
+
require 'fluent/plugin/filter'
|
4
|
+
require 'logfmt'
|
5
|
+
|
6
|
+
module Fluent::Plugin
|
7
|
+
class KVPFilter < Filter
|
8
|
+
Fluent::Plugin.register_filter('kvp', self)
|
9
|
+
|
10
|
+
config_param :parse_key, :string, :default => 'message'
|
11
|
+
config_param :fields_key, :string, :default => nil
|
12
|
+
config_param :pattern, :string,
|
13
|
+
:default => %{([a-zA-Z_]\\w*)=((['"]).*?(\\3)|[\\w.@$%/+-]*)}
|
14
|
+
config_param :strict_key_value, :bool, :default => false
|
15
|
+
config_param :remove_parse_key, :bool, :default => false
|
16
|
+
|
17
|
+
def configure(conf)
|
18
|
+
super
|
19
|
+
end
|
20
|
+
|
21
|
+
def compiled_pattern
|
22
|
+
@compiled_pattern ||= Regexp.new(pattern)
|
23
|
+
end
|
24
|
+
|
25
|
+
def filter(tag, time, record)
|
26
|
+
source = record[parse_key].to_s
|
27
|
+
target = fields_key ? (record[fields_key] ||= {}) : record
|
28
|
+
|
29
|
+
if strict_key_value
|
30
|
+
# Use logfmt to parse it (key=value)
|
31
|
+
parsed = Logfmt.parse(source)
|
32
|
+
target.merge!(parsed)
|
33
|
+
else
|
34
|
+
source.scan(compiled_pattern) do |match|
|
35
|
+
(key, value, begining_quote, ending_quote) = match
|
36
|
+
next if key.nil?
|
37
|
+
next if target.has_key?(key)
|
38
|
+
value = value.to_s
|
39
|
+
from_pos = begining_quote.to_s.length
|
40
|
+
to_pos = value.length - ending_quote.to_s.length - 1
|
41
|
+
target[key] = value[from_pos..to_pos]
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
if remove_parse_key
|
46
|
+
record.delete parse_key
|
47
|
+
end
|
48
|
+
|
49
|
+
return record
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
data/test/filter_kvp.rb
ADDED
@@ -0,0 +1,269 @@
|
|
1
|
+
# require 'fluent/test'
|
2
|
+
# require 'fluent/plugin/filter_kvp'
|
3
|
+
|
4
|
+
# class KVPFilterTest < Test::Unit::TestCase
|
5
|
+
# def setup
|
6
|
+
# Fluent::Test.setup
|
7
|
+
# end
|
8
|
+
|
9
|
+
# def create_driver(conf='', tag='orig.test.tag')
|
10
|
+
# Fluent::Test::OutputTestDriver.new(Fluent::OutputFieldsParser, tag).configure(conf)
|
11
|
+
# end
|
12
|
+
|
13
|
+
# def test_config_defaults
|
14
|
+
# d = create_driver()
|
15
|
+
|
16
|
+
# orig_message = %{parse this num=-56.7 tok=abc%25 null=}
|
17
|
+
# d.run do
|
18
|
+
# d.emit({
|
19
|
+
# 'message' => orig_message,
|
20
|
+
# 'other_key' => %{ test2 a=b },
|
21
|
+
# })
|
22
|
+
# end
|
23
|
+
|
24
|
+
# emits = d.emits
|
25
|
+
# assert_equal 1, emits.size
|
26
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
27
|
+
# assert_equal(
|
28
|
+
# {
|
29
|
+
# 'message' => orig_message,
|
30
|
+
# 'other_key' => %{ test2 a=b },
|
31
|
+
# 'num' => '-56.7',
|
32
|
+
# 'tok' => 'abc%25',
|
33
|
+
# 'null' => '',
|
34
|
+
# },
|
35
|
+
# emits[0][2]
|
36
|
+
# )
|
37
|
+
# end
|
38
|
+
|
39
|
+
# def test_quoted_values
|
40
|
+
# d = create_driver()
|
41
|
+
|
42
|
+
# orig_message = %{blax dq="asd ' asd +3" sq='as " s " 4' s=yu 6}
|
43
|
+
# d.run do
|
44
|
+
# d.emit({
|
45
|
+
# 'message' => orig_message,
|
46
|
+
# })
|
47
|
+
# end
|
48
|
+
|
49
|
+
# emits = d.emits
|
50
|
+
# assert_equal 1, emits.size
|
51
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
52
|
+
# assert_equal(
|
53
|
+
# {
|
54
|
+
# 'message' => orig_message,
|
55
|
+
# 'dq' => "asd ' asd +3",
|
56
|
+
# 'sq' => 'as " s " 4',
|
57
|
+
# 's' => 'yu'
|
58
|
+
# },
|
59
|
+
# emits[0][2]
|
60
|
+
# )
|
61
|
+
# end
|
62
|
+
|
63
|
+
# def test_parsed_key_is_missing
|
64
|
+
# d = create_driver()
|
65
|
+
|
66
|
+
# d.run do
|
67
|
+
# d.emit({})
|
68
|
+
# end
|
69
|
+
|
70
|
+
# emits = d.emits
|
71
|
+
# assert_equal 1, emits.size
|
72
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
73
|
+
# assert_equal(
|
74
|
+
# {},
|
75
|
+
# emits[0][2]
|
76
|
+
# )
|
77
|
+
# end
|
78
|
+
|
79
|
+
# def test_existing_keys_are_not_overriden
|
80
|
+
# d = create_driver()
|
81
|
+
|
82
|
+
# orig_message = %{mock a=77 message=blax a=999 e=5}
|
83
|
+
# d.run do
|
84
|
+
# d.emit({'message' => orig_message, 'e' => nil })
|
85
|
+
# end
|
86
|
+
|
87
|
+
# emits = d.emits
|
88
|
+
# assert_equal 1, emits.size
|
89
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
90
|
+
# assert_equal(
|
91
|
+
# {
|
92
|
+
# 'message' => orig_message,
|
93
|
+
# 'a' => '77',
|
94
|
+
# 'e' => nil,
|
95
|
+
# },
|
96
|
+
# emits[0][2]
|
97
|
+
# )
|
98
|
+
# end
|
99
|
+
|
100
|
+
# def test_tag_prefixes
|
101
|
+
# d = create_driver(%{
|
102
|
+
# remove_tag_prefix orig
|
103
|
+
# add_tag_prefix new
|
104
|
+
# })
|
105
|
+
|
106
|
+
# d.run do
|
107
|
+
# d.emit({ 'message' => 'abc' })
|
108
|
+
# end
|
109
|
+
|
110
|
+
# emits = d.emits
|
111
|
+
# assert_equal 1, emits.size
|
112
|
+
# assert_equal "new.test.tag", emits[0][0]
|
113
|
+
|
114
|
+
# d = create_driver(%{
|
115
|
+
# remove_tag_prefix orig
|
116
|
+
# add_tag_prefix new
|
117
|
+
# }, tag=nil)
|
118
|
+
|
119
|
+
# d.run do
|
120
|
+
# d.emit({ 'message' => 'abc' })
|
121
|
+
# end
|
122
|
+
|
123
|
+
# emits = d.emits
|
124
|
+
# assert_equal 1, emits.size
|
125
|
+
# assert_equal "new", emits[0][0]
|
126
|
+
|
127
|
+
# d = create_driver(%{
|
128
|
+
# remove_tag_prefix orig
|
129
|
+
# add_tag_prefix new
|
130
|
+
# }, tag='original')
|
131
|
+
|
132
|
+
# d.run do
|
133
|
+
# d.emit({ 'message' => 'abc' })
|
134
|
+
# end
|
135
|
+
|
136
|
+
# emits = d.emits
|
137
|
+
# assert_equal 1, emits.size
|
138
|
+
# assert_equal "new.original", emits[0][0]
|
139
|
+
|
140
|
+
# d = create_driver(%{
|
141
|
+
# remove_tag_prefix orig
|
142
|
+
# add_tag_prefix new
|
143
|
+
# }, tag='orig')
|
144
|
+
|
145
|
+
# d.run do
|
146
|
+
# d.emit({ 'message' => 'abc' })
|
147
|
+
# end
|
148
|
+
|
149
|
+
# emits = d.emits
|
150
|
+
# assert_equal 1, emits.size
|
151
|
+
# assert_equal "new", emits[0][0]
|
152
|
+
# end
|
153
|
+
|
154
|
+
# def test_parse_key
|
155
|
+
# d = create_driver('parse_key custom_key')
|
156
|
+
|
157
|
+
# d.run do
|
158
|
+
# d.emit({
|
159
|
+
# 'message' => %{ test2 c=d },
|
160
|
+
# 'custom_key' => %{ test2 a=b },
|
161
|
+
# })
|
162
|
+
# d.emit({})
|
163
|
+
# end
|
164
|
+
|
165
|
+
# emits = d.emits
|
166
|
+
# assert_equal 2, emits.size
|
167
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
168
|
+
# assert_equal(
|
169
|
+
# {
|
170
|
+
# 'message' => %{ test2 c=d },
|
171
|
+
# 'custom_key' => %{ test2 a=b },
|
172
|
+
# 'a' => 'b'
|
173
|
+
# },
|
174
|
+
# emits[0][2]
|
175
|
+
# )
|
176
|
+
# assert_equal(
|
177
|
+
# {
|
178
|
+
# },
|
179
|
+
# emits[1][2]
|
180
|
+
# )
|
181
|
+
# end
|
182
|
+
|
183
|
+
# def test_fields_key
|
184
|
+
# d = create_driver("fields_key output-key")
|
185
|
+
|
186
|
+
# orig_message = %{parse this num=-56.7 tok=abc%25 message=a+b}
|
187
|
+
# d.run do
|
188
|
+
# d.emit({'message' => orig_message})
|
189
|
+
# end
|
190
|
+
|
191
|
+
# emits = d.emits
|
192
|
+
# assert_equal 1, emits.size
|
193
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
194
|
+
# assert_equal(
|
195
|
+
# {
|
196
|
+
# 'message' => orig_message,
|
197
|
+
# 'output-key' => {
|
198
|
+
# 'num' => '-56.7',
|
199
|
+
# 'tok' => 'abc%25',
|
200
|
+
# 'message' => 'a+b',
|
201
|
+
# }
|
202
|
+
# },
|
203
|
+
# emits[0][2]
|
204
|
+
# )
|
205
|
+
# end
|
206
|
+
|
207
|
+
# def test_custom_pattern
|
208
|
+
# d = create_driver("pattern (\\w+):(\\d+)")
|
209
|
+
|
210
|
+
# orig_message = %{parse this a:44 b:ignore-this h=7 bbb:999}
|
211
|
+
# d.run do
|
212
|
+
# d.emit({'message' => orig_message})
|
213
|
+
# d.emit({'message' => 'a'})
|
214
|
+
# end
|
215
|
+
|
216
|
+
# emits = d.emits
|
217
|
+
# assert_equal 2, emits.size
|
218
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
219
|
+
# assert_equal(
|
220
|
+
# {
|
221
|
+
# 'message' => orig_message,
|
222
|
+
# 'a' => '44',
|
223
|
+
# 'bbb' => '999',
|
224
|
+
# },
|
225
|
+
# emits[0][2]
|
226
|
+
# )
|
227
|
+
# assert_equal(
|
228
|
+
# {
|
229
|
+
# 'message' => 'a',
|
230
|
+
# },
|
231
|
+
# emits[1][2]
|
232
|
+
# )
|
233
|
+
# end
|
234
|
+
|
235
|
+
# def test_strict_key_value
|
236
|
+
# d = create_driver("strict_key_value true")
|
237
|
+
|
238
|
+
# orig_message = %{msg="Audit log" user=Johnny action="add-user" dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
|
239
|
+
# d.run do
|
240
|
+
# d.emit({'message' => orig_message})
|
241
|
+
# d.emit({'message' => 'a'})
|
242
|
+
# end
|
243
|
+
|
244
|
+
# emits = d.emits
|
245
|
+
# assert_equal 2, emits.size
|
246
|
+
# assert_equal "orig.test.tag", emits[0][0]
|
247
|
+
# assert_equal(
|
248
|
+
# {
|
249
|
+
# 'message' => orig_message,
|
250
|
+
# "msg"=>"Audit log",
|
251
|
+
# 'user' => "Johnny",
|
252
|
+
# 'action' => 'add-user',
|
253
|
+
# 'dontignore' => "don't-ignore-this",
|
254
|
+
# 'result' => 'success',
|
255
|
+
# 'iVal' => 23,
|
256
|
+
# 'fVal' => 1.02,
|
257
|
+
# 'bVal' => "true"
|
258
|
+
# },
|
259
|
+
# emits[0][2]
|
260
|
+
# )
|
261
|
+
# assert_equal(
|
262
|
+
# {
|
263
|
+
# 'message' => 'a',
|
264
|
+
# },
|
265
|
+
# emits[1][2]
|
266
|
+
# )
|
267
|
+
# end
|
268
|
+
|
269
|
+
# end
|
metadata
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-kvp-filter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matt DeBoer
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-02-20 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.14.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: logfmt
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: test-unit
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Fluent filter plugin for parsing key/value fields in records
|
70
|
+
email:
|
71
|
+
- matt.deboer@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".travis.yml"
|
78
|
+
- ".travis/deploy.sh"
|
79
|
+
- Gemfile
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- VERSION
|
83
|
+
- fluent-plugin-kvp-filter.gemspec
|
84
|
+
- lib/fluent/plugin/filter_kvp.rb
|
85
|
+
- test/filter_kvp.rb
|
86
|
+
homepage: https://github.com/matt-deboer/fluent-plugin-kvp-filter
|
87
|
+
licenses:
|
88
|
+
- MIT
|
89
|
+
metadata: {}
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ">="
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubyforge_project:
|
106
|
+
rubygems_version: 2.4.5
|
107
|
+
signing_key:
|
108
|
+
specification_version: 4
|
109
|
+
summary: Fluent filter plugin for parsing key/value fields in records
|
110
|
+
test_files: []
|