fluent-plugin-keyvalue-parser 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/README.md +104 -0
- data/Rakefile +9 -0
- data/fluent-plugin-keyvalue-parser.gemspec +27 -0
- data/lib/fluent/plugin/parser_keyvalue.rb +139 -0
- metadata +91 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7190c9f67c5deb19e52a51b95a10648f7d86f6df
|
4
|
+
data.tar.gz: 3e7a26b53a596d609fbec7fbca341164e1dc75e9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: cf3ebd7f9f3a1f4bee6f00ac6f436aa9625a11df7ae19c60e5dfb7113bf7f86652f4490b659632e84e2543c88983765001e048c5ddd16a0e45f4c4c116104934
|
7
|
+
data.tar.gz: 712c43402ae7aa17aed13fdb1520a71e9e427c0e94e7f23e8ca4d68618da873b5e61f49b3dcdd9c1978c7f67a62a6aebc9692aa91494bc205c29210a559b603d
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
#fluent-plugin-keyvalue-parser
|
2
|
+
|
3
|
+
[Fluent](http://www.fluentd.org/) parser plugin for key:value formatted logs.
|
4
|
+
|
5
|
+
|
6
|
+
##Installation
|
7
|
+
|
8
|
+
```shell
|
9
|
+
$ td-agent-gem install fluent-plugin-keyvalue-parser
|
10
|
+
```
|
11
|
+
|
12
|
+
##How to use
|
13
|
+
|
14
|
+
Edit `/etc/td-agent/td-agent.conf` file.
|
15
|
+
|
16
|
+
* with tail plugin
|
17
|
+
```conf
|
18
|
+
<source>
|
19
|
+
type tail
|
20
|
+
path /var/log/netscreen.log
|
21
|
+
tag netscreen_logs
|
22
|
+
pair_delimiter ","
|
23
|
+
key_value_seperator "="
|
24
|
+
pos_file /var/run/td-agent/netscreen-log.pos
|
25
|
+
format keyvalue
|
26
|
+
</source>
|
27
|
+
```
|
28
|
+
* with parser plugin
|
29
|
+
```conf
|
30
|
+
<filter tag>
|
31
|
+
type parser
|
32
|
+
format keyvalue
|
33
|
+
pair_delimiter ","
|
34
|
+
key_value_seperator "="
|
35
|
+
key_name keyToParse
|
36
|
+
</filter>
|
37
|
+
```
|
38
|
+
using above configuration,
|
39
|
+
```
|
40
|
+
key1=val1,key2=value2,"some key" = somevalue,diff_key="another value"
|
41
|
+
```
|
42
|
+
will be parsed as
|
43
|
+
|
44
|
+
```json
|
45
|
+
{"key1":"val1", "key2":"value2","some key":"somevalue","diff_key":"another value"}
|
46
|
+
```
|
47
|
+
|
48
|
+
####NOTE
|
49
|
+
* if the key is not in quotes and pair_delimiter occures in key,plugin will handle it.
|
50
|
+
|
51
|
+
eg:
|
52
|
+
|
53
|
+
In below log, *pair_delimiter = " " (space)* is occured in key 'src zone'.
|
54
|
+
|
55
|
+
`devname=FT6H duration=194 service=http proto=6 `**`src zone=Trust`**` port=40055 policy_id=194`
|
56
|
+
|
57
|
+
will be parsed as
|
58
|
+
```json
|
59
|
+
{"devname":"FT6H", "duration":"194","service":"http","src zone":"Trust","policy_id":"194"}
|
60
|
+
```
|
61
|
+
* But if value is not quoted, you should use optional parameter *'adjustment_rules'* to correct the parsing.
|
62
|
+
|
63
|
+
## Option Parameters
|
64
|
+
|
65
|
+
- **pair_delimiter**
|
66
|
+
|
67
|
+
delimiter which seperate each key-value pairs. can be multi-character.
|
68
|
+
whitespaces or tabs can be given in quotes: ie, " " or "\t" .
|
69
|
+
By default it is ",".
|
70
|
+
|
71
|
+
- **key_value_seperator**
|
72
|
+
|
73
|
+
A string or character that seprates key and its value.
|
74
|
+
By default it is "="
|
75
|
+
- **adjustment_rules**
|
76
|
+
|
77
|
+
Regular expression rules for some keys, represented as json , to adjust parsed records accordingly.
|
78
|
+
|
79
|
+
{key1:regex1,key2:regex2}
|
80
|
+
|
81
|
+
eg:
|
82
|
+
|
83
|
+
normally following logs,
|
84
|
+
|
85
|
+
`devname=FT6H `**`service=http`**`proto=6 src zone=Trust dst zone=Untrust`
|
86
|
+
|
87
|
+
`devname=FT6H `**`service=NETBIOS (NS)`**`proto=17 src zone=Trust dst zone=Untrust`
|
88
|
+
|
89
|
+
will be parsed as
|
90
|
+
|
91
|
+
```json
|
92
|
+
{"devname":"FT6H","service":"http","proto":"6","src zone":"Trust","dst zone":"Untrust"}
|
93
|
+
|
94
|
+
{"devname":"FT6H","service":"NETBIOS","(NS) proto":"6","src zone":"Trust","dst zone":"Untrust"}
|
95
|
+
```
|
96
|
+
in second case, key *"service"* only received first part of its value, becouse value not quoted and delimiter(here space) occured in the value.
|
97
|
+
|
98
|
+
Also next key *"proto"* is wrongly parsed as *"(NS) proto"*.
|
99
|
+
|
100
|
+
to rectify this problem, we can use,
|
101
|
+
|
102
|
+
`adjustment_rules {"service":"NETBIOS \\(.*\\)"}` in configuration.
|
103
|
+
|
104
|
+
this will parse *service* key with a value containing *NETBIOS (NS)* whenever it occures.
|
data/Rakefile
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
#lib = File.expand_path('../lib', __FILE__)
|
3
|
+
#$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
$:.push File.expand_path('../lib', __FILE__)
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "fluent-plugin-keyvalue-parser"
|
8
|
+
spec.version = "0.1.4"
|
9
|
+
spec.authors = ["Arun M J"]
|
10
|
+
spec.email = ["arunmj001@gmail.com"]
|
11
|
+
spec.homepage = "https://github.com/arunmj/fluent-plugin-keyvalue-parser"
|
12
|
+
spec.description = %q{Fluentd parser plugin for key-value formatted logs.}
|
13
|
+
spec.summary = %q{This Fluentd parser plugin can be used to parse key-value pairs.}
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.has_rdoc = false
|
16
|
+
|
17
|
+
spec.files = `git ls-files`.split("\n")
|
18
|
+
spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
spec.executables = `git ls-files -- bin/*`.split("\n").map{| f | File.basename(f)}
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_dependency 'fluentd', "~> 0.10"
|
23
|
+
spec.add_development_dependency "rake"
|
24
|
+
spec.add_development_dependency "rspec"
|
25
|
+
|
26
|
+
spec.required_ruby_version = '~> 2.0'
|
27
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'fluent/parser'
|
2
|
+
require 'fluent/log'
|
3
|
+
require 'fluent/time'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
|
7
|
+
module Fluent
|
8
|
+
class TextParser
|
9
|
+
class KeyValueParser < Parser
|
10
|
+
|
11
|
+
QUOTE = "\""
|
12
|
+
|
13
|
+
# Register this parser as "keyvalue"
|
14
|
+
Plugin.register_parser("keyvalue", self)
|
15
|
+
|
16
|
+
config_param :pair_delimiter, :string, :default => " "
|
17
|
+
config_param :key_value_seperator, :string, :default => ","
|
18
|
+
|
19
|
+
config_param :adjustment_rules , :default => false do |val|
|
20
|
+
rule_hash = false
|
21
|
+
if val != ""
|
22
|
+
begin
|
23
|
+
rule_hash_raw = JSON.parse(val)
|
24
|
+
rescue JSON::ParserError => ex
|
25
|
+
# Fluent::ConfigParseError, "got incomplete JSON" will be raised
|
26
|
+
raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
|
27
|
+
end
|
28
|
+
raise Fluent::ConfigError, "adjustment_rules is not a hash" unless rule_hash_raw.is_a?(Hash)
|
29
|
+
#rule_hash = make_rule_hash(rule_hash_raw)
|
30
|
+
rule_hash = {}
|
31
|
+
rule_hash_raw.each do |k,v|
|
32
|
+
regx_str = "(?<scavenged>#{v})#{@pair_delimiter}(?<remnants>.*$)"
|
33
|
+
rule_regex = //.class.new(regx_str)
|
34
|
+
rule_hash[k] = rule_regex
|
35
|
+
end
|
36
|
+
end
|
37
|
+
rule_hash
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
def configure(conf)
|
43
|
+
super
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse(text)
|
47
|
+
|
48
|
+
record = {}
|
49
|
+
is_at_key =true
|
50
|
+
is_quote_open = false
|
51
|
+
key_name = ""
|
52
|
+
value = ""
|
53
|
+
|
54
|
+
text.each_char do |chr|
|
55
|
+
|
56
|
+
if is_quote_open && chr != QUOTE
|
57
|
+
if is_at_key
|
58
|
+
key_name << chr
|
59
|
+
else
|
60
|
+
value << chr
|
61
|
+
end
|
62
|
+
next
|
63
|
+
end
|
64
|
+
|
65
|
+
case chr
|
66
|
+
when QUOTE
|
67
|
+
is_quote_open = !is_quote_open
|
68
|
+
next
|
69
|
+
|
70
|
+
when @pair_delimiter
|
71
|
+
if is_at_key
|
72
|
+
key_name << chr
|
73
|
+
next
|
74
|
+
end
|
75
|
+
record[key_name] = value
|
76
|
+
key_name = ""
|
77
|
+
value = ""
|
78
|
+
is_at_key = true
|
79
|
+
next
|
80
|
+
when @key_value_seperator
|
81
|
+
if is_at_key then
|
82
|
+
is_at_key = false
|
83
|
+
else
|
84
|
+
value << chr
|
85
|
+
end
|
86
|
+
next
|
87
|
+
else
|
88
|
+
if is_at_key
|
89
|
+
key_name << chr
|
90
|
+
else
|
91
|
+
value << chr
|
92
|
+
end
|
93
|
+
next
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
if key_name != ""
|
98
|
+
if value != "" then
|
99
|
+
record[key_name] = value
|
100
|
+
else
|
101
|
+
record[record.keys.last] << @pair_delimiter << key_name
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
if @adjustment_rules
|
106
|
+
record = adjust_record(record)
|
107
|
+
end
|
108
|
+
yield nil,record
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
|
113
|
+
def adjust_record(record)
|
114
|
+
record.keys.each_with_index do |k,i|
|
115
|
+
if adjustment_rules.key? k
|
116
|
+
$log.debug "adjusting record ... for key : #{k}"
|
117
|
+
neighbour_key = record.keys[i+1]
|
118
|
+
valstr = [record[k],neighbour_key].join(@pair_delimiter)
|
119
|
+
puts "origina : " + valstr
|
120
|
+
m = adjustment_rules[k].match(valstr)
|
121
|
+
unless m
|
122
|
+
puts "no match"
|
123
|
+
else
|
124
|
+
puts m[0],m[1],m[2]
|
125
|
+
m.names.each do |x|
|
126
|
+
puts x + " : " + m[x]
|
127
|
+
end
|
128
|
+
record[k] = m['scavenged']
|
129
|
+
remnants = m['remnants']
|
130
|
+
record[remnants] = record.delete neighbour_key
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
return record
|
135
|
+
end
|
136
|
+
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-keyvalue-parser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Arun M J
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-03-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.10'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.10'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Fluentd parser plugin for key-value formatted logs.
|
56
|
+
email:
|
57
|
+
- arunmj001@gmail.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- Gemfile
|
63
|
+
- README.md
|
64
|
+
- Rakefile
|
65
|
+
- fluent-plugin-keyvalue-parser.gemspec
|
66
|
+
- lib/fluent/plugin/parser_keyvalue.rb
|
67
|
+
homepage: https://github.com/arunmj/fluent-plugin-keyvalue-parser
|
68
|
+
licenses:
|
69
|
+
- MIT
|
70
|
+
metadata: {}
|
71
|
+
post_install_message:
|
72
|
+
rdoc_options: []
|
73
|
+
require_paths:
|
74
|
+
- lib
|
75
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - "~>"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '2.0'
|
80
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
requirements: []
|
86
|
+
rubyforge_project:
|
87
|
+
rubygems_version: 2.5.1
|
88
|
+
signing_key:
|
89
|
+
specification_version: 4
|
90
|
+
summary: This Fluentd parser plugin can be used to parse key-value pairs.
|
91
|
+
test_files: []
|