fluent-plugin-keyvalue-parser 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7190c9f67c5deb19e52a51b95a10648f7d86f6df
4
+ data.tar.gz: 3e7a26b53a596d609fbec7fbca341164e1dc75e9
5
+ SHA512:
6
+ metadata.gz: cf3ebd7f9f3a1f4bee6f00ac6f436aa9625a11df7ae19c60e5dfb7113bf7f86652f4490b659632e84e2543c88983765001e048c5ddd16a0e45f4c4c116104934
7
+ data.tar.gz: 712c43402ae7aa17aed13fdb1520a71e9e427c0e94e7f23e8ca4d68618da873b5e61f49b3dcdd9c1978c7f67a62a6aebc9692aa91494bc205c29210a559b603d
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,104 @@
1
+ #fluent-plugin-keyvalue-parser
2
+
3
+ [Fluent](http://www.fluentd.org/) parser plugin for key:value formatted logs.
4
+
5
+
6
+ ##Installation
7
+
8
+ ```shell
9
+ $ td-agent-gem install fluent-plugin-keyvalue-parser
10
+ ```
11
+
12
+ ##How to use
13
+
14
+ Edit `/etc/td-agent/td-agent.conf` file.
15
+
16
+ * with tail plugin
17
+ ```conf
18
+ <source>
19
+ type tail
20
+ path /var/log/netscreen.log
21
+ tag netscreen_logs
22
+ pair_delimiter ","
23
+ key_value_seperator "="
24
+ pos_file /var/run/td-agent/netscreen-log.pos
25
+ format keyvalue
26
+ </source>
27
+ ```
28
+ * with parser plugin
29
+ ```conf
30
+ <filter tag>
31
+ type parser
32
+ format keyvalue
33
+ pair_delimiter ","
34
+ key_value_seperator "="
35
+ key_name keyToParse
36
+ </filter>
37
+ ```
38
+ using above configuration,
39
+ ```
40
+ key1=val1,key2=value2,"some key" = somevalue,diff_key="another value"
41
+ ```
42
+ will be parsed as
43
+
44
+ ```json
45
+ {"key1":"val1", "key2":"value2","some key":"somevalue","diff_key":"another value"}
46
+ ```
47
+
48
+ ####NOTE
49
+ * if the key is not in quotes and pair_delimiter occures in key,plugin will handle it.
50
+
51
+ eg:
52
+
53
+ In below log, *pair_delimiter = " " (space)* is occured in key 'src zone'.
54
+
55
+ `devname=FT6H duration=194 service=http proto=6 `**`src zone=Trust`**` port=40055 policy_id=194`
56
+
57
+ will be parsed as
58
+ ```json
59
+ {"devname":"FT6H", "duration":"194","service":"http","src zone":"Trust","policy_id":"194"}
60
+ ```
61
+ * But if value is not quoted, you should use optional parameter *'adjustment_rules'* to correct the parsing.
62
+
63
+ ## Option Parameters
64
+
65
+ - **pair_delimiter**
66
+
67
+ delimiter which seperate each key-value pairs. can be multi-character.
68
+ whitespaces or tabs can be given in quotes: ie, " " or "\t" .
69
+ By default it is ",".
70
+
71
+ - **key_value_seperator**
72
+
73
+ A string or character that seprates key and its value.
74
+ By default it is "="
75
+ - **adjustment_rules**
76
+
77
+ Regular expression rules for some keys, represented as json , to adjust parsed records accordingly.
78
+
79
+ {key1:regex1,key2:regex2}
80
+
81
+ eg:
82
+
83
+ normally following logs,
84
+
85
+ `devname=FT6H `**`service=http`**`proto=6 src zone=Trust dst zone=Untrust`
86
+
87
+ `devname=FT6H `**`service=NETBIOS (NS)`**`proto=17 src zone=Trust dst zone=Untrust`
88
+
89
+ will be parsed as
90
+
91
+ ```json
92
+ {"devname":"FT6H","service":"http","proto":"6","src zone":"Trust","dst zone":"Untrust"}
93
+
94
+ {"devname":"FT6H","service":"NETBIOS","(NS) proto":"6","src zone":"Trust","dst zone":"Untrust"}
95
+ ```
96
+ in second case, key *"service"* only received first part of its value, becouse value not quoted and delimiter(here space) occured in the value.
97
+
98
+ Also next key *"proto"* is wrongly parsed as *"(NS) proto"*.
99
+
100
+ to rectify this problem, we can use,
101
+
102
+ `adjustment_rules {"service":"NETBIOS \\(.*\\)"}` in configuration.
103
+
104
+ this will parse *service* key with a value containing *NETBIOS (NS)* whenever it occures.
@@ -0,0 +1,9 @@
1
+ # encoding: utf-8
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rspec/core'
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new(:spec) {| spec |
7
+ spec.pattern = FileList['spec/*_spec.rb']
8
+ }
9
+ task :default => :spec
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ #lib = File.expand_path('../lib', __FILE__)
3
+ #$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ $:.push File.expand_path('../lib', __FILE__)
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "fluent-plugin-keyvalue-parser"
8
+ spec.version = "0.1.4"
9
+ spec.authors = ["Arun M J"]
10
+ spec.email = ["arunmj001@gmail.com"]
11
+ spec.homepage = "https://github.com/arunmj/fluent-plugin-keyvalue-parser"
12
+ spec.description = %q{Fluentd parser plugin for key-value formatted logs.}
13
+ spec.summary = %q{This Fluentd parser plugin can be used to parse key-value pairs.}
14
+ spec.license = "MIT"
15
+ spec.has_rdoc = false
16
+
17
+ spec.files = `git ls-files`.split("\n")
18
+ spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
19
+ spec.executables = `git ls-files -- bin/*`.split("\n").map{| f | File.basename(f)}
20
+ spec.require_paths = ['lib']
21
+
22
+ spec.add_dependency 'fluentd', "~> 0.10"
23
+ spec.add_development_dependency "rake"
24
+ spec.add_development_dependency "rspec"
25
+
26
+ spec.required_ruby_version = '~> 2.0'
27
+ end
@@ -0,0 +1,139 @@
1
+ require 'fluent/parser'
2
+ require 'fluent/log'
3
+ require 'fluent/time'
4
+ require 'json'
5
+
6
+
7
+ module Fluent
8
+ class TextParser
9
+ class KeyValueParser < Parser
10
+
11
+ QUOTE = "\""
12
+
13
+ # Register this parser as "keyvalue"
14
+ Plugin.register_parser("keyvalue", self)
15
+
16
+ config_param :pair_delimiter, :string, :default => " "
17
+ config_param :key_value_seperator, :string, :default => ","
18
+
19
+ config_param :adjustment_rules , :default => false do |val|
20
+ rule_hash = false
21
+ if val != ""
22
+ begin
23
+ rule_hash_raw = JSON.parse(val)
24
+ rescue JSON::ParserError => ex
25
+ # Fluent::ConfigParseError, "got incomplete JSON" will be raised
26
+ raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
27
+ end
28
+ raise Fluent::ConfigError, "adjustment_rules is not a hash" unless rule_hash_raw.is_a?(Hash)
29
+ #rule_hash = make_rule_hash(rule_hash_raw)
30
+ rule_hash = {}
31
+ rule_hash_raw.each do |k,v|
32
+ regx_str = "(?<scavenged>#{v})#{@pair_delimiter}(?<remnants>.*$)"
33
+ rule_regex = //.class.new(regx_str)
34
+ rule_hash[k] = rule_regex
35
+ end
36
+ end
37
+ rule_hash
38
+ end
39
+
40
+
41
+
42
+ def configure(conf)
43
+ super
44
+ end
45
+
46
+ def parse(text)
47
+
48
+ record = {}
49
+ is_at_key =true
50
+ is_quote_open = false
51
+ key_name = ""
52
+ value = ""
53
+
54
+ text.each_char do |chr|
55
+
56
+ if is_quote_open && chr != QUOTE
57
+ if is_at_key
58
+ key_name << chr
59
+ else
60
+ value << chr
61
+ end
62
+ next
63
+ end
64
+
65
+ case chr
66
+ when QUOTE
67
+ is_quote_open = !is_quote_open
68
+ next
69
+
70
+ when @pair_delimiter
71
+ if is_at_key
72
+ key_name << chr
73
+ next
74
+ end
75
+ record[key_name] = value
76
+ key_name = ""
77
+ value = ""
78
+ is_at_key = true
79
+ next
80
+ when @key_value_seperator
81
+ if is_at_key then
82
+ is_at_key = false
83
+ else
84
+ value << chr
85
+ end
86
+ next
87
+ else
88
+ if is_at_key
89
+ key_name << chr
90
+ else
91
+ value << chr
92
+ end
93
+ next
94
+ end
95
+ end
96
+
97
+ if key_name != ""
98
+ if value != "" then
99
+ record[key_name] = value
100
+ else
101
+ record[record.keys.last] << @pair_delimiter << key_name
102
+ end
103
+ end
104
+
105
+ if @adjustment_rules
106
+ record = adjust_record(record)
107
+ end
108
+ yield nil,record
109
+ end
110
+
111
+
112
+
113
+ def adjust_record(record)
114
+ record.keys.each_with_index do |k,i|
115
+ if adjustment_rules.key? k
116
+ $log.debug "adjusting record ... for key : #{k}"
117
+ neighbour_key = record.keys[i+1]
118
+ valstr = [record[k],neighbour_key].join(@pair_delimiter)
119
+ puts "origina : " + valstr
120
+ m = adjustment_rules[k].match(valstr)
121
+ unless m
122
+ puts "no match"
123
+ else
124
+ puts m[0],m[1],m[2]
125
+ m.names.each do |x|
126
+ puts x + " : " + m[x]
127
+ end
128
+ record[k] = m['scavenged']
129
+ remnants = m['remnants']
130
+ record[remnants] = record.delete neighbour_key
131
+ end
132
+ end
133
+ end
134
+ return record
135
+ end
136
+
137
+ end
138
+ end
139
+ end
metadata ADDED
@@ -0,0 +1,91 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-keyvalue-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.4
5
+ platform: ruby
6
+ authors:
7
+ - Arun M J
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-03-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.10'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.10'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: Fluentd parser plugin for key-value formatted logs.
56
+ email:
57
+ - arunmj001@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - Gemfile
63
+ - README.md
64
+ - Rakefile
65
+ - fluent-plugin-keyvalue-parser.gemspec
66
+ - lib/fluent/plugin/parser_keyvalue.rb
67
+ homepage: https://github.com/arunmj/fluent-plugin-keyvalue-parser
68
+ licenses:
69
+ - MIT
70
+ metadata: {}
71
+ post_install_message:
72
+ rdoc_options: []
73
+ require_paths:
74
+ - lib
75
+ required_ruby_version: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '2.0'
80
+ required_rubygems_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ">="
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ requirements: []
86
+ rubyforge_project:
87
+ rubygems_version: 2.5.1
88
+ signing_key:
89
+ specification_version: 4
90
+ summary: This Fluentd parser plugin can be used to parse key-value pairs.
91
+ test_files: []