fluent-plugin-filter-kv-parser 0.0.4 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/fluent-plugin-filter-kv-parser.gemspec +1 -1
- data/lib/fluent/plugin/filter_key_value_parser.rb +11 -6
- data/test/test_filter_key_value_parser.rb +38 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a03fd43ca0868a32f5f9b2fb708b0dbbeba01213f7cb6c14cba269dbcb0b4b35
|
4
|
+
data.tar.gz: 3370ceff8ef70d6ae8084beb43019f222212a179614a235c0b4fec9a278116d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be070361f7952a1b4bfd04b02a95f3a756cf66ed258a6462784ace866e0ded8ee7d53ced926f26a72fc7498884f37137931f8395416d9c66cfa41bf68f5079e5
|
7
|
+
data.tar.gz: a426868603e91dc578921ec8750912cc83cd8d6063477353a842c838a16c0276487d4ce8b001e2c9937969d77aab1a20f1ba29c183e015adf24e181d4d9d75ee
|
data/README.md
CHANGED
@@ -24,9 +24,11 @@ gem install fluent-plugin-filter-kv-parser
|
|
24
24
|
remove_key false
|
25
25
|
remove_prefix /^prefix/
|
26
26
|
keys_delimiter /\s+/
|
27
|
-
|
27
|
+
kv_delimiter_char '='
|
28
28
|
filtered_keys akey,bkey,ckey
|
29
29
|
filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
|
30
|
+
filter_out_lines_without_keys false
|
31
|
+
keys_prefix prefix
|
30
32
|
</filter>
|
31
33
|
|
32
34
|
<match **>
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-filter-kv-parser"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.9"
|
7
7
|
s.authors = ["Al-waleed Shihadeh"]
|
8
8
|
s.email = ["wshihadeh.dev@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"
|
@@ -7,13 +7,15 @@ module Fluent
|
|
7
7
|
|
8
8
|
config_param :key, :string, default: 'log'
|
9
9
|
config_param :remove_key, :bool, default: false
|
10
|
+
config_param :filter_out_lines_without_keys, :bool, default: false
|
10
11
|
config_param :use_regex, :bool, default: false
|
11
12
|
config_param :remove_prefix, :string, default: ''
|
12
13
|
config_param :keys_delimiter, :string, default: '/\s+/'
|
13
|
-
config_param :
|
14
|
+
config_param :kv_delimiter_char, :string, default: '='
|
14
15
|
config_param :filtered_keys, :string, default: nil
|
15
16
|
config_param :filtered_keys_regex, :string, default: nil
|
16
17
|
config_param :filtered_keys_delimiter, :string, default: ','
|
18
|
+
config_param :keys_prefix, :string, default: nil
|
17
19
|
|
18
20
|
|
19
21
|
def configure(conf)
|
@@ -39,28 +41,31 @@ module Fluent
|
|
39
41
|
def filter(tag, time, record)
|
40
42
|
return if record[@key].nil?
|
41
43
|
|
42
|
-
|
44
|
+
keys = extracted_keys(extract_log_line(record[@key]))
|
43
45
|
|
44
|
-
|
46
|
+
return if @filter_out_lines_without_keys && keys.empty?
|
47
|
+
|
48
|
+
record.merge! keys
|
45
49
|
record.tap { |r| r.delete(@key) if @remove_key }.compact
|
46
50
|
end
|
47
51
|
|
48
52
|
private
|
49
53
|
|
50
54
|
def regex_filter(line)
|
51
|
-
"#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>[^=]+)\s/).to_h
|
55
|
+
"#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>([^=]+|[^ ]+))\s/).to_h
|
52
56
|
end
|
53
57
|
|
54
58
|
def extracted_keys(line)
|
55
59
|
keys = @use_regex ? regex_filter(line) : delimiter_filter(line)
|
56
60
|
filtered_keys = @filtered_keys_list.empty? ? keys : keys.slice(*@filtered_keys_list)
|
57
|
-
@filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
|
61
|
+
filtered_keys = @filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
|
62
|
+
@keys_prefix.nil? ? filtered_keys : filtered_keys.transform_keys!{ |key| "#{@keys_prefix}.#{key}" }
|
58
63
|
end
|
59
64
|
|
60
65
|
def delimiter_filter(line)
|
61
66
|
items = {}
|
62
67
|
line.split(@keys_delimiter).each do |kv|
|
63
|
-
key, value = kv.split(@
|
68
|
+
key, value = kv.split(@kv_delimiter_char, 2)
|
64
69
|
items[key] = value if value
|
65
70
|
end
|
66
71
|
items
|
@@ -188,4 +188,42 @@ class KeyValueFilterTest < Test::Unit::TestCase
|
|
188
188
|
assert_equal "0", filtered['sub_key']
|
189
189
|
assert_equal false, filtered.key?("log")
|
190
190
|
end
|
191
|
+
|
192
|
+
test 'test_filter_out_lines_without_keys' do
|
193
|
+
d = create_driver(%[
|
194
|
+
key log
|
195
|
+
remove_key true
|
196
|
+
use_regex true
|
197
|
+
filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
|
198
|
+
filter_out_lines_without_keys true
|
199
|
+
])
|
200
|
+
msg = {
|
201
|
+
'time' => '2013-02-12 22:01:15 UTC',
|
202
|
+
'log' => "akey=10 bkey=11 ckey=11",
|
203
|
+
}
|
204
|
+
|
205
|
+
msg2 = {
|
206
|
+
'time' => '2013-02-12 22:01:15 UTC',
|
207
|
+
'log' => "Start Request to test lines without any keys",
|
208
|
+
}
|
209
|
+
filtered = filter(d, [msg, msg2])
|
210
|
+
|
211
|
+
assert_equal 1, filtered.count
|
212
|
+
assert_equal 4, filtered.first[2].count
|
213
|
+
assert_equal "10", filtered.first[2]['akey']
|
214
|
+
end
|
215
|
+
|
216
|
+
test 'test_keys_prefix' do
|
217
|
+
d = create_driver(%[
|
218
|
+
key log
|
219
|
+
keys_prefix test
|
220
|
+
])
|
221
|
+
msg = {
|
222
|
+
'time' => '2013-02-12 22:01:15 UTC',
|
223
|
+
'log' => 'Start Request key=10 akey=20 zkey=30 dkey=40',
|
224
|
+
}
|
225
|
+
filtered = filter(d, [msg]).first[2]
|
226
|
+
assert_equal false, filtered.key?("key")
|
227
|
+
assert_equal true, filtered.key?("test.key")
|
228
|
+
end
|
191
229
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-filter-kv-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Al-waleed Shihadeh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
- !ruby/object:Gem::Version
|
122
122
|
version: '0'
|
123
123
|
requirements: []
|
124
|
-
rubygems_version: 3.0.3
|
124
|
+
rubygems_version: 3.0.3.1
|
125
125
|
signing_key:
|
126
126
|
specification_version: 4
|
127
127
|
summary: A Fluentd filter plugin to parse key value items
|