fluent-plugin-filter-kv-parser 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 27a9342381452e0d88dac38dc1c64dbbd1b913368c6b444f0adb27c182f7a547
4
- data.tar.gz: fe84db2f5478cbb4e0694c5a7e45d7889d2c88689e420080de49f862848f4949
3
+ metadata.gz: d03d782c5c6058dff073d09bf35fa161acb555661e6f825ff205919051f3e19a
4
+ data.tar.gz: b5edd6ad840a00eb31bd957b5548f01e41530d71649f68c3ae3b7f3312d60af4
5
5
  SHA512:
6
- metadata.gz: 4c496463a7dbf8c59eeea45e4295ed9f18d114a2957262ea2983fabfa179ef56a7cad533bb3070292b2d72700ee2778ce23fb10ef7516ccbd4d9f37fcbd7bc4e
7
- data.tar.gz: 2d6a20e24e5219fa16f3af51a3b1a1916e178b41ecef581abf57193219091545b79164bafdc805d1b41056632e382989e255e68bd5a51e283761a580190b7f81
6
+ metadata.gz: 49db420a06c68ce7c32dcaa3465d5d11d8fcff4ce206c95220479c6b139cd06fcbea8c0dd5e2b76db07e74855b07bc6f792384c4c29ff35e2a1504f03c303638
7
+ data.tar.gz: 36c16f8fb13e98ba8317062bd288a2f92ce9baf5419c6bae8ac0aa26b70328582f006424ae83dcc2ead3075b72f6d8877af7ad467993b74ed2b5045ad665a334
data/README.md CHANGED
@@ -20,6 +20,8 @@ gem install fluent-plugin-filter-kv-parser
20
20
  <filter **>
21
21
  type key_value_parser
22
22
  key log
23
+ use_regex true
24
+ remove_key false
23
25
  remove_prefix /^prefix/
24
26
  keys_delimiter /\s+/
25
27
  kv_delimiter_chart '='
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-filter-kv-parser"
6
- s.version = "0.0.2"
6
+ s.version = "0.0.3"
7
7
  s.authors = ["Al-waleed Shihadeh"]
8
8
  s.email = ["wshihadeh.dev@gmail.com"]
9
9
  s.homepage = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"
@@ -7,6 +7,7 @@ module Fluent
7
7
 
8
8
  config_param :key, :string, default: 'log'
9
9
  config_param :remove_key, :bool, default: false
10
+ config_param :use_regex, :bool, default: false
10
11
  config_param :remove_prefix, :string, default: ''
11
12
  config_param :keys_delimiter, :string, default: '/\s+/'
12
13
  config_param :kv_delimiter_chart, :string, default: '='
@@ -24,10 +25,14 @@ module Fluent
24
25
  end
25
26
 
26
27
  def filter(tag, time, record)
28
+ return if record[@key].nil?
29
+
27
30
  log_line = extract_log_line record[@key]
28
- log_line.split(@keys_delimiter).each do |kv|
29
- key, value = kv.split(@kv_delimiter_chart, 2)
30
- record[key] = value if value
31
+
32
+ if @use_regex
33
+ record.merge! regex_filter(log_line)
34
+ else
35
+ record.merge! delimiter_filter(log_line)
31
36
  end
32
37
 
33
38
  record.tap { |r| r.delete(@key) if @remove_key }.compact
@@ -35,6 +40,19 @@ module Fluent
35
40
 
36
41
  private
37
42
 
43
+ def regex_filter(line)
44
+ "#{line} ".scan(/(?<key>[^ ]+)=(?<value>[^=]+)\s/).to_h
45
+ end
46
+
47
+ def delimiter_filter(line)
48
+ items = {}
49
+ line.split(@keys_delimiter).each do |kv|
50
+ key, value = kv.split(@kv_delimiter_chart, 2)
51
+ items[key] = value if value
52
+ end
53
+ items
54
+ end
55
+
38
56
  def extract_log_line(line)
39
57
  line.gsub(@remove_prefix,'').strip
40
58
  end
@@ -83,4 +83,72 @@ class KeyValueFilterTest < Test::Unit::TestCase
83
83
  assert_equal 5, filtered.count
84
84
  assert_equal false, filtered.key?("log")
85
85
  end
86
+
87
+ test 'test_empty_line' do
88
+ d = create_driver(%[
89
+ key log
90
+ remove_key true
91
+ remove_prefix /^[^ ]+\s[^ ]+/
92
+ ])
93
+ msg = {
94
+ 'time' => '2013-02-12 22:01:15 UTC',
95
+ 'log' => 'Start Request',
96
+ }
97
+ filtered = filter(d, [msg]).first[2]
98
+ assert_equal 1, filtered.count
99
+ assert_equal false, filtered.key?("log")
100
+ end
101
+
102
+ test 'test_two_log_key' do
103
+ d = create_driver(%[
104
+ key log
105
+ remove_key true
106
+ remove_prefix /^[^ ]+\s[^ ]+/
107
+ ])
108
+ msg = {
109
+ 'time' => '2013-02-12 22:01:15 UTC',
110
+ 'log' => 'Start Request key=10 akey=20 zkey=30 dkey=40',
111
+ }
112
+ msg2 = {
113
+ 'time' => '2013-02-12 22:01:15 UTC',
114
+ 'log' => 'Start Request key=10 akey=20 zkey=30 dkey=40',
115
+ }
116
+ filtered = filter(d, [msg, msg2])
117
+ assert_equal 2, filtered.count
118
+ end
119
+
120
+ test 'test_no_log_key' do
121
+ d = create_driver(%[
122
+ key log
123
+ remove_key true
124
+ remove_prefix /^[^ ]+\s[^ ]+/
125
+ ])
126
+ msg = {
127
+ 'time' => '2013-02-12 22:01:15 UTC',
128
+ }
129
+ msg2 = {
130
+ 'time' => '2013-02-12 22:01:15 UTC',
131
+ 'log' => 'Start Request key=10 akey=20 zkey=30 dkey=40',
132
+ }
133
+ filtered = filter(d, [msg, msg2])
134
+ assert_equal 1, filtered.count
135
+ end
136
+
137
+ test 'test_with_space_and_regex' do
138
+ d = create_driver(%[
139
+ key log
140
+ remove_key true
141
+ remove_prefix /^[^ ]+\s[^ ]+/
142
+ use_regex true
143
+ ])
144
+ msg = {
145
+ 'time' => '2013-02-12 22:01:15 UTC',
146
+ 'log' => "Start Request key=10 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
147
+ }
148
+ filtered = filter(d, [msg]).first[2]
149
+ puts filtered.inspect
150
+ assert_equal 6, filtered.count
151
+ assert_equal "'this is a miltispace line'", filtered['skey']
152
+ assert_equal false, filtered.key?("log")
153
+ end
86
154
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-filter-kv-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Al-waleed Shihadeh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-08 00:00:00.000000000 Z
11
+ date: 2020-06-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd