fluent-plugin-filter-kv-parser 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d03d782c5c6058dff073d09bf35fa161acb555661e6f825ff205919051f3e19a
-  data.tar.gz: b5edd6ad840a00eb31bd957b5548f01e41530d71649f68c3ae3b7f3312d60af4
+  metadata.gz: 06f855fd104485ee6469bda9fd116eb9a5a0e04d186297b246d21ee37f82f931
+  data.tar.gz: 14f5e141dff49e437dcb3d9dd047869057d1a9c2ca5de02b695fbfb06d60b107
 SHA512:
-  metadata.gz: 49db420a06c68ce7c32dcaa3465d5d11d8fcff4ce206c95220479c6b139cd06fcbea8c0dd5e2b76db07e74855b07bc6f792384c4c29ff35e2a1504f03c303638
-  data.tar.gz: 36c16f8fb13e98ba8317062bd288a2f92ce9baf5419c6bae8ac0aa26b70328582f006424ae83dcc2ead3075b72f6d8877af7ad467993b74ed2b5045ad665a334
+  metadata.gz: a93b2f140f1f0914dbefa08d19d7f9f50033ff4c09e6fe20ca61c01281b27aae32e28051fd87f56723989e3b5ed03b5df3de976a3163deb4355959d121b4e12e
+  data.tar.gz: c7adfcf6a29ff9016495800e3e95c9cb2b23baa9233ce9a621adf63960849e83df20235753e1214f4e813d085af0bf5afd7a958032cd8ba3191ed56157a5bbe6

data/README.md

@@ -25,6 +25,8 @@ gem install fluent-plugin-filter-kv-parser
   remove_prefix /^prefix/
   keys_delimiter /\s+/
   kv_delimiter_chart '='
+  filtered_keys akey,bkey,ckey
+  filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
 </filter>
 
 <match **>

data/fluent-plugin-filter-kv-parser.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-filter-kv-parser"
-  s.version     = "0.0.3"
+  s.version     = "0.0.4"
   s.authors     = ["Al-waleed Shihadeh"]
   s.email       = ["wshihadeh.dev@gmail.com"]
   s.homepage    = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"

data/lib/fluent/plugin/filter_key_value_parser.rb

@@ -11,17 +11,29 @@ module Fluent
     config_param :remove_prefix, :string, default: ''
     config_param :keys_delimiter, :string, default: '/\s+/'
     config_param :kv_delimiter_chart, :string, default: '='
+    config_param :filtered_keys, :string, default: nil
+    config_param :filtered_keys_regex, :string, default: nil
+    config_param :filtered_keys_delimiter, :string, default: ','
 
 
     def configure(conf)
       super
-      if @keys_delimiter[0] == '/' and @keys_delimiter[-1] == '/'
+
+      regex = /^\/.+\/$/
+
+      if regex.match(@keys_delimiter.to_s)
         @keys_delimiter = Regexp.new(@keys_delimiter[1..-2])
       end
 
-      if @remove_prefix[0] == '/' and @remove_prefix[-1] == '/'
+      if regex.match(@remove_prefix.to_s)
         @remove_prefix = Regexp.new(@remove_prefix[1..-2])
       end
+
+      if regex.match(@filtered_keys_regex.to_s)
+         @filtered_keys_regex = Regexp.new(@filtered_keys_regex[1..-2])
+      end
+
+      @filtered_keys_list = parse_filtered_keys_parameter
     end
 
     def filter(tag, time, record)
@@ -29,19 +41,20 @@ module Fluent
 
       log_line = extract_log_line record[@key]
 
-      if @use_regex
-        record.merge! regex_filter(log_line)
-      else
-        record.merge! delimiter_filter(log_line)
-      end
-
+      record.merge! extracted_keys(log_line)
       record.tap { |r| r.delete(@key) if @remove_key }.compact
     end
 
     private
 
     def regex_filter(line)
-      "#{line} ".scan(/(?<key>[^ ]+)=(?<value>[^=]+)\s/).to_h
+      "#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>[^=]+)\s/).to_h
+    end
+
+    def extracted_keys(line)
+      keys = @use_regex ? regex_filter(line) : delimiter_filter(line)
+      filtered_keys = @filtered_keys_list.empty? ? keys : keys.slice(*@filtered_keys_list)
+      @filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
     end
 
     def delimiter_filter(line)
@@ -53,6 +66,10 @@ module Fluent
       items
     end
 
+    def parse_filtered_keys_parameter
+      @filtered_keys.to_s.split(@filtered_keys_delimiter)
+    end
+
     def extract_log_line(line)
       line.gsub(@remove_prefix,'').strip
     end

data/test/test_filter_key_value_parser.rb

@@ -146,9 +146,46 @@ class KeyValueFilterTest < Test::Unit::TestCase
       'log'  => "Start Request key=10 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
     }
     filtered = filter(d, [msg]).first[2]
-    puts filtered.inspect
     assert_equal 6, filtered.count
     assert_equal "'this is a miltispace line'", filtered['skey']
     assert_equal false,  filtered.key?("log")
   end
+
+  test 'test_filter_keys' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+        filtered_keys key,gkeyn,nkey,skey,akey,zkey
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request key=10 gkey=100 nkey=108 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 6, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_filter_keys_with_regex' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+        filtered_keys key,gkeyn,nkey,skey,akey,zkey
+        filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request sub_key=0 sub_akey=11 sub_zkey=12 key=10 gkey=100 nkey=108 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 9, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal "0", filtered['sub_key']
+    assert_equal false,  filtered.key?("log")
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-kv-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Al-waleed Shihadeh