fluent-plugin-filter-kv-parser 0.0.1 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ed4b252f8b033a1adf97ff84262c182e16c2e280d0e8a647fca1c93b28851d1
-  data.tar.gz: 9b4801ce9b57c7b2a8705c8f85b963616d2e004f6653ed861700fa3e2e4b4cfe
+  metadata.gz: 8932148ec7a3bfaa0a6241823bda594b85d5a91b24dca0156fb2e7497e4f5eef
+  data.tar.gz: 810626b5c05a84a382719b9bb40393ee3f901b7d3d9b8366114e1b8633e08621
 SHA512:
-  metadata.gz: f81e9efa55ee3c61d780633e8108e0b3d6f11f83cfc13c1e20a45d4002fa5431e77eaa22df2e381b44b410aa61e1ba93e2284f1bfc017a5e2870a1d460c4d60d
-  data.tar.gz: b949db4abb5d6a62c7abaed2eeb825b100a1a49d27f5f8c0dfdb821ccc9b0da170833ee63a72cefe82331c09f5de396dfc69fe3798b7a31c8bb961c9035b5ac8
+  metadata.gz: 89089861029085470ada61bb7d34bee99d33c46b3d089b56fc840c4f2881a479fab0a31a0f6ea663171d1c3bb5e7eface8cd5e92cd0c57b4769179e55c6ba610
+  data.tar.gz: 8bba97ee414b53f06222af5f6f8bad5ef89e4e2538896a8f9e0a3b5c239ef594f796a7da033843cac4e62f1207eb38591a42827147757c6adfc38f9fdb8cf3b4

data/README.md

@@ -20,9 +20,14 @@ gem install fluent-plugin-filter-kv-parser
 <filter **>
   type key_value_parser
   key log
+  use_regex true
+  remove_key false
   remove_prefix /^prefix/
   keys_delimiter /\s+/
   kv_delimiter_chart '='
+  filtered_keys akey,bkey,ckey
+  filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+  filter_out_lines_without_keys false
 </filter>
 
 <match **>

data/fluent-plugin-filter-kv-parser.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-filter-kv-parser"
-  s.version     = "0.0.1"
+  s.version     = "0.0.6"
   s.authors     = ["Al-waleed Shihadeh"]
   s.email       = ["wshihadeh.dev@gmail.com"]
   s.homepage    = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"

data/lib/fluent/plugin/filter_key_value_parser.rb

@@ -7,34 +7,72 @@ module Fluent
 
     config_param :key, :string, default: 'log'
     config_param :remove_key, :bool, default: false
+    config_param :filter_out_lines_without_keys, :bool, default: false
+    config_param :use_regex, :bool, default: false
     config_param :remove_prefix, :string, default: ''
     config_param :keys_delimiter, :string, default: '/\s+/'
     config_param :kv_delimiter_chart, :string, default: '='
+    config_param :filtered_keys, :string, default: nil
+    config_param :filtered_keys_regex, :string, default: nil
+    config_param :filtered_keys_delimiter, :string, default: ','
 
 
     def configure(conf)
       super
-      if @keys_delimiter[0] == '/' and @keys_delimiter[-1] == '/'
+
+      regex = /^\/.+\/$/
+
+      if regex.match(@keys_delimiter.to_s)
         @keys_delimiter = Regexp.new(@keys_delimiter[1..-2])
       end
 
-      if @remove_prefix[0] == '/' and @remove_prefix[-1] == '/'
+      if regex.match(@remove_prefix.to_s)
         @remove_prefix = Regexp.new(@remove_prefix[1..-2])
       end
+
+      if regex.match(@filtered_keys_regex.to_s)
+         @filtered_keys_regex = Regexp.new(@filtered_keys_regex[1..-2])
+      end
+
+      @filtered_keys_list = parse_filtered_keys_parameter
     end
 
     def filter(tag, time, record)
-      log_line = extract_log_line record[@key]
-      log_line.split(@keys_delimiter).each do |kv|
-        key, value = kv.split(@kv_delimiter_chart, 2)
-        record[key] = value
-      end
+      return if record[@key].nil?
+
+      keys = extracted_keys(extract_log_line(record[@key]))
 
-      record.tap { |r| r.delete(@key) if @remove_key }
+      return if @filter_out_lines_without_keys && keys.empty?
+
+      record.merge! keys
+      record.tap { |r| r.delete(@key) if @remove_key }.compact
     end
 
     private
 
+    def regex_filter(line)
+      "#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>([^=]+|[^ ]+))\s/).to_h
+    end
+
+    def extracted_keys(line)
+      keys = @use_regex ? regex_filter(line) : delimiter_filter(line)
+      filtered_keys = @filtered_keys_list.empty? ? keys : keys.slice(*@filtered_keys_list)
+      @filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
+    end
+
+    def delimiter_filter(line)
+      items = {}
+      line.split(@keys_delimiter).each do |kv|
+        key, value = kv.split(@kv_delimiter_chart, 2)
+        items[key] = value if value
+      end
+      items
+    end
+
+    def parse_filtered_keys_parameter
+      @filtered_keys.to_s.split(@filtered_keys_delimiter)
+    end
+
     def extract_log_line(line)
       line.gsub(@remove_prefix,'').strip
     end

data/test/test_filter_key_value_parser.rb

@@ -83,4 +83,133 @@ class KeyValueFilterTest < Test::Unit::TestCase
     assert_equal 5, filtered.count
     assert_equal false,  filtered.key?("log")
   end
+
+  test 'test_empty_line' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request',
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 1, filtered.count
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_two_log_key' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    msg2 = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    filtered = filter(d, [msg, msg2])
+    assert_equal 2, filtered.count
+  end
+
+  test 'test_no_log_key' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+    }
+    msg2 = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    filtered = filter(d, [msg, msg2])
+    assert_equal 1, filtered.count
+  end
+
+  test 'test_with_space_and_regex' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request key=10 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 6, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_filter_keys' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+        filtered_keys key,gkeyn,nkey,skey,akey,zkey
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request key=10 gkey=100 nkey=108 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 6, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_filter_keys_with_regex' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+        filtered_keys key,gkeyn,nkey,skey,akey,zkey
+        filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request sub_key=0 sub_akey=11 sub_zkey=12 key=10 gkey=100 nkey=108 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 9, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal "0", filtered['sub_key']
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_filter_out_lines_without_keys' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        use_regex true
+        filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+        filter_out_lines_without_keys true
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "akey=10 bkey=11 ckey=11",
+    }
+
+    msg2 = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request to test lines without any keys",
+    }
+    filtered = filter(d, [msg, msg2])
+
+    assert_equal 1, filtered.count
+    assert_equal 4, filtered.first[2].count
+    assert_equal "10", filtered.first[2]['akey']
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-kv-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.6
 platform: ruby
 authors:
 - Al-waleed Shihadeh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-05 00:00:00.000000000 Z
+date: 2020-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd