fluent-plugin-filter-kv-parser 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27a9342381452e0d88dac38dc1c64dbbd1b913368c6b444f0adb27c182f7a547
-  data.tar.gz: fe84db2f5478cbb4e0694c5a7e45d7889d2c88689e420080de49f862848f4949
+  metadata.gz: d03d782c5c6058dff073d09bf35fa161acb555661e6f825ff205919051f3e19a
+  data.tar.gz: b5edd6ad840a00eb31bd957b5548f01e41530d71649f68c3ae3b7f3312d60af4
 SHA512:
-  metadata.gz: 4c496463a7dbf8c59eeea45e4295ed9f18d114a2957262ea2983fabfa179ef56a7cad533bb3070292b2d72700ee2778ce23fb10ef7516ccbd4d9f37fcbd7bc4e
-  data.tar.gz: 2d6a20e24e5219fa16f3af51a3b1a1916e178b41ecef581abf57193219091545b79164bafdc805d1b41056632e382989e255e68bd5a51e283761a580190b7f81
+  metadata.gz: 49db420a06c68ce7c32dcaa3465d5d11d8fcff4ce206c95220479c6b139cd06fcbea8c0dd5e2b76db07e74855b07bc6f792384c4c29ff35e2a1504f03c303638
+  data.tar.gz: 36c16f8fb13e98ba8317062bd288a2f92ce9baf5419c6bae8ac0aa26b70328582f006424ae83dcc2ead3075b72f6d8877af7ad467993b74ed2b5045ad665a334

data/README.md

@@ -20,6 +20,8 @@ gem install fluent-plugin-filter-kv-parser
 <filter **>
   type key_value_parser
   key log
+  use_regex true
+  remove_key false
   remove_prefix /^prefix/
   keys_delimiter /\s+/
   kv_delimiter_chart '='

data/fluent-plugin-filter-kv-parser.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-filter-kv-parser"
-  s.version     = "0.0.2"
+  s.version     = "0.0.3"
   s.authors     = ["Al-waleed Shihadeh"]
   s.email       = ["wshihadeh.dev@gmail.com"]
   s.homepage    = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"

data/lib/fluent/plugin/filter_key_value_parser.rb

@@ -7,6 +7,7 @@ module Fluent
 
     config_param :key, :string, default: 'log'
     config_param :remove_key, :bool, default: false
+    config_param :use_regex, :bool, default: false
     config_param :remove_prefix, :string, default: ''
     config_param :keys_delimiter, :string, default: '/\s+/'
     config_param :kv_delimiter_chart, :string, default: '='
@@ -24,10 +25,14 @@ module Fluent
     end
 
     def filter(tag, time, record)
+      return if record[@key].nil?
+
       log_line = extract_log_line record[@key]
-      log_line.split(@keys_delimiter).each do |kv|
-        key, value = kv.split(@kv_delimiter_chart, 2)
-        record[key] = value if value
+
+      if @use_regex
+        record.merge! regex_filter(log_line)
+      else
+        record.merge! delimiter_filter(log_line)
       end
 
       record.tap { |r| r.delete(@key) if @remove_key }.compact
@@ -35,6 +40,19 @@ module Fluent
 
     private
 
+    def regex_filter(line)
+      "#{line} ".scan(/(?<key>[^ ]+)=(?<value>[^=]+)\s/).to_h
+    end
+
+    def delimiter_filter(line)
+      items = {}
+      line.split(@keys_delimiter).each do |kv|
+        key, value = kv.split(@kv_delimiter_chart, 2)
+        items[key] = value if value
+      end
+      items
+    end
+
     def extract_log_line(line)
       line.gsub(@remove_prefix,'').strip
     end

data/test/test_filter_key_value_parser.rb

@@ -83,4 +83,72 @@ class KeyValueFilterTest < Test::Unit::TestCase
     assert_equal 5, filtered.count
     assert_equal false,  filtered.key?("log")
   end
+
+  test 'test_empty_line' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request',
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal 1, filtered.count
+    assert_equal false,  filtered.key?("log")
+  end
+
+  test 'test_two_log_key' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    msg2 = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    filtered = filter(d, [msg, msg2])
+    assert_equal 2, filtered.count
+  end
+
+  test 'test_no_log_key' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+    }
+    msg2 = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    filtered = filter(d, [msg, msg2])
+    assert_equal 1, filtered.count
+  end
+
+  test 'test_with_space_and_regex' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        remove_prefix /^[^ ]+\s[^ ]+/
+        use_regex true
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request key=10 skey='this is a miltispace line' akey=20 zkey=30 dkey=4",
+    }
+    filtered = filter(d, [msg]).first[2]
+    puts filtered.inspect
+    assert_equal 6, filtered.count
+    assert_equal "'this is a miltispace line'", filtered['skey']
+    assert_equal false,  filtered.key?("log")
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-kv-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Al-waleed Shihadeh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-08 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd