fluent-plugin-filter-kv-parser 0.0.4 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06f855fd104485ee6469bda9fd116eb9a5a0e04d186297b246d21ee37f82f931
-  data.tar.gz: 14f5e141dff49e437dcb3d9dd047869057d1a9c2ca5de02b695fbfb06d60b107
+  metadata.gz: a03fd43ca0868a32f5f9b2fb708b0dbbeba01213f7cb6c14cba269dbcb0b4b35
+  data.tar.gz: 3370ceff8ef70d6ae8084beb43019f222212a179614a235c0b4fec9a278116d5
 SHA512:
-  metadata.gz: a93b2f140f1f0914dbefa08d19d7f9f50033ff4c09e6fe20ca61c01281b27aae32e28051fd87f56723989e3b5ed03b5df3de976a3163deb4355959d121b4e12e
-  data.tar.gz: c7adfcf6a29ff9016495800e3e95c9cb2b23baa9233ce9a621adf63960849e83df20235753e1214f4e813d085af0bf5afd7a958032cd8ba3191ed56157a5bbe6
+  metadata.gz: be070361f7952a1b4bfd04b02a95f3a756cf66ed258a6462784ace866e0ded8ee7d53ced926f26a72fc7498884f37137931f8395416d9c66cfa41bf68f5079e5
+  data.tar.gz: a426868603e91dc578921ec8750912cc83cd8d6063477353a842c838a16c0276487d4ce8b001e2c9937969d77aab1a20f1ba29c183e015adf24e181d4d9d75ee

data/README.md

@@ -24,9 +24,11 @@ gem install fluent-plugin-filter-kv-parser
   remove_key false
   remove_prefix /^prefix/
   keys_delimiter /\s+/
-  kv_delimiter_chart '='
+  kv_delimiter_char '='
   filtered_keys akey,bkey,ckey
   filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+  filter_out_lines_without_keys false
+  keys_prefix prefix
 </filter>
 
 <match **>

data/fluent-plugin-filter-kv-parser.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-filter-kv-parser"
-  s.version     = "0.0.4"
+  s.version     = "0.0.9"
   s.authors     = ["Al-waleed Shihadeh"]
   s.email       = ["wshihadeh.dev@gmail.com"]
   s.homepage    = "https://github.com/wshihadeh/fluent-plugin-filter-kv-parser.git"

data/lib/fluent/plugin/filter_key_value_parser.rb

@@ -7,13 +7,15 @@ module Fluent
 
     config_param :key, :string, default: 'log'
     config_param :remove_key, :bool, default: false
+    config_param :filter_out_lines_without_keys, :bool, default: false
     config_param :use_regex, :bool, default: false
     config_param :remove_prefix, :string, default: ''
     config_param :keys_delimiter, :string, default: '/\s+/'
-    config_param :kv_delimiter_chart, :string, default: '='
+    config_param :kv_delimiter_char, :string, default: '='
     config_param :filtered_keys, :string, default: nil
     config_param :filtered_keys_regex, :string, default: nil
     config_param :filtered_keys_delimiter, :string, default: ','
+    config_param :keys_prefix, :string, default: nil
 
 
     def configure(conf)
@@ -39,28 +41,31 @@ module Fluent
     def filter(tag, time, record)
       return if record[@key].nil?
 
-      log_line = extract_log_line record[@key]
+      keys = extracted_keys(extract_log_line(record[@key]))
 
-      record.merge! extracted_keys(log_line)
+      return if @filter_out_lines_without_keys && keys.empty?
+
+      record.merge! keys
       record.tap { |r| r.delete(@key) if @remove_key }.compact
     end
 
     private
 
     def regex_filter(line)
-      "#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>[^=]+)\s/).to_h
+      "#{line} ".scan(/(?<key>[a-zA-Z_0-9]+)=(?<value>([^=]+|[^ ]+))\s/).to_h
     end
 
     def extracted_keys(line)
       keys = @use_regex ? regex_filter(line) : delimiter_filter(line)
       filtered_keys = @filtered_keys_list.empty? ? keys : keys.slice(*@filtered_keys_list)
-      @filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
+      filtered_keys = @filtered_keys_regex.nil? ? filtered_keys : filtered_keys.merge(keys.select{ |k,v| @filtered_keys_regex.match(k.to_s)})
+      @keys_prefix.nil? ? filtered_keys : filtered_keys.transform_keys!{ |key| "#{@keys_prefix}.#{key}" }
     end
 
     def delimiter_filter(line)
       items = {}
       line.split(@keys_delimiter).each do |kv|
-        key, value = kv.split(@kv_delimiter_chart, 2)
+        key, value = kv.split(@kv_delimiter_char, 2)
         items[key] = value if value
       end
       items

data/test/test_filter_key_value_parser.rb

@@ -188,4 +188,42 @@ class KeyValueFilterTest < Test::Unit::TestCase
     assert_equal "0", filtered['sub_key']
     assert_equal false,  filtered.key?("log")
   end
+
+  test 'test_filter_out_lines_without_keys' do
+    d = create_driver(%[
+        key log
+        remove_key true
+        use_regex true
+        filtered_keys_regex /^sub_[a-zA-Z_0-9]+/
+        filter_out_lines_without_keys true
+      ])
+    msg = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "akey=10 bkey=11 ckey=11",
+    }
+
+    msg2 = {
+      'time' => '2013-02-12 22:01:15 UTC',
+      'log'  => "Start Request to test lines without any keys",
+    }
+    filtered = filter(d, [msg, msg2])
+
+    assert_equal 1, filtered.count
+    assert_equal 4, filtered.first[2].count
+    assert_equal "10", filtered.first[2]['akey']
+  end
+
+  test 'test_keys_prefix' do
+    d = create_driver(%[
+        key log
+        keys_prefix test
+      ])
+    msg = {
+      'time'      => '2013-02-12 22:01:15 UTC',
+      'log'      => 'Start Request key=10 akey=20 zkey=30 dkey=40',
+    }
+    filtered = filter(d, [msg]).first[2]
+    assert_equal false,  filtered.key?("key")
+    assert_equal true,  filtered.key?("test.key")
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-filter-kv-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.9
 platform: ruby
 authors:
 - Al-waleed Shihadeh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2022-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A Fluentd filter plugin to parse key value items