fluent-plugin-kubernetes_metadata_filter 3.1.3 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e5a90f306729056c922bded6d93b4718b5741306bf2a3ed57f694ceca07316a
-  data.tar.gz: 98e5e2334f10e40123654286af7ee2841e0691969c251de9e6437430af01ae7b
+  metadata.gz: 1425638a4efbcc9ed55bee88eb1563ba071c6d7afc9ca0b9f556a4528bfb6f05
+  data.tar.gz: 531e074b6271d05fade448979dc11cd7d2d075bd2941587cb2341fd8e3f7076f
 SHA512:
-  metadata.gz: 6fd2015c041919bdde2a3a6ef817de65054b3819b684ad6ce3b216b49c0a850281b1e14ca04c2db8556d4c6d6716cb39f2d882c2c1d2ab4cba4ddc0b46dde929
-  data.tar.gz: 35f99fc6cb137c0e2714be1c515651358fd93af8bfa18e883c1f7747f04fd4f6d43cc9b4f2e8e88a62a618087f71f3c7bdd8a42ba343dba5d7a8ec2ebc1eb731
+  metadata.gz: 78aca0f62942579b5c1be309857a920ee623ff71903e9d00cc6415b7b8dcc37e7a23d4be8baa024c54959963fb4c356a352e42229c08b7257bd65281bf987faf
+  data.tar.gz: 5dda7ace28a80bf02c28ac7f2dc016ad83f056e1d85c95bc6b599554d8ef9145dc4643c490776ce7deaf1f8e968caa720cf024a27236e45cd844cbd88538d661

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (3.1.3)
-      fluentd (>= 0.14.0, < 1.16)
+    fluent-plugin-kubernetes_metadata_filter (3.3.0)
+      fluentd (>= 0.14.0, < 1.17)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
@@ -16,8 +16,8 @@ GEM
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.10)
-    cool.io (1.7.1)
+    concurrent-ruby (1.2.2)
+    cool.io (1.8.0)
     copyright-header (1.0.22)
       github-linguist
     crack (0.4.5)
@@ -30,17 +30,17 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.15.3)
+    fluentd (1.16.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.3.0, < 3.0.0)
-      sigdump (~> 0.2.2)
+      serverengine (>= 2.3.2, < 3.0.0)
+      sigdump (~> 0.2.5)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
-      webrick (>= 1.4.2, < 1.8.0)
+      webrick (~> 1.4)
       yajl-ruby (~> 1.0)
     github-linguist (7.21.0)
       charlock_holmes (~> 0.7.7)
@@ -48,32 +48,33 @@ GEM
       mini_mime (~> 1.0)
       rugged (~> 1.0)
     hashdiff (1.0.1)
-    http (4.4.1)
-      addressable (~> 2.3)
+    http (5.1.1)
+      addressable (~> 2.8)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
-      http-parser (~> 1.2.0)
+      llhttp-ffi (~> 0.4.0)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.3)
-      ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.8.0)
-    jsonpath (1.1.2)
+    jsonpath (1.1.3)
       multi_json
-    kubeclient (4.10.1)
-      http (>= 3.0, < 5.0)
+    kubeclient (4.11.0)
+      http (>= 3.0, < 6.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
+    llhttp-ffi (0.4.0)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
     lru_redux (1.1.0)
-    mime-types (3.4.1)
+    mime-types (3.5.0)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
+    mime-types-data (3.2023.0808)
     mini_mime (1.1.2)
     minitest (4.7.5)
-    msgpack (1.6.0)
+    msgpack (1.7.2)
     multi_json (1.15.0)
     netrc (0.11.0)
     parallel (1.22.1)
@@ -105,9 +106,9 @@ GEM
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     rugged (1.4.3)
-    serverengine (2.3.0)
+    serverengine (2.3.2)
       sigdump (~> 0.2.2)
-    sigdump (0.2.4)
+    sigdump (0.2.5)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -120,9 +121,9 @@ GEM
     test-unit-rr (1.0.5)
       rr (>= 1.1.1)
       test-unit (>= 2.5.2)
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2022.6)
+    tzinfo-data (1.2023.3)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
@@ -133,7 +134,7 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.7.0)
+    webrick (1.8.1)
     yajl-ruby (1.4.3)
 
 PLATFORMS

data/README.md

@@ -55,6 +55,8 @@ when true (default: `true`)
   to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
   the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`)
 * `skip_labels` - Skip all label fields from the metadata.
+* `skip_pod_labels` - Skip only pod label fields from the metadata.
+* `skip_namespace_labels` - Skip only namespace label fields from the metadata.
 * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
 * `skip_master_url` - Skip the master_url field from the metadata.
 * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '3.1.3'
+  gem.version       = '3.3.0'
   gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
   gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.6.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.16']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.17']
   gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -77,6 +77,8 @@ module Fluent::Plugin
     # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
     config_param :ssl_partial_chain, :bool, default: false
     config_param :skip_labels, :bool, default: false
+    config_param :skip_pod_labels, :bool, default: false
+    config_param :skip_namespace_labels, :bool, default: false
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
@@ -198,7 +200,7 @@ module Fluent::Plugin
       @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
 
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
-      
+
       # Use Kubernetes default service account if we're in a pod.
       if @kubernetes_url.nil?
         log.debug 'Kubernetes URL is not set - inspecting environ'
@@ -351,7 +353,7 @@ module Fluent::Plugin
           tag_match_data['pod_uuid']
         else
           tag_match_data['docker_id']
-        end 
+        end
         docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
         metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
                                                 cache_key, time, batch_miss_cache, docker_id)

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -40,7 +40,7 @@ module KubernetesMetadata
 
     def parse_namespace_metadata(namespace_object)
       labels = ''
-      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
+      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless (@skip_labels || @skip_namespace_labels)
 
       annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
 
@@ -55,7 +55,7 @@ module KubernetesMetadata
 
     def parse_pod_metadata(pod_object)
       labels = ''
-      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
+      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless (@skip_labels || @skip_pod_labels)
 
       annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
 

data/test/cassettes/kubernetes_get_namespace_default.yml

@@ -53,7 +53,10 @@ http_interactions:
             "selfLink": "/api/v1/namespaces/default",
             "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
             "resourceVersion": "6",
-            "creationTimestamp": "2015-05-08T09:22:01Z"
+            "creationTimestamp": "2015-05-08T09:22:01Z",
+            "labels": {
+              "tenant": "test"
+            }
           },
           "spec": {
             "finalizers": [

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -49,7 +49,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         d.instance.dump_stats
         assert_false(d.instance.instance_variable_get("@curr_time").nil?)
       end
-      
+
       test 'disables stats when <= zero' do
         d = create_driver('stats_interval 0')
         assert_equal(0, d.instance.stats_interval)
@@ -199,6 +199,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'pod_name' => 'fabric8-console-controller-98rqc',
               'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
               'namespace_name' => 'default',
+              'namespace_labels' => {
+                'tenant' => 'test'
+              },
               'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
               'pod_ip' => '172.17.0.8',
               'master_url' => 'https://localhost:8443',
@@ -231,6 +234,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'pod_name' => 'fabric8-console-controller-98rqc',
               'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
               'namespace_name' => 'default',
+              'namespace_labels' => {
+                'tenant' => 'test'
+              },
               'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
               'pod_ip' => '172.17.0.8',
               'master_url' => 'https://localhost:8443',
@@ -330,6 +336,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -360,6 +369,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -410,6 +422,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -585,8 +600,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'host' => 'jimmi-redhat.localnet',
             'pod_name' => 'fabric8-console-controller-98rqc',
             'container_name' => 'fabric8-console-container',
-            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_name' => 'default',
+            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'container_image' => 'fabric8/hawtio-kubernetes:latest',
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
@@ -628,8 +643,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_name' => 'fabric8-console-container',
             'container_image' => 'fabric8/hawtio-kubernetes:latest',
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_name' => 'default',
+            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -643,7 +658,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
 
     test 'processes all events when reading from MessagePackEventStream' do
-      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, 
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
                          { name: 'kubernetes_get_api_v1' },
                          { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default' }]) do
@@ -675,6 +690,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -718,5 +736,75 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+
+    test 'with docker & kubernetes metadata using skip namespace labels config param' do
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
+                         { name: 'kubernetes_get_namespace_default' }]) do
+        filtered = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          stats_interval 0
+          skip_namespace_labels true
+          skip_master_url true
+        ')
+        expected_kube_metadata = {
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host' => 'jimmi-redhat.localnet',
+            'pod_name' => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'container_image' => 'fabric8/hawtio-kubernetes:latest',
+            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+            'namespace_name' => 'default',
+            "namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
+            'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_ip' => '172.17.0.8',
+            'labels' => {
+              'component' => 'fabric8Console'
+            }
+          }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
+    test 'with docker & kubernetes metadata using skip pod labels config param' do
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
+                         { name: 'kubernetes_get_namespace_default' }]) do
+        filtered = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          stats_interval 0
+          skip_pod_labels true
+          skip_master_url true
+        ')
+        expected_kube_metadata = {
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host' => 'jimmi-redhat.localnet',
+            'pod_name' => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'container_image' => 'fabric8/hawtio-kubernetes:latest',
+            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+            'namespace_name' => 'default',
+            "namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
+            'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_ip' => '172.17.0.8',
+          }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.3.0
 platform: ruby
 authors:
 - OpenShift Cluster Logging
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-28 00:00:00.000000000 Z
+date: 2023-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -20,7 +20,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
@@ -290,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata