fluent-plugin-kubernetes_metadata_filter 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b386e5a9f06d9356809efffdf54c368b21a6cdda6a8b1687ce9e60b2352e38c7
-  data.tar.gz: a9cd4f1169c0318ee418943614ba6c80fd8ae99837605d0597e4964ff3dc1dfb
+  metadata.gz: 1425638a4efbcc9ed55bee88eb1563ba071c6d7afc9ca0b9f556a4528bfb6f05
+  data.tar.gz: 531e074b6271d05fade448979dc11cd7d2d075bd2941587cb2341fd8e3f7076f
 SHA512:
-  metadata.gz: 144b68bc69c03c9d4812b2ff6897dcc5b3f56152e396ff1b2d75217369bb5cf7c94882767b25793a84604e62f62dd98f212d79e03ca7c52942e21d03623b441f
-  data.tar.gz: 2bddc63b3576adfe0a6d0052f8727b3b88635c47dec586f6e5fbaf9785b498564d67205b4c4b0247023ebfe50fa0efec0ad3de6dd6b803aff60a66ae7bdc56ff
+  metadata.gz: 78aca0f62942579b5c1be309857a920ee623ff71903e9d00cc6415b7b8dcc37e7a23d4be8baa024c54959963fb4c356a352e42229c08b7257bd65281bf987faf
+  data.tar.gz: 5dda7ace28a80bf02c28ac7f2dc016ad83f056e1d85c95bc6b599554d8ef9145dc4643c490776ce7deaf1f8e968caa720cf024a27236e45cd844cbd88538d661

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (3.2.0)
+    fluent-plugin-kubernetes_metadata_filter (3.3.0)
       fluentd (>= 0.14.0, < 1.17)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -17,7 +17,7 @@ GEM
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     concurrent-ruby (1.2.2)
-    cool.io (1.7.1)
+    cool.io (1.8.0)
     copyright-header (1.0.22)
       github-linguist
     crack (0.4.5)
@@ -30,13 +30,13 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.16.0)
+    fluentd (1.16.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.3.2, < 3.0.0)
-      sigdump (~> 0.2.2)
+      sigdump (~> 0.2.5)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
@@ -58,7 +58,7 @@ GEM
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http_parser.rb (0.8.0)
-    jsonpath (1.1.2)
+    jsonpath (1.1.3)
       multi_json
     kubeclient (4.11.0)
       http (>= 3.0, < 6.0)
@@ -69,12 +69,12 @@ GEM
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
     lru_redux (1.1.0)
-    mime-types (3.4.1)
+    mime-types (3.5.0)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.0218.1)
+    mime-types-data (3.2023.0808)
     mini_mime (1.1.2)
     minitest (4.7.5)
-    msgpack (1.7.0)
+    msgpack (1.7.2)
     multi_json (1.15.0)
     netrc (0.11.0)
     parallel (1.22.1)
@@ -108,7 +108,7 @@ GEM
     rugged (1.4.3)
     serverengine (2.3.2)
       sigdump (~> 0.2.2)
-    sigdump (0.2.4)
+    sigdump (0.2.5)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/README.md

@@ -55,6 +55,8 @@ when true (default: `true`)
   to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
   the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`)
 * `skip_labels` - Skip all label fields from the metadata.
+* `skip_pod_labels` - Skip only pod label fields from the metadata.
+* `skip_namespace_labels` - Skip only namespace label fields from the metadata.
 * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
 * `skip_master_url` - Skip the master_url field from the metadata.
 * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '3.2.0'
+  gem.version       = '3.3.0'
   gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
   gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -77,6 +77,8 @@ module Fluent::Plugin
     # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
     config_param :ssl_partial_chain, :bool, default: false
     config_param :skip_labels, :bool, default: false
+    config_param :skip_pod_labels, :bool, default: false
+    config_param :skip_namespace_labels, :bool, default: false
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
@@ -198,7 +200,7 @@ module Fluent::Plugin
       @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
 
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
-      
+
       # Use Kubernetes default service account if we're in a pod.
       if @kubernetes_url.nil?
         log.debug 'Kubernetes URL is not set - inspecting environ'
@@ -351,7 +353,7 @@ module Fluent::Plugin
           tag_match_data['pod_uuid']
         else
           tag_match_data['docker_id']
-        end 
+        end
         docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
         metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
                                                 cache_key, time, batch_miss_cache, docker_id)

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -40,7 +40,7 @@ module KubernetesMetadata
 
     def parse_namespace_metadata(namespace_object)
       labels = ''
-      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
+      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless (@skip_labels || @skip_namespace_labels)
 
       annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
 
@@ -55,7 +55,7 @@ module KubernetesMetadata
 
     def parse_pod_metadata(pod_object)
       labels = ''
-      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
+      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless (@skip_labels || @skip_pod_labels)
 
       annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
 

data/test/cassettes/kubernetes_get_namespace_default.yml

@@ -53,7 +53,10 @@ http_interactions:
             "selfLink": "/api/v1/namespaces/default",
             "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
             "resourceVersion": "6",
-            "creationTimestamp": "2015-05-08T09:22:01Z"
+            "creationTimestamp": "2015-05-08T09:22:01Z",
+            "labels": {
+              "tenant": "test"
+            }
           },
           "spec": {
             "finalizers": [

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -49,7 +49,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         d.instance.dump_stats
         assert_false(d.instance.instance_variable_get("@curr_time").nil?)
       end
-      
+
       test 'disables stats when <= zero' do
         d = create_driver('stats_interval 0')
         assert_equal(0, d.instance.stats_interval)
@@ -199,6 +199,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'pod_name' => 'fabric8-console-controller-98rqc',
               'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
               'namespace_name' => 'default',
+              'namespace_labels' => {
+                'tenant' => 'test'
+              },
               'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
               'pod_ip' => '172.17.0.8',
               'master_url' => 'https://localhost:8443',
@@ -231,6 +234,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'pod_name' => 'fabric8-console-controller-98rqc',
               'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
               'namespace_name' => 'default',
+              'namespace_labels' => {
+                'tenant' => 'test'
+              },
               'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
               'pod_ip' => '172.17.0.8',
               'master_url' => 'https://localhost:8443',
@@ -330,6 +336,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -360,6 +369,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -410,6 +422,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -585,8 +600,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'host' => 'jimmi-redhat.localnet',
             'pod_name' => 'fabric8-console-controller-98rqc',
             'container_name' => 'fabric8-console-container',
-            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_name' => 'default',
+            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'container_image' => 'fabric8/hawtio-kubernetes:latest',
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
@@ -628,8 +643,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_name' => 'fabric8-console-container',
             'container_image' => 'fabric8/hawtio-kubernetes:latest',
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_name' => 'default',
+            'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -643,7 +658,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
 
     test 'processes all events when reading from MessagePackEventStream' do
-      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, 
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
                          { name: 'kubernetes_get_api_v1' },
                          { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default' }]) do
@@ -675,6 +690,9 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_name' => 'default',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
@@ -718,5 +736,75 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+
+    test 'with docker & kubernetes metadata using skip namespace labels config param' do
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
+                         { name: 'kubernetes_get_namespace_default' }]) do
+        filtered = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          stats_interval 0
+          skip_namespace_labels true
+          skip_master_url true
+        ')
+        expected_kube_metadata = {
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host' => 'jimmi-redhat.localnet',
+            'pod_name' => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'container_image' => 'fabric8/hawtio-kubernetes:latest',
+            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+            'namespace_name' => 'default',
+            "namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
+            'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_ip' => '172.17.0.8',
+            'labels' => {
+              'component' => 'fabric8Console'
+            }
+          }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
+    test 'with docker & kubernetes metadata using skip pod labels config param' do
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
+                         { name: 'kubernetes_get_namespace_default' }]) do
+        filtered = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          stats_interval 0
+          skip_pod_labels true
+          skip_master_url true
+        ')
+        expected_kube_metadata = {
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host' => 'jimmi-redhat.localnet',
+            'pod_name' => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'container_image' => 'fabric8/hawtio-kubernetes:latest',
+            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+            'namespace_name' => 'default',
+            "namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
+            'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_ip' => '172.17.0.8',
+          }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - OpenShift Cluster Logging
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-13 00:00:00.000000000 Z
+date: 2023-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd