fluent-plugin-kubernetes_metadata_filter 3.3.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1425638a4efbcc9ed55bee88eb1563ba071c6d7afc9ca0b9f556a4528bfb6f05
-  data.tar.gz: 531e074b6271d05fade448979dc11cd7d2d075bd2941587cb2341fd8e3f7076f
+  metadata.gz: 57614dafa7cf3621cb5d3ade3bdfe3a72a37476d61bf2eb5ee5b371dd3174699
+  data.tar.gz: c4e298eca7e0049b747d92079d6a867d4e2d6fcce3f35dd44ed362a9eef33373
 SHA512:
-  metadata.gz: 78aca0f62942579b5c1be309857a920ee623ff71903e9d00cc6415b7b8dcc37e7a23d4be8baa024c54959963fb4c356a352e42229c08b7257bd65281bf987faf
-  data.tar.gz: 5dda7ace28a80bf02c28ac7f2dc016ad83f056e1d85c95bc6b599554d8ef9145dc4643c490776ce7deaf1f8e968caa720cf024a27236e45cd844cbd88538d661
+  metadata.gz: ddd8c41c2f3717b1c890b746bbc3cf0b80844162f79d54524565c3cac259ce0667ca73aaaeeaffc7e8ba6782e28eaeea5b7c4274b382ce564821a7cfb19c8cff
+  data.tar.gz: 05a5d2f83f666514dec72fa6946f260f48be0dec11a8a8ae2efe14028974d1ba2475b384423339a5763afa869d050767e74a504ae3c2c16064040e78e55f1d1e

data/.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 install: &install
   name: Install bundle
   command: |
-            gem install bundler
+            gem install bundler -v 2.4.22
             bundle config set --local path vendor/bundle
             bundle install
 
@@ -11,7 +11,7 @@ missingdeps: &missingdeps
   name: Install missing dependecies
   command: |
             cat /etc/os-release
-            printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
+            sed -e 's/^# deb-src/deb-src/g' /etc/apt/sources.list > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
             sudo apt-get --allow-releaseinfo-change update
             sudo apt-get install cmake libicu-dev libssl-dev
@@ -21,12 +21,9 @@ test: &test
   command: bundle exec rake test --trace
 
 executors:
-  ruby-2-6:
-    docker:
-      - image: circleci/ruby:2.6.3
   ruby-2-7:
     docker:
-      - image: circleci/ruby:2.7.1
+      - image: cimg/ruby:2.7.8
 jobs:
   "ruby-test":
     parameters:
@@ -47,7 +44,5 @@ jobs:
 workflows:
   "test_multiple_ruby_versions":
     jobs:
-      - ruby-test:
-          ruby-version: ruby-2-6
       - ruby-test:
           ruby-version: ruby-2-7

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (3.3.0)
-      fluentd (>= 0.14.0, < 1.17)
+    fluent-plugin-kubernetes_metadata_filter (3.5.0)
+      fluentd (>= 0.14.0, < 1.18)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
@@ -12,27 +12,32 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
+    base64 (0.2.0)
     bump (0.10.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.2.2)
-    cool.io (1.8.0)
+    concurrent-ruby (1.3.3)
+    cool.io (1.8.1)
     copyright-header (1.0.22)
       github-linguist
     crack (0.4.5)
       rexml
+    csv (3.3.0)
     docile (1.4.0)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
+    domain_name (0.6.20240107)
+    drb (2.2.1)
     escape_utils (1.2.2)
-    ffi (1.15.5)
-    ffi-compiler (1.0.1)
-      ffi (>= 1.0.0)
+    ffi (1.17.0)
+    ffi-compiler (1.3.2)
+      ffi (>= 1.15.5)
       rake
-    fluentd (1.16.2)
+    fluentd (1.17.0)
+      base64 (~> 0.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
+      csv (~> 3.2)
+      drb (~> 2.2)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.3.2, < 3.0.0)
@@ -48,30 +53,31 @@ GEM
       mini_mime (~> 1.0)
       rugged (~> 1.0)
     hashdiff (1.0.1)
-    http (5.1.1)
+    http (5.2.0)
       addressable (~> 2.8)
+      base64 (~> 0.1)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
-      llhttp-ffi (~> 0.4.0)
+      llhttp-ffi (~> 0.5.0)
     http-accept (1.7.0)
-    http-cookie (1.0.5)
+    http-cookie (1.0.6)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http_parser.rb (0.8.0)
-    jsonpath (1.1.3)
+    jsonpath (1.1.5)
       multi_json
     kubeclient (4.11.0)
       http (>= 3.0, < 6.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
-    llhttp-ffi (0.4.0)
+    llhttp-ffi (0.5.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
     lru_redux (1.1.0)
-    mime-types (3.5.0)
+    mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.0808)
+    mime-types-data (3.2024.0604)
     mini_mime (1.1.2)
     minitest (4.7.5)
     msgpack (1.7.2)
@@ -84,7 +90,7 @@ GEM
     public_suffix (4.0.7)
     rainbow (3.1.1)
     rake (13.0.6)
-    recursive-open-struct (1.1.3)
+    recursive-open-struct (1.2.1)
     regexp_parser (2.5.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -123,11 +129,8 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2023.3)
+    tzinfo-data (1.2024.1)
       tzinfo (>= 1.0.0)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.2)
     unicode-display_width (2.2.0)
     vcr (6.0.0)
     webmock (3.14.0)

data/README.md

@@ -64,6 +64,7 @@ when true (default: `true`)
 * `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
 * `open_timeout` - The time in seconds to wait for a connection to kubernetes service. (default: `3`)
 * `read_timeout` - The time in seconds to wait for a read from kubernetes service. (default: `10`)
+* `include_ownerrefs_metadata` - If set to true, it will include metadata (`kind` & `name`) in `kubernetes.ownerrefs` about the controller that owns the pod. (default: `false`)
 
 
 Reading from a JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '3.3.0'
+  gem.version       = '3.5.0'
   gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
   gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -15,9 +15,9 @@ Gem::Specification.new do |gem|
 
   gem.files         = `git ls-files`.split($/)
 
-  gem.required_ruby_version = '>= 2.6.0'
+  gem.required_ruby_version = '>= 2.7.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.17']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.18']
   gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -82,6 +82,7 @@ module Fluent::Plugin
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
+    config_param :include_ownerrefs_metadata, :bool, default: false
 
     # A classname in the form of Test::APIAdapter which will try
     # to be resolved from a relative named file 'test_api_adapter'

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -82,14 +82,27 @@ module KubernetesMetadata
         log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end
 
+      ownerrefs_meta = []
+      begin
+        pod_object[:metadata][:ownerReferences].each do |owner_reference|
+          ownerrefs_meta.append({
+            'kind' => owner_reference[:kind],
+            'name' => owner_reference[:name]
+          })
+        end
+      rescue StandardError => e
+        log.warn("parsing ownerrefs meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
+      end if @include_ownerrefs_metadata && pod_object[:metadata][:ownerReferences]
+
       kubernetes_metadata = {
         'namespace_name' => pod_object[:metadata][:namespace],
         'pod_id' => pod_object[:metadata][:uid],
         'pod_name' => pod_object[:metadata][:name],
         'pod_ip' => pod_object[:status][:podIP],
         'containers' => syms_to_strs(container_meta),
-        'host' => pod_object[:spec][:nodeName]
-      }
+        'host' => pod_object[:spec][:nodeName],
+        'ownerrefs' => (ownerrefs_meta if @include_ownerrefs_metadata)
+      }.compact
       kubernetes_metadata['annotations'] = annotations unless annotations.empty?
       kubernetes_metadata['labels'] = labels unless labels.empty?
       kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url

data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml

@@ -0,0 +1,156 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            },
+            "ownerReferences": [
+              {
+                "apiVersion": "apps/v1",
+                "blockOwnerDeletion": true,
+                "controller": true,
+                "kind": "ReplicaSet",
+                "name": "fabric8-console-controller",
+                "uid": "dd1ac3e1-bc88-4c54-be0f-26ac2406d884"
+              }
+            ]
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -805,6 +805,47 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+    test 'with docker & kubernetes metadata using include ownerrefs metadata' do
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
+                         { name: 'kubernetes_get_api_v1' },
+                         { name: 'kubernetes_get_pod_with_ownerrefs' },
+                         { name: 'kubernetes_get_namespace_default' }]) do
+        filtered = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          stats_interval 0
+          skip_pod_labels true
+          skip_master_url true
+          include_ownerrefs_metadata true
+        ')
+        expected_kube_metadata = {
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host' => 'jimmi-redhat.localnet',
+            'pod_name' => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'container_image' => 'fabric8/hawtio-kubernetes:latest',
+            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+            'namespace_name' => 'default',
+            "namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
+            'namespace_labels' => {
+              'tenant' => 'test'
+            },
+            'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_ip' => '172.17.0.8',
+            'ownerrefs' => [{
+              'kind' => 'ReplicaSet',
+              'name' => 'fabric8-console-controller'
+            }]
+          }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
 
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.5.0
 platform: ruby
 authors:
 - OpenShift Cluster Logging
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2024-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -20,7 +20,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '1.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '1.18'
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
@@ -259,6 +259,7 @@ files:
 - test/cassettes/kubernetes_get_pod.yml
 - test/cassettes/kubernetes_get_pod_container_init.yml
 - test/cassettes/kubernetes_get_pod_using_token.yml
+- test/cassettes/kubernetes_get_pod_with_ownerrefs.yml
 - test/cassettes/metadata_from_tag_and_journald_fields.yml
 - test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
 - test/cassettes/valid_kubernetes_api_server.yml
@@ -283,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="