fluent-plugin-kubernetes_metadata_filter 2.7.2 → 2.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3815fdb2ea8a0bbe7c2bccdb63edc62c92edd96f8c7b31a5bade805b738a5b09
-  data.tar.gz: bb92cd1dad2eedb6e7e45013e7089a2a321ba0e64817a5b67b40a9e2bb2e7f3c
+  metadata.gz: db4f0f5b097c04e8f958729fcd3946a08cff0629d38aa0844c062c511a318cf6
+  data.tar.gz: c828b3a3979a151d6fdabc73f0dd576c3e0e599e7fb9bb47ec4fe194daa63efa
 SHA512:
-  metadata.gz: be02a0549e1b0bc829bb737e88926c77e74e3b416bf1f5b826df8947bad65001e2fc6f199f7da216e75fc7287fd653ba344cebf1ab652ffd7185806ed61c93bc
-  data.tar.gz: e0d470d9ed7d97b518958d2b9999a179079c77a1ddc680c07e2c201a83790de5bc73b7abda17946a3a17652d6eff97fe6b904007470d78b209b36823b952853f
+  metadata.gz: 93aa9bfb3f313a035f5e00d8e82d8c898714058e6e591eb2b42a417f4c2ad58e8b4058d77b9a8943172231e936404d5d4cd5239f30e57f39eb629fd36bf973ca
+  data.tar.gz: a2615917cede625dda017f0b722999d3771dc8ce78f54b685d557884ad8bc13909899b34d8264e6f1d0a0b028f611c5635832f28f8198536837317f40064c157

data/.circleci/config.yml

@@ -12,7 +12,7 @@ missingdeps: &missingdeps
             cat /etc/os-release
             printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
-            sudo apt-get update
+            sudo apt-get --allow-releaseinfo-change update
             sudo apt-get install cmake libicu-dev libssl-dev
 
 test: &test

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.7.2)
-      fluentd (>= 0.14.0, < 1.14)
-      kubeclient (< 5)
+    fluent-plugin-kubernetes_metadata_filter (2.9.1)
+      fluentd (>= 0.14.0, < 1.15)
+      kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
 GEM
@@ -26,11 +26,11 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.15.3)
+    ffi (1.15.4)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.13.2)
+    fluentd (1.14.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.8.0)
@@ -70,7 +70,7 @@ GEM
     lru_redux (1.1.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0704)
+    mime-types-data (3.2021.0901)
     mini_mime (1.0.2)
     minitest (4.7.5)
     msgpack (1.4.2)
@@ -122,11 +122,11 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
+    tzinfo-data (1.2021.3)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
     unicode-display_width (2.0.0)
     vcr (6.0.0)
     webmock (3.11.1)

data/README.md

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.7.2'
+  gem.version       = '2.9.1'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.5.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.14']
-  gem.add_runtime_dependency 'kubeclient', '< 5'
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
+  gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 
   gem.add_development_dependency 'bump'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -58,6 +58,8 @@ module Fluent::Plugin
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    config_param :de_slash, :bool, default: false
+    config_param :de_slash_separator, :string, default: '__'
     # if reading from the journal, the record will contain the following fields in the following
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
@@ -166,6 +168,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
+      if @de_slash && @de_slash_separator.include?('/')
+        raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
+      end
+
       if @cache_ttl < 0
         log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
@@ -381,6 +387,16 @@ module Fluent::Plugin
       end
     end
 
+    def de_slash!(h)
+      h.keys.each do |ref|
+        next unless h[ref] && ref =~ /\//
+
+        v = h.delete(ref)
+        newref = ref.to_s.gsub('/', @de_slash_separator)
+        h[newref] = v
+      end
+    end
+
     # copied from activesupport
     def present?(object)
       object.respond_to?(:empty?) ? !object.empty? : !!object

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -47,6 +47,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       kubernetes_metadata = {
         'namespace_id' => namespace_object[:metadata][:uid],
         'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
@@ -65,13 +69,17 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
 
       # collect container information
       container_meta = {}
       begin
         pod_object[:status][:containerStatuses].each do |container_status|
           # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status[:containerID].sub(%r{^[-_a-zA-Z0-9]+://}, '')
+          container_id = container_status[:containerID] ? container_status[:containerID].sub(%r{^[-_a-zA-Z0-9]+://}, '') : container_status[:name]
           container_meta[container_id] = if @skip_container_metadata
                                            {
                                              'name' => container_status[:name]
@@ -83,9 +91,9 @@ module KubernetesMetadata
                                              'image_id' => container_status[:imageID]
                                            }
                                          end
-        end
-      rescue StandardError
-        log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ")
+        end if pod_object[:status] && pod_object[:status][:containerStatuses]
+      rescue StandardError=>e
+        log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end
 
       kubernetes_metadata = {

data/test/cassettes/kubernetes_get_pod_container_init.yml

@@ -0,0 +1,145 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "waiting": {
+                    "reason": "ContainerCreating"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": ""
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -165,6 +165,43 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       plugin.filter_stream('tag', Fluent::MultiEventStream.new)
     end
 
+    sub_test_case 'parsing_pod_metadata' do
+      test 'when container_status is missing from the pod status' do
+        VCR.use_cassettes(
+          [
+            { name: 'valid_kubernetes_api_server' },
+            { name: 'kubernetes_get_api_v1' },
+            { name: 'kubernetes_get_namespace_default' },
+            { name: 'kubernetes_get_pod_container_init' }
+          ]) do
+          filtered = emit({}, '
+            kubernetes_url https://localhost:8443
+            watch false
+            cache_size 1
+          ')
+          expected_kube_metadata = {
+            'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+              'container_name'=>'fabric8-console-container',
+              'host' => 'jimmi-redhat.localnet',
+              'pod_name' => 'fabric8-console-controller-98rqc',
+              'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+              'namespace_name' => 'default',
+              'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+              'pod_ip' => '172.17.0.8',
+              'master_url' => 'https://localhost:8443',
+              'labels' => {
+                'component' => 'fabric8Console'
+              }
+            }
+          }
+          assert_equal(expected_kube_metadata, filtered[0])
+        end
+      end
+    end
+
     test 'inability to connect to the api server handles exception and doensnt block pipeline' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }]) do
         driver = create_driver('
@@ -432,13 +469,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
 
-    test 'with kubernetes dotted labels, de_dot enabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
+          de_dot true
+          de_slash true
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -452,14 +491,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_labels' => {
-              'kubernetes_io/namespacetest' => 'somevalue'
+              'kubernetes_io__namespacetest' => 'somevalue'
             },
             'namespace_name' => 'default',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
             'labels' => {
-              'kubernetes_io/test' => 'somevalue'
+              'kubernetes_io__test' => 'somevalue'
             }
           }
         }
@@ -467,14 +506,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
 
-    test 'with kubernetes dotted labels, de_dot disabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           de_dot false
+          de_slash false
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -506,11 +546,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid de_dot_separator config' do
       assert_raise Fluent::ConfigError do
         create_driver('
+          de_dot true
           de_dot_separator contains.
         ')
       end
     end
 
+    test 'invalid de_slash_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_slash true
+          de_slash_separator contains/
+        ')
+      end
+    end
+
     test 'with records from journald and docker & kubernetes metadata' do
       # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
       tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'

data/test/plugin/watch_test.rb

@@ -69,6 +69,8 @@ class WatchTest < Test::Unit::TestCase
 
     def logger.error(message, error)
     end
+    def logger.warn(message)
+    end
     logger
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.9.1
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-20 00:00:00.000000000 Z
+date: 2021-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -19,7 +19,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,21 +29,27 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
@@ -224,12 +230,13 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
 - test/cassettes/kubernetes_get_api_v1.yml
 - test/cassettes/kubernetes_get_api_v1_using_token.yml
 - test/cassettes/kubernetes_get_namespace_default.yml
 - test/cassettes/kubernetes_get_namespace_default_using_token.yml
 - test/cassettes/kubernetes_get_pod.yml
+- test/cassettes/kubernetes_get_pod_container_init.yml
 - test/cassettes/kubernetes_get_pod_using_token.yml
 - test/cassettes/metadata_from_tag_and_journald_fields.yml
 - test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
@@ -263,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata