fluent-plugin-kubernetes_metadata_filter 2.7.1 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2e43f94ecc34a05001fc55388edb83d428cd2f8f74f4b81fe9b851f42657968
-  data.tar.gz: 199b8d7382a9de36b2150fbeb0d78b0ff3b6e5f03d0eb1bc52dc778f9d7bb3f0
+  metadata.gz: 1637f0d5dc3b0962bea8bb25f8cda869044b45fd33d27fee1ef7a58d2c403f4c
+  data.tar.gz: 0bf235def6a0eab1ff766617b704bd3b637403ad88d568c8d087352473a24dc0
 SHA512:
-  metadata.gz: 8ae1a9211df6278a8a54041d7db08d50bc1946f9ad3a63499e2e46facf95f4637591f4518418bf13aab67cae88a773caec0400e8264b02333576f1fd52f42231
-  data.tar.gz: 007a4a1701bfeb551fcc1efef903a4c4cbc62fbca94eca67ed9295bcd883191dbf3bdaff4d15ec5b11f3d7e6c3fdb14a21cd857f20338bb021e9b2cef4a6b930
+  metadata.gz: 5f31264c9339108f83573e9566ec2f93630afa7c64d4f00a996431925e487a5c74575603183072c10803defac132ef066ef0ec021ad37446e358cef4cad0a05e
+  data.tar.gz: ca81025a46a85d4e87feededf0f9ef2c8047cd3a8b84b75a71baf16feabc364ca69ce4ed69f3f796dbd1542bc602799dac53f87bbb9e7df980a99e4872c7955f

data/.circleci/config.yml

@@ -12,7 +12,7 @@ missingdeps: &missingdeps
             cat /etc/os-release
             printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
-            sudo apt-get update
+            sudo apt-get --allow-releaseinfo-change update
             sudo apt-get install cmake libicu-dev libssl-dev
 
 test: &test

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.7.1)
-      fluentd (>= 0.14.0, < 1.14)
-      kubeclient (< 5)
+    fluent-plugin-kubernetes_metadata_filter (2.9.0)
+      fluentd (>= 0.14.0, < 1.15)
+      kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.1)
     bump (0.10.0)
@@ -26,14 +26,14 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.15.1)
+    ffi (1.15.4)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.13.0)
+    fluentd (1.14.0)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
@@ -59,7 +59,7 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.6.0)
+    http_parser.rb (0.7.0)
     jsonpath (1.1.0)
       multi_json
     kubeclient (4.9.2)
@@ -70,7 +70,7 @@ GEM
     lru_redux (1.1.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0225)
+    mime-types-data (3.2021.0901)
     mini_mime (1.0.2)
     minitest (4.7.5)
     msgpack (1.4.2)
@@ -126,7 +126,7 @@ GEM
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
     unicode-display_width (2.0.0)
     vcr (6.0.0)
     webmock (3.11.1)

data/README.md

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.7.1'
+  gem.version       = '2.9.0'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.5.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.14']
-  gem.add_runtime_dependency 'kubeclient', '< 5'
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
+  gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 
   gem.add_development_dependency 'bump'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -58,6 +58,8 @@ module Fluent::Plugin
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    config_param :de_slash, :bool, default: false
+    config_param :de_slash_separator, :string, default: '__'
     # if reading from the journal, the record will contain the following fields in the following
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
@@ -166,6 +168,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
+      if @de_slash && @de_slash_separator.include?('/')
+        raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
+      end
+
       if @cache_ttl < 0
         log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
@@ -381,6 +387,16 @@ module Fluent::Plugin
       end
     end
 
+    def de_slash!(h)
+      h.keys.each do |ref|
+        next unless h[ref] && ref =~ /\//
+
+        v = h.delete(ref)
+        newref = ref.to_s.gsub('/', @de_slash_separator)
+        h[newref] = v
+      end
+    end
+
     # copied from activesupport
     def present?(object)
       object.respond_to?(:empty?) ? !object.empty? : !!object

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -47,6 +47,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       kubernetes_metadata = {
         'namespace_id' => namespace_object[:metadata][:uid],
         'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
@@ -65,6 +69,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
 
       # collect container information
       container_meta = {}
@@ -84,8 +92,8 @@ module KubernetesMetadata
                                            }
                                          end
         end
-      rescue StandardError
-        log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ")
+      rescue StandardError=>e
+        log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end
 
       kubernetes_metadata = {

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -432,13 +432,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
 
-    test 'with kubernetes dotted labels, de_dot enabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
+          de_dot true
+          de_slash true
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -452,14 +454,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_labels' => {
-              'kubernetes_io/namespacetest' => 'somevalue'
+              'kubernetes_io__namespacetest' => 'somevalue'
             },
             'namespace_name' => 'default',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
             'labels' => {
-              'kubernetes_io/test' => 'somevalue'
+              'kubernetes_io__test' => 'somevalue'
             }
           }
         }
@@ -467,14 +469,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
 
-    test 'with kubernetes dotted labels, de_dot disabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           de_dot false
+          de_slash false
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -506,11 +509,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid de_dot_separator config' do
       assert_raise Fluent::ConfigError do
         create_driver('
+          de_dot true
           de_dot_separator contains.
         ')
       end
     end
 
+    test 'invalid de_slash_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_slash true
+          de_slash_separator contains/
+        ')
+      end
+    end
+
     test 'with records from journald and docker & kubernetes metadata' do
       # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
       tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'

data/test/plugin/watch_test.rb

@@ -69,6 +69,8 @@ class WatchTest < Test::Unit::TestCase
 
     def logger.error(message, error)
     end
+    def logger.warn(message)
+    end
     logger
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.7.1
+  version: 2.9.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-09 00:00:00.000000000 Z
+date: 2021-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -19,7 +19,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,21 +29,27 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
@@ -224,7 +230,7 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
 - test/cassettes/kubernetes_get_api_v1.yml
 - test/cassettes/kubernetes_get_api_v1_using_token.yml
 - test/cassettes/kubernetes_get_namespace_default.yml
@@ -263,7 +269,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata