fluent-plugin-kubernetes_metadata_filter 2.8.1 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71fd60517a92c89e8c35bc91a3147487ff0637ae41378ecf82b047424cdb203f
-  data.tar.gz: 310b9a7bbb7e5bd59d1aa3deedb699114f561295b4b3a97fdf54fadcf6d7ee08
+  metadata.gz: 1637f0d5dc3b0962bea8bb25f8cda869044b45fd33d27fee1ef7a58d2c403f4c
+  data.tar.gz: 0bf235def6a0eab1ff766617b704bd3b637403ad88d568c8d087352473a24dc0
 SHA512:
-  metadata.gz: 76ccb8d8b2a2d26ca1e2823a93c38fef974e76526f402f11a10c8ea4697cbda765370eb202177490e9a4aadb10fef4f9980ba4e38cb39953c55ae3bf49ac3bf4
-  data.tar.gz: 8ea9696e6df02bf264a7f0e68ab36732adae9a007379e6bbd841633b6ad6dd830d9241a30bdcd0744183151898fdcc71711be14656973a47d8afa7a1f273abb1
+  metadata.gz: 5f31264c9339108f83573e9566ec2f93630afa7c64d4f00a996431925e487a5c74575603183072c10803defac132ef066ef0ec021ad37446e358cef4cad0a05e
+  data.tar.gz: ca81025a46a85d4e87feededf0f9ef2c8047cd3a8b84b75a71baf16feabc364ca69ce4ed69f3f796dbd1542bc602799dac53f87bbb9e7df980a99e4872c7955f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.8.1)
+    fluent-plugin-kubernetes_metadata_filter (2.9.0)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux

data/README.md

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.8.1'
+  gem.version       = '2.9.0'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -58,6 +58,8 @@ module Fluent::Plugin
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    config_param :de_slash, :bool, default: false
+    config_param :de_slash_separator, :string, default: '__'
     # if reading from the journal, the record will contain the following fields in the following
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
@@ -166,6 +168,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
+      if @de_slash && @de_slash_separator.include?('/')
+        raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
+      end
+
       if @cache_ttl < 0
         log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
@@ -381,6 +387,16 @@ module Fluent::Plugin
       end
     end
 
+    def de_slash!(h)
+      h.keys.each do |ref|
+        next unless h[ref] && ref =~ /\//
+
+        v = h.delete(ref)
+        newref = ref.to_s.gsub('/', @de_slash_separator)
+        h[newref] = v
+      end
+    end
+
     # copied from activesupport
     def present?(object)
       object.respond_to?(:empty?) ? !object.empty? : !!object

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -47,6 +47,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       kubernetes_metadata = {
         'namespace_id' => namespace_object[:metadata][:uid],
         'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
@@ -65,6 +69,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
 
       # collect container information
       container_meta = {}

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -432,13 +432,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
 
-    test 'with kubernetes dotted labels, de_dot enabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
+          de_dot true
+          de_slash true
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -452,14 +454,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_labels' => {
-              'kubernetes_io/namespacetest' => 'somevalue'
+              'kubernetes_io__namespacetest' => 'somevalue'
             },
             'namespace_name' => 'default',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
             'labels' => {
-              'kubernetes_io/test' => 'somevalue'
+              'kubernetes_io__test' => 'somevalue'
             }
           }
         }
@@ -467,14 +469,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
 
-    test 'with kubernetes dotted labels, de_dot disabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           de_dot false
+          de_slash false
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -506,11 +509,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid de_dot_separator config' do
       assert_raise Fluent::ConfigError do
         create_driver('
+          de_dot true
           de_dot_separator contains.
         ')
       end
     end
 
+    test 'invalid de_slash_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_slash true
+          de_slash_separator contains/
+        ')
+      end
+    end
+
     test 'with records from journald and docker & kubernetes metadata' do
       # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
       tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.9.0
 platform: ruby
 authors:
 - Jimmi Dyson
@@ -230,7 +230,7 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
 - test/cassettes/kubernetes_get_api_v1.yml
 - test/cassettes/kubernetes_get_api_v1_using_token.yml
 - test/cassettes/kubernetes_get_namespace_default.yml