fluent-plugin-kubernetes_metadata_filter 2.1.6 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 4a83bf6d7f3f2f80e380d89a9aa78024c706a4312b971d8d53ffe79a67bdb322
-  data.tar.gz: 241adc577d1c9eb2bdb97ef2a7d7ef6511cbbe836e15b91301f04657e4345feb
+SHA1:
+  metadata.gz: 5d86f8aaca3c3fffa77b39ea1bf73d5176ee518e
+  data.tar.gz: 681f0b65a2e7ff7b73bac23dc7e9f02fcb2f11de
 SHA512:
-  metadata.gz: 2e3923244e262befe237d2f35cc8f54788558e22ff53cec53bd4edd044931ea35682bee853eda5775254df44026cfdedd16bda247aee2b2f443b1cb73e853514
-  data.tar.gz: bda740f2a37e0a2510277b84c32e469360b8652327d47b994b464942ab17e246b6eda6f16ad43c4b65b806b8928188886524684ae37199abf9cd58edab84445b
+  metadata.gz: 2f0f86f42ec6e6276f532d58d64c656608cae4290c26dd2f22c3365ff7787cc830b8ec60bacb418acf4e8c1fff7cb911dc972ccf55cf15e7ee49433b87117b9b
+  data.tar.gz: 48e27fc22bc59fdc929a5d9c8bbf833e77aaf1d5ad9d90d32bf1e451fe53adf01809a5f76bb3c2854b355064ce386830f30073b1b9340823419cfce851341fd0

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2.1
+
+install: &install
+  name: Install bundle
+  command: bundle install --path vendor/bundle
+
+missingdeps: &missingdeps
+  name: Install missing dependecies
+  command: |
+            cat /etc/os-release
+            printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /tmp/sources.list
+            sudo cp /tmp/sources.list /etc/apt/sources.list
+            sudo apt-get update
+            sudo apt-get install cmake libicu-dev libssl-dev
+  
+test: &test
+  name: Test bundle
+  command: bundle exec rake test
+
+executors:
+  ruby-2-4:
+    docker:
+      - image: circleci/ruby:2.4.6
+  ruby-2-5:
+    docker:
+      - image: circleci/ruby:2.5.5
+  ruby-2-6:
+    docker:
+      - image: circleci/ruby:2.6.3
+jobs:
+  "ruby-test":
+    parameters:
+      ruby-version:
+        type: executor
+    executor: << parameters.ruby-version >>
+    working_directory: ~/fluent-plugin-kubernetes_metadata_filter
+    steps:
+      - run: *missingdeps
+      - checkout
+      - run: *install
+      - run: *test
+      - store_test_results:
+            path: coverage
+      - store_artifacts:
+            path: coverage
+
+workflows:
+  "test_multiple_ruby_versions":
+    jobs:
+      - ruby-test:
+          ruby-version: ruby-2-4
+      - ruby-test:
+          ruby-version: ruby-2-5
+      - ruby-test:
+          ruby-version: ruby-2-6
+      

data/README.md

@@ -57,7 +57,10 @@ when true (default: `true`)
 * `ssl_partial_chain` - if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
   to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
   the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`)
-
+* `skip_labels` - Skip all label fields from the metadata.
+* `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
+* `skip_master_url` - Skip the master_url field from the metadata.
+* `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less. 
 **NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
 payload.  The following configuration options are removed:
 

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.1.6"
+  gem.version       = "2.2.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -16,7 +16,6 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.has_rdoc      = false
 
   gem.required_ruby_version = '>= 2.1.0'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -76,6 +76,10 @@ module Fluent::Plugin
     # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
     # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
     config_param :ssl_partial_chain, :bool, default: false
+    config_param :skip_labels, :bool, default: false
+    config_param :skip_container_metadata, :bool, default: false
+    config_param :skip_master_url, :bool, default: false
+    config_param :skip_namespace_metadata, :bool, default: false
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
@@ -291,7 +295,7 @@ module Fluent::Plugin
       if @kubernetes_url.present?
         pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
 
-        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id)
+        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
           metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
           metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
         end

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -31,7 +31,7 @@ module KubernetesMetadata
         end
         metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
           @stats.bump(:namespace_cache_miss)
-          m = fetch_namespace_metadata(namespace_name)
+          m = fetch_namespace_metadata(namespace_name) unless @skip_namespace_metadata
           (m.nil? || m.empty?) ?  {'namespace_id'=>ids[:namespace_id]} : m
         end)
       else
@@ -39,6 +39,11 @@ module KubernetesMetadata
         @stats.bump(:id_cache_miss)
         return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
         pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
+        if @skip_namespace_metadata
+          ids = { :pod_id=> pod_metadata['pod_id'] }
+          @id_cache[key] = ids
+          return pod_metadata
+        end
         namespace_metadata = fetch_namespace_metadata(namespace_name)
         ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
         if !ids[:pod_id].nil? && !ids[:namespace_id].nil?

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -32,10 +32,12 @@ module KubernetesMetadata
     end
 
     def parse_namespace_metadata(namespace_object)
-      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h)
+      labels = String.new
+      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h) unless @skip_labels
+
       annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
       if @de_dot
-        self.de_dot!(labels)
+        self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
       kubernetes_metadata = {
@@ -48,10 +50,12 @@ module KubernetesMetadata
     end
 
     def parse_pod_metadata(pod_object)
-      labels = syms_to_strs(pod_object['metadata']['labels'].to_h)
+      labels = String.new
+      labels = syms_to_strs(pod_object['metadata']['labels'].to_h) unless @skip_labels
+
       annotations = match_annotations(syms_to_strs(pod_object['metadata']['annotations'].to_h))
       if @de_dot
-        self.de_dot!(labels)
+        self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
 
@@ -61,11 +65,17 @@ module KubernetesMetadata
         pod_object['status']['containerStatuses'].each do|container_status|
           # get plain container id (eg. docker://hash -> hash)
           container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
-          container_meta[container_id] = {
-              'name' => container_status['name'],
-              'image' => container_status['image'],
-              'image_id' => container_status['imageID']
-          }
+          unless @skip_container_metadata
+            container_meta[container_id] = {
+                'name' => container_status['name'],
+                'image' => container_status['image'],
+                'image_id' => container_status['imageID']
+            }
+          else
+            container_meta[container_id] = {
+                'name' => container_status['name']
+            }
+          end
         end
       rescue
         log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
@@ -76,11 +86,11 @@ module KubernetesMetadata
           'pod_id'         => pod_object['metadata']['uid'],
           'pod_name'       => pod_object['metadata']['name'],
           'containers'     => syms_to_strs(container_meta),
-          'labels'         => labels,
-          'host'           => pod_object['spec']['nodeName'],
-          'master_url'     => @kubernetes_url
+          'host'           => pod_object['spec']['nodeName']
       }
       kubernetes_metadata['annotations'] = annotations unless annotations.empty?
+      kubernetes_metadata['labels'] = labels unless labels.empty?
+      kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
       return kubernetes_metadata
     end
 

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -938,5 +938,33 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[1])
       end
     end
+
+    test 'with docker & kubernetes metadata using skip config params' do
+      VCR.use_cassette('kubernetes_docker_metadata') do
+        filtered = emit({},'
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          skip_labels true
+          skip_container_metadata true
+          skip_master_url true
+          skip_namespace_metadata true
+        ')
+        expected_kube_metadata = {
+            'docker' => {
+                'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+                'host'               => 'jimmi-redhat.localnet',
+                'pod_name'           => 'fabric8-console-controller-98rqc',
+                'container_name'     => 'fabric8-console-container',
+                'namespace_name'     => 'default',
+                'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d'
+            }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.2.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-12 00:00:00.000000000 Z
+date: 2019-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -205,12 +205,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- circle.yml
 - fluent-plugin-kubernetes_metadata_filter.gemspec
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
 - lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb
@@ -256,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata

data/circle.yml

@@ -1,17 +0,0 @@
-machine:
-  ruby:
-    version: 2.3.1
-
-dependencies:
-  override:
-    - 'rvm-exec 2.1.9 bundle install'
-    - 'rvm-exec 2.2.5 bundle install'
-    - 'rvm-exec 2.3.1 bundle install'
-    - 'rvm-exec 2.4.3 bundle install'
-
-test:
-  override:
-    - 'rvm-exec 2.1.9 bundle exec rake test'
-    - 'rvm-exec 2.2.5 bundle exec rake test'
-    - 'rvm-exec 2.3.1 bundle exec rake test'
-    - 'rvm-exec 2.4.3 bundle exec rake test'