fluent-plugin-kubernetes_metadata_filter 2.9.4 → 2.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98c4002df3754262c547e0b22acf366e62bca72f0142a20430f6a0f6c91dadea
-  data.tar.gz: 3f9e5d2d19b505c7cad15d113b69dea7ccb0943a3d4605d121fdf2234952c4a7
+  metadata.gz: f8ed17c405e64607d3fee4b581345f32388a7b1f52db9633d83e6863f6da5358
+  data.tar.gz: e5d6002813b1ffb266b43eda059a2f7a6b050bb69162b52a48cf68e5bf116e7d
 SHA512:
-  metadata.gz: 5474457d9a35d7951a8a380c0099deeb16af6d849a9f036c467ba47e0fcc47ce2ff7f77bbd75ad7d6487c9f172897e63406a91fd9f08dc6494eb32332e2b9373
-  data.tar.gz: 91a14c8edabcdd1eaa7b915c14bfd643b7829ae0cc8951ed8f9e77819483638207a5c6264094974971dcf4d7684c50c62a249cee4de36d23574b72cbe4e10589
+  metadata.gz: c39196e160ffbf56e328f1b4baf2ce61cafbb3152e1c1bfc109a445a78083ed4bd25f28f81356472dffb975a2905cd90ef951f33ba1777572e1b3c86810323cd
+  data.tar.gz: 99377e8dbf5b6f95259b02ea923d22c1195fb582ac89ede2734fe29200305543e6916113ee49ec7c1fe6a4a839882038925925c724d662d27b5ef34f7bdeaf9e

data/.circleci/config.yml

@@ -4,7 +4,8 @@ install: &install
   name: Install bundle
   command: |
             gem install bundler
-            bundle install --path vendor/bundle
+            bundle config set --local path vendor/bundle
+            bundle install
 
 missingdeps: &missingdeps
   name: Install missing dependecies
@@ -20,9 +21,6 @@ test: &test
   command: bundle exec rake test --trace
 
 executors:
-  ruby-2-5:
-    docker:
-      - image: circleci/ruby:2.5.5
   ruby-2-6:
     docker:
       - image: circleci/ruby:2.6.3
@@ -49,8 +47,6 @@ jobs:
 workflows:
   "test_multiple_ruby_versions":
     jobs:
-      - ruby-test:
-          ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
       - ruby-test:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.9.4)
+    fluent-plugin-kubernetes_metadata_filter (2.11.1)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -16,7 +16,7 @@ GEM
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     cool.io (1.7.1)
     copyright-header (1.0.22)
       github-linguist
@@ -26,16 +26,16 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.15.4)
+    ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.3)
+    fluentd (1.14.6)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
+      serverengine (>= 2.2.5, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -54,15 +54,15 @@ GEM
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.8.0)
-    jsonpath (1.1.0)
+    jsonpath (1.1.2)
       multi_json
-    kubeclient (4.9.2)
+    kubeclient (4.9.3)
       http (>= 3.0, < 5.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
@@ -70,10 +70,10 @@ GEM
     lru_redux (1.1.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.1115)
+    mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
     minitest (4.7.5)
-    msgpack (1.4.2)
+    msgpack (1.5.2)
     multi_json (1.15.0)
     netrc (0.11.0)
     parallel (1.21.0)
@@ -105,7 +105,7 @@ GEM
       parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     rugged (1.2.0)
-    serverengine (2.2.4)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -122,11 +122,11 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.5)
+    tzinfo-data (1.2022.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8)
+    unf_ext (0.0.8.2)
     unicode-display_width (2.1.0)
     vcr (6.0.0)
     webmock (3.14.0)
@@ -134,7 +134,7 @@ GEM
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
-    yajl-ruby (1.4.1)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -155,4 +155,4 @@ DEPENDENCIES
   yajl-ruby
 
 BUNDLED WITH
-   2.3.4
+   2.3.16

data/README.md

@@ -17,6 +17,7 @@ that rely on the authenticity of the namespace for proper log isolation.
 
 | fluent-plugin-kubernetes_metadata_filter  | fluentd | ruby |
 |-------------------|---------|------|
+| >= 2.10.0 | >= v1.10.0 | >= 2.6 |
 | >= 2.5.0 | >= v1.10.0 | >= 2.5 |
 | >= 2.0.0 | >= v0.14.20 | >= 2.1 |
 |  < 2.0.0 | >= v0.12.0 | >= 1.9 |

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,9 +5,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.9.4'
-  gem.authors       = ['Jimmi Dyson']
-  gem.email         = ['jimmidyson@gmail.com']
+  gem.version       = '2.11.1'
+  gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
+  gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
   gem.summary       = 'Fluentd filter plugin to add Kubernetes metadata'
   gem.homepage      = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -118,11 +118,21 @@ module Fluent::Plugin
       @stats.bump(:pod_cache_api_updates)
       log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
       @cache[metadata['pod_id']] = metadata
-    rescue StandardError => e
-      @stats.bump(:pod_cache_api_nil_error)
-      log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      {}
-    end
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
+        else
+          log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:pod_cache_api_nil_error)
+        end
+        {}
+      rescue StandardError => e
+        @stats.bump(:pod_cache_api_nil_error)
+        log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
+      end
 
     def dump_stats
       @curr_time = Time.now
@@ -150,15 +160,27 @@ module Fluent::Plugin
       @stats.bump(:namespace_cache_api_updates)
       log.trace("parsed metadata for #{namespace_name}: #{metadata}")
       @namespace_cache[metadata['namespace_id']] = metadata
-    rescue StandardError => e
-      @stats.bump(:namespace_cache_api_nil_error)
-      log.debug "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      {}
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
+        else
+          log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:namespace_cache_api_nil_error)
+        end
+        {}
+      rescue StandardError => e
+        @stats.bump(:namespace_cache_api_nil_error)
+        log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
     end
 
     def initialize
       super
       @prev_time = Time.now
+      @ssl_options = {}
+      @auth_options = {}
     end
 
     def configure(conf)
@@ -230,7 +252,7 @@ module Fluent::Plugin
       end
 
       if present?(@kubernetes_url)
-        ssl_options = {
+        @ssl_options = {
           client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
           client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
           ca_file: @ca_file,
@@ -249,24 +271,14 @@ module Fluent::Plugin
                       0x80000
                     end
           ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
-          ssl_options[:cert_store] = ssl_store
+          @ssl_options[:cert_store] = ssl_store
         end
 
-        auth_options = {}
-
         if present?(@bearer_token_file)
-          bearer_token = File.read(@bearer_token_file)
-          auth_options[:bearer_token] = bearer_token
+          @auth_options[:bearer_token_file] = @bearer_token_file
         end
 
-        log.debug 'Creating K8S client'
-        @client = Kubeclient::Client.new(
-          @kubernetes_url,
-          @apiVersion,
-          ssl_options: ssl_options,
-          auth_options: auth_options,
-          as: :parsed_symbolized
-        )
+        create_client()
 
         if @test_api_adapter
           log.info "Extending client with test api adaper #{@test_api_adapter}"
@@ -305,6 +317,18 @@ module Fluent::Plugin
       end
     end
 
+    def create_client()
+      log.debug 'Creating K8S client'
+      @client = nil
+      @client = Kubeclient::Client.new(
+        @kubernetes_url,
+        @apiVersion,
+        ssl_options: @ssl_options,
+        auth_options: @auth_options,
+        as: :parsed_symbolized
+      )
+    end
+
     def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
       metadata = {
         'docker' => { 'container_id' => "" },

data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb

@@ -26,7 +26,7 @@ module KubernetesMetadata
       def api_valid?
         true
       end
-      def get_namespace(namespace_name)
+      def get_namespace(namespace_name, unused, options)
         return {
           metadata: {
             name: namespace_name,
@@ -38,7 +38,7 @@ module KubernetesMetadata
         }
       end
 
-      def get_pod(pod_name, namespace_name)
+      def get_pod(pod_name, namespace_name, options)
         return {
           metadata: {
             name: pod_name,

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -46,6 +46,39 @@ module KubernetesMetadata
         @stats.bump(:namespace_watch_gone_errors)
         log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
         namespace_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          namespace_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
+          @stats.bump(:namespace_watch_failures)
+          if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              'Exception encountered parsing namespace watch event. ' \
+              'The connection might have been closed. Sleeping for ' \
+              "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
+              'seconds and resetting the namespace watcher.', e
+            )
+            sleep(Thread.current[:namespace_watch_retry_backoff_interval])
+            Thread.current[:namespace_watch_retry_count] += 1
+            Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            namespace_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              'Exception encountered parsing namespace watch event. The ' \
+              'connection might have been closed. Retried ' \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError, message
+          end
+        end
       rescue StandardError => e
         @stats.bump(:namespace_watch_failures)
         if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -47,6 +47,39 @@ module KubernetesMetadata
         @stats.bump(:pod_watch_gone_errors)
         log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
         pod_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          namespace_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
+          @stats.bump(:pod_watch_failures)
+          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Sleeping for ' \
+              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+              'seconds and resetting the pod watcher.', e
+            )
+            sleep(Thread.current[:pod_watch_retry_backoff_interval])
+            Thread.current[:pod_watch_retry_count] += 1
+            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            pod_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Retried ' \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError, message
+          end
+        end
       rescue StandardError => e
         @stats.bump(:pod_watch_failures)
         if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.9.4
+  version: 2.11.1
 platform: ruby
 authors:
+- OpenShift Cluster Logging
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-05 00:00:00.000000000 Z
+date: 2022-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -206,6 +207,7 @@ dependencies:
         version: '0'
 description: Filter plugin to add Kubernetes metadata
 email:
+- team-logging@redhat.com
 - jimmidyson@gmail.com
 executables: []
 extensions: []
@@ -271,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata