fluent-plugin-kubernetes_metadata_filter 2.10.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8dcf9979d36d0b08161765e73d3ae244efba378ac3dd0d4d10f582be6abcc488
-  data.tar.gz: 9ea29129e636535052a9183119b61647ae419773e8174ca86adb9a9ebbbdebd8
+  metadata.gz: c9c8ca7a91fa8c0fa0d87086c446978ed54afd901a635889cc9c0c6276258588
+  data.tar.gz: a67192dc10a2ceda476cad7ee35e28973670603bd86aa6c083d47c7cbebf9b85
 SHA512:
-  metadata.gz: 1dc73fe9ef457fdd9bf424fe935ac99060df59d7c67273dc2c1320ae891cb70c98483c427613a675603557a7bc675d48029d4eefa69eb29b8c2b1f9536cccad2
-  data.tar.gz: fb160f1bc82037cc0ed3ddd7fae884aeab816c8d246256a40d7e5831b5ff64ee93512955709830028180017df4889421c9920f9364d9282029e8ceda5fe000c6
+  metadata.gz: 4143d6376a80ba53f0cf1b9699d23d4862b9f748e927585694dbd942edec3d214989c59a1cfd90bd7b02379ee5beeea3a01a9d7ec5947752e8e1e1becb2e7bd5
+  data.tar.gz: 12b6b13c20272ca6d285856a816b52ddffa92ac53b93a91222adb8e9f0be07d1a21808d61c2d4364473e0be57b1f4af6414627cca9a08ce88143e02b2f4bc7b1

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.10.0)
-      fluentd (>= 0.14.0, < 1.15)
+    fluent-plugin-kubernetes_metadata_filter (2.13.0)
+      fluentd (>= 0.14.0, < 1.16)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
@@ -25,28 +25,28 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    escape_utils (1.2.1)
+    escape_utils (1.2.2)
     ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.6)
+    fluentd (1.15.0)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.17.0)
+    github-linguist (7.21.0)
       charlock_holmes (~> 0.7.7)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
-      rugged (>= 0.25.1)
+      rugged (~> 1.0)
     hashdiff (1.0.1)
     http (4.4.1)
       addressable (~> 2.3)
@@ -54,13 +54,13 @@ GEM
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.8.0)
-    jsonpath (1.1.0)
+    jsonpath (1.1.2)
       multi_json
     kubeclient (4.9.3)
       http (>= 3.0, < 5.0)
@@ -73,39 +73,39 @@ GEM
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
     minitest (4.7.5)
-    msgpack (1.5.1)
+    msgpack (1.5.3)
     multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.21.0)
-    parser (3.0.2.0)
+    parallel (1.22.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
     power_assert (2.0.1)
-    public_suffix (4.0.6)
-    rainbow (3.0.0)
+    public_suffix (4.0.7)
+    rainbow (3.1.1)
     rake (13.0.6)
     recursive-open-struct (1.1.3)
-    regexp_parser (2.1.1)
+    regexp_parser (2.5.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rr (3.0.8)
-    rubocop (1.22.3)
+    rr (3.0.9)
+    rubocop (1.28.2)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.12.0, < 2.0)
+      rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.12.0)
-      parser (>= 3.0.1.1)
+    rubocop-ast (1.17.0)
+      parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
-    rugged (1.2.0)
-    serverengine (2.2.5)
+    rugged (1.4.3)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -113,7 +113,7 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
+    simplecov_json_formatter (0.1.4)
     strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
@@ -126,15 +126,15 @@ GEM
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.1)
-    unicode-display_width (2.1.0)
+    unf_ext (0.0.8.2)
+    unicode-display_width (2.2.0)
     vcr (6.0.0)
     webmock (3.14.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
-    yajl-ruby (1.4.2)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -155,4 +155,4 @@ DEPENDENCIES
   yajl-ruby
 
 BUNDLED WITH
-   2.3.4
+   2.3.16

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.10.0'
+  gem.version       = '2.13.0'
   gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
   gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -15,9 +15,9 @@ Gem::Specification.new do |gem|
 
   gem.files         = `git ls-files`.split($/)
 
-  gem.required_ruby_version = '>= 2.5.0'
+  gem.required_ruby_version = '>= 2.6.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.16']
   gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -118,11 +118,21 @@ module Fluent::Plugin
       @stats.bump(:pod_cache_api_updates)
       log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
       @cache[metadata['pod_id']] = metadata
-    rescue StandardError => e
-      @stats.bump(:pod_cache_api_nil_error)
-      log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      {}
-    end
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
+        else
+          log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:pod_cache_api_nil_error)
+        end
+        {}
+      rescue StandardError => e
+        @stats.bump(:pod_cache_api_nil_error)
+        log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
+      end
 
     def dump_stats
       @curr_time = Time.now
@@ -150,15 +160,27 @@ module Fluent::Plugin
       @stats.bump(:namespace_cache_api_updates)
       log.trace("parsed metadata for #{namespace_name}: #{metadata}")
       @namespace_cache[metadata['namespace_id']] = metadata
-    rescue StandardError => e
-      @stats.bump(:namespace_cache_api_nil_error)
-      log.debug "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      {}
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
+        else
+          log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:namespace_cache_api_nil_error)
+        end
+        {}
+      rescue StandardError => e
+        @stats.bump(:namespace_cache_api_nil_error)
+        log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
     end
 
     def initialize
       super
       @prev_time = Time.now
+      @ssl_options = {}
+      @auth_options = {}
     end
 
     def configure(conf)
@@ -230,7 +252,7 @@ module Fluent::Plugin
       end
 
       if present?(@kubernetes_url)
-        ssl_options = {
+        @ssl_options = {
           client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
           client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
           ca_file: @ca_file,
@@ -249,24 +271,14 @@ module Fluent::Plugin
                       0x80000
                     end
           ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
-          ssl_options[:cert_store] = ssl_store
+          @ssl_options[:cert_store] = ssl_store
         end
 
-        auth_options = {}
-
         if present?(@bearer_token_file)
-          bearer_token = File.read(@bearer_token_file)
-          auth_options[:bearer_token] = bearer_token
+          @auth_options[:bearer_token_file] = @bearer_token_file
         end
 
-        log.debug 'Creating K8S client'
-        @client = Kubeclient::Client.new(
-          @kubernetes_url,
-          @apiVersion,
-          ssl_options: ssl_options,
-          auth_options: auth_options,
-          as: :parsed_symbolized
-        )
+        create_client()
 
         if @test_api_adapter
           log.info "Extending client with test api adaper #{@test_api_adapter}"
@@ -305,6 +317,18 @@ module Fluent::Plugin
       end
     end
 
+    def create_client()
+      log.debug 'Creating K8S client'
+      @client = nil
+      @client = Kubeclient::Client.new(
+        @kubernetes_url,
+        @apiVersion,
+        ssl_options: @ssl_options,
+        auth_options: @auth_options,
+        as: :parsed_symbolized
+      )
+    end
+
     def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
       metadata = {
         'docker' => { 'container_id' => "" },
@@ -331,44 +355,36 @@ module Fluent::Plugin
       metadata
     end
 
-    def filter_stream(tag, es)
-      return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
-      new_es = Fluent::MultiEventStream.new
+    def filter(tag, time, record)
       tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
-      tag_metadata = nil
       batch_miss_cache = {}
-      es.each do |time, record|
-        if tag_match_data && tag_metadata.nil?
-          cache_key =  if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
-            tag_match_data['pod_uuid']
-          else
-            tag_match_data['docker_id']
-          end 
-          docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
-          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-                                                 cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
-        end
-        metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
-        if (@use_journal || @use_journal.nil?) &&
-           (j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
-          metadata = j_metadata
-        end
-        if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
-           record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
-           record['kubernetes'].key?('namespace_name') &&
-           record['kubernetes'].key?('pod_name') &&
-           record['kubernetes'].key?('container_name') &&
-           record['docker'].key?('container_id') &&
-           (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
-                                                 record['kubernetes']['container_name'], record['docker']['container_id'],
-                                                 create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
-          metadata = k_metadata
-        end
-        record = record.merge(metadata) if metadata
-        new_es.add(time, record)
+      if tag_match_data
+        cache_key =  if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
+          tag_match_data['pod_uuid']
+        else
+          tag_match_data['docker_id']
+        end 
+        docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
+        tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
+                                                cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
+      end
+      metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
+      if (@use_journal || @use_journal.nil?) &&
+          (j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
+        metadata = j_metadata
+      end
+      if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
+          record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
+          record['kubernetes'].key?('namespace_name') &&
+          record['kubernetes'].key?('pod_name') &&
+          record['kubernetes'].key?('container_name') &&
+          record['docker'].key?('container_id') &&
+          (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
+                                                record['kubernetes']['container_name'], record['docker']['container_id'],
+                                                create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
+        metadata = k_metadata
       end
-      dump_stats
-      new_es
+      metadata ? record.merge(metadata) : record
     end
 
     def get_metadata_for_journal_record(record, time, batch_miss_cache)

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -46,6 +46,39 @@ module KubernetesMetadata
         @stats.bump(:namespace_watch_gone_errors)
         log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
         namespace_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          namespace_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
+          @stats.bump(:namespace_watch_failures)
+          if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              'Exception encountered parsing namespace watch event. ' \
+              'The connection might have been closed. Sleeping for ' \
+              "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
+              'seconds and resetting the namespace watcher.', e
+            )
+            sleep(Thread.current[:namespace_watch_retry_backoff_interval])
+            Thread.current[:namespace_watch_retry_count] += 1
+            Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            namespace_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              'Exception encountered parsing namespace watch event. The ' \
+              'connection might have been closed. Retried ' \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError, message
+          end
+        end
       rescue StandardError => e
         @stats.bump(:namespace_watch_failures)
         if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -47,6 +47,39 @@ module KubernetesMetadata
         @stats.bump(:pod_watch_gone_errors)
         log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
         pod_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          namespace_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
+          @stats.bump(:pod_watch_failures)
+          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Sleeping for ' \
+              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+              'seconds and resetting the pod watcher.', e
+            )
+            sleep(Thread.current[:pod_watch_retry_backoff_interval])
+            Thread.current[:pod_watch_retry_count] += 1
+            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            pod_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Retried ' \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError, message
+          end
+        end
       rescue StandardError => e
         @stats.bump(:pod_watch_failures)
         if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -156,15 +156,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       d.filtered.map(&:last)
     end
 
-    test 'nil event stream' do
-      # not certain how this is possible but adding test to properly
-      # guard against this condition we have seen - test for nil,
-      # empty, no empty method, not an event stream
-      plugin = create_driver.instance
-      plugin.filter_stream('tag', nil)
-      plugin.filter_stream('tag', Fluent::MultiEventStream.new)
-    end
-
     sub_test_case 'parsing_pod_metadata when container_status is missing from the pod status' do
       test 'using the tag_to_kubernetes_name_regexp for /var/log/containers ' do
         VCR.use_cassettes(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.10.0
+  version: 2.13.0
 platform: ruby
 authors:
 - OpenShift Cluster Logging
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-12 00:00:00.000000000 Z
+date: 2022-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -20,7 +20,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
@@ -266,14 +266,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata