fluent-plugin-kubernetes_metadata_filter 2.4.1 → 2.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 636af222c31e7e4db80d96594f76d3f4918f981120e9a4006e414ddd6b94df15
-  data.tar.gz: a4edbb9c153da167210ee0fc879d38e37428839ed640264ce81ad824e7991004
+  metadata.gz: 15b2d6f98c168e196ed582fda2cdc02775e66725c7f827c0b2e8aec361fea719
+  data.tar.gz: 3cef1e6ce88263b9202ffc897ddf833dd4056cc4ebbe73e211901957e71d08b2
 SHA512:
-  metadata.gz: a7b667937578e95e259752e9ad64391774103d1ad2ea7f569259ee6f0d6addfbb0e0960fe95379300052d8863342bc6cb0235f95af8f0346a03f6dba3dffffb2
-  data.tar.gz: b8e0a412669d5fe57ce3c9cb8ed641e75c0c67d42305cdf88b91838621e781d172a19e079a4e312117e3e1dfd411f5c91c92a0e1626ed6a9c152e0633f64bcb2
+  metadata.gz: 9a44d475049124488353e2e87dd6124eb05eff10d0f3f5dc2234f947379a5050bf7240e0b371b0e825097bacce38e056a0873f135b4cbae216c5a391fbce2bc4
+  data.tar.gz: ec40671937051616e5a57d89397d67a5dcc85cc6040ac53871602eee9b7947d9d0ab8337800ac51ca73aec1086ff8157e311edf299e502bac96e764cc5185549

data/.circleci/config.yml

@@ -2,13 +2,15 @@ version: 2.1
 
 install: &install
   name: Install bundle
-  command: bundle install --path vendor/bundle
+  command: |
+            gem install bundler
+            bundle install --path vendor/bundle
 
 missingdeps: &missingdeps
   name: Install missing dependecies
   command: |
             cat /etc/os-release
-            printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /tmp/sources.list
+            printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
             sudo apt-get update
             sudo apt-get install cmake libicu-dev libssl-dev
@@ -53,4 +55,3 @@ workflows:
           ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
-      

data/README.md

@@ -61,6 +61,7 @@ when true (default: `true`)
 * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
 * `skip_master_url` - Skip the master_url field from the metadata.
 * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
+* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
 
 **NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
 payload.  The following configuration options are removed:
@@ -80,16 +81,36 @@ then the plugin will parse those values using `container_name_to_kubernetes_rege
 - Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
 lookup the metdata
 
-Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
+Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
+
+```
+fluent-gem install fluent-plugin-multi-format-parser
+```
+
+The config block could look like this:
 ```
 <source>
   @type tail
   path /var/log/containers/*.log
   pos_file fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  tag kubernetes.*
+  <parse>
+    @type multi_format
+    <pattern>
+      format json
+      time_key time
+      time_type string
+      time_format "%Y-%m-%dT%H:%M:%S.%NZ"
+      keep_time_key false
+    </pattern>
+    <pattern>
+      format regexp
+      expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
+      time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
+      keep_time_key false
+    </pattern>
+  </parse>
 </source>
 
 <filter kubernetes.var.log.containers.**.log>
@@ -128,6 +149,22 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
   @type stdout
 </match>
 ```
+## Log content as JSON
+In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```
 
 ## Environment variables for Kubernetes
 

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.4.1"
+  gem.version       = "2.4.2"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "lru_redux"
   gem.add_runtime_dependency "kubeclient", '< 5'
 
-  gem.add_development_dependency "bundler", "~> 2.0.2"
+  gem.add_development_dependency "bundler", "~> 2.0"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "minitest", "~> 4.0"
   gem.add_development_dependency "test-unit", "~> 3.0.2"

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -22,6 +22,8 @@ require_relative 'kubernetes_metadata_common'
 require_relative 'kubernetes_metadata_stats'
 require_relative 'kubernetes_metadata_watch_namespaces'
 require_relative 'kubernetes_metadata_watch_pods'
+
+require 'fluent/plugin_helper/thread'
 require 'fluent/plugin/filter'
 require 'resolv'
 
@@ -37,6 +39,8 @@ module Fluent::Plugin
 
     Fluent::Plugin.register_filter('kubernetes_metadata', self)
 
+    helpers :thread
+
     config_param :kubernetes_url, :string, default: nil
     config_param :cache_size, :integer, default: 1000
     config_param :cache_ttl, :integer, default: 60 * 60
@@ -80,6 +84,12 @@ module Fluent::Plugin
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
+    # The time interval in seconds for retry backoffs when watch connections fail.
+    config_param :watch_retry_interval, :bool, default: 1
+    # The base number of exponential backoff for retries.
+    config_param :watch_retry_exponential_backoff_base, :bool, default: 2
+    # The maximum number of times to retry pod and namespace watches.
+    config_param :watch_retry_max_times, :bool, default: 10
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
@@ -264,9 +274,14 @@ module Fluent::Plugin
         end
 
         if @watch
-          thread = Thread.new(self) { |this| this.start_pod_watch }
-          thread.abort_on_exception = true
-          namespace_thread = Thread.new(self) { |this| this.start_namespace_watch }
+          pod_thread = thread_create :"pod_watch_thread" do
+            set_up_pod_thread
+          end
+          pod_thread.abort_on_exception = true
+
+          namespace_thread = thread_create :"namespace_watch_thread" do
+            set_up_namespace_thread
+          end
           namespace_thread.abort_on_exception = true
         end
       end

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -23,16 +23,81 @@ module KubernetesMetadata
 
     include ::KubernetesMetadata::Common
 
+    def set_up_namespace_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      namespace_watcher = start_namespace_watch
+      Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:namespace_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      while thread_current_running?
+        begin
+          namespace_watcher ||= get_namespaces_and_start_watcher
+          process_namespace_watcher_notices(namespace_watcher)
+        rescue Exception => e
+          @stats.bump(:namespace_watch_failures)
+          if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              "Exception encountered parsing namespace watch event. " \
+              "The connection might have been closed. Sleeping for " \
+              "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
+              "seconds and resetting the namespace watcher.", e)
+            sleep(Thread.current[:namespace_watch_retry_backoff_interval])
+            Thread.current[:namespace_watch_retry_count] += 1
+            Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            namespace_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              "Exception encountered parsing namespace watch event. The " \
+              "connection might have been closed. Retried " \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError.new(message)
+          end
+        end
+      end
+    end
+
     def start_namespace_watch
-      begin
-        resource_version = @client.get_namespaces.resourceVersion
-        watcher          = @client.watch_namespaces(resource_version)
-      rescue Exception=>e
-        message = "start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
-        log.debug(message)
-        raise Fluent::ConfigError, message
+      return get_namespaces_and_start_watcher
+    rescue Exception => e
+      message = "start_namespace_watch: Exception encountered setting up " \
+                "namespace watch from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all namespaces, record the resourceVersion and return a watcher
+    # starting from that resourceVersion.
+    def get_namespaces_and_start_watcher
+      options = {
+        resource_version: '0'  # Fetch from API server.
+      }
+      namespaces = @client.get_namespaces(options)
+      namespaces.each do |namespace|
+        cache_key = namespace.metadata['uid']
+        @namespace_cache[cache_key] = parse_namespace_metadata(namespace)
+        @stats.bump(:namespace_cache_host_updates)
       end
+      options[:resource_version] = namespaces.resourceVersion
+      watcher = @client.watch_namespaces(options)
+      watcher
+    end
+
+    # Process a watcher notice and potentially raise an exception.
+    def process_namespace_watcher_notices(watcher)
       watcher.each do |notice|
         case notice.type
           when 'MODIFIED'
@@ -45,16 +110,15 @@ module KubernetesMetadata
               @stats.bump(:namespace_cache_watch_misses)
             end
           when 'DELETED'
-            # ignore and let age out for cases where 
+            # ignore and let age out for cases where
             # deleted but still processing logs
             @stats.bump(:namespace_cache_watch_deletes_ignored)
           else
             # Don't pay attention to creations, since the created namespace may not
-            # be used by any pod on this node.
+            # be used by any namespace on this node.
             @stats.bump(:namespace_cache_watch_ignored)
         end
       end
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -23,29 +23,84 @@ module KubernetesMetadata
 
     include ::KubernetesMetadata::Common
 
-    def start_pod_watch
-      begin
-        options = {
-          resource_version: '0'  # Fetch from API server.
-        }
-        if ENV['K8S_NODE_NAME']
-          options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
-        end
-        pods = @client.get_pods(options)
-        pods.each do |pod|
-          cache_key = pod.metadata['uid']
-          @cache[cache_key] = parse_pod_metadata(pod)
-          @stats.bump(:pod_cache_host_updates)
+    def set_up_pod_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      pod_watcher = start_pod_watch
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:pod_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      while thread_current_running?
+        begin
+          pod_watcher ||= get_pods_and_start_watcher
+          process_pod_watcher_notices(pod_watcher)
+        rescue Exception => e
+          @stats.bump(:pod_watch_failures)
+          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              "Exception encountered parsing pod watch event. The " \
+              "connection might have been closed. Sleeping for " \
+              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+              "seconds and resetting the pod watcher.", e)
+            sleep(Thread.current[:pod_watch_retry_backoff_interval])
+            Thread.current[:pod_watch_retry_count] += 1
+            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            pod_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              "Exception encountered parsing pod watch event. The " \
+              "connection might have been closed. Retried " \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError.new(message)
+          end
         end
-        options[:resource_version] = pods.resourceVersion
-        watcher = @client.watch_pods(options)
-      rescue Exception => e
-        message = "Exception encountered fetching metadata from Kubernetes API endpoint: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
+      end
+    end
 
-        raise Fluent::ConfigError, message
+    def start_pod_watch
+      get_pods_and_start_watcher
+    rescue Exception => e
+      message = "start_pod_watch: Exception encountered setting up pod watch " \
+                "from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all pods, record the resourceVersion and return a watcher starting
+    # from that resourceVersion.
+    def get_pods_and_start_watcher
+      options = {
+        resource_version: '0'  # Fetch from API server.
+      }
+      if ENV['K8S_NODE_NAME']
+        options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
+      end
+      pods = @client.get_pods(options)
+      pods.each do |pod|
+        cache_key = pod.metadata['uid']
+        @cache[cache_key] = parse_pod_metadata(pod)
+        @stats.bump(:pod_cache_host_updates)
       end
+      options[:resource_version] = pods.resourceVersion
+      watcher = @client.watch_pods(options)
+      watcher
+    end
 
+    # Process a watcher notice and potentially raise an exception.
+    def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
         case notice.type
           when 'MODIFIED'

data/test/plugin/test_watch_namespaces.rb

@@ -25,6 +25,24 @@ class WatchNamespacesTestTest < WatchTest
      include KubernetesMetadata::WatchNamespaces
 
      setup do
+       @initial = Kubeclient::Common::EntityList.new(
+         'NamespaceList',
+         '123',
+         [
+           Kubeclient::Resource.new({
+                                      'metadata' => {
+                                        'name' => 'initial',
+                                        'uid' => 'initial_uid'
+                                      }
+                                    }),
+           Kubeclient::Resource.new({
+                                      'metadata' => {
+                                        'name' => 'modified',
+                                        'uid' => 'modified_uid'
+                                      }
+                                    })
+         ])
+
        @created = OpenStruct.new(
          type: 'CREATED',
          object: {
@@ -54,9 +72,31 @@ class WatchNamespacesTestTest < WatchTest
        )
      end
 
+    test 'namespace list caches namespaces' do
+      @client.stub :get_namespaces, @initial do
+        process_namespace_watcher_notices(start_namespace_watch)
+        assert_equal(true, @namespace_cache.key?('initial_uid'))
+        assert_equal(true, @namespace_cache.key?('modified_uid'))
+        assert_equal(2, @stats[:namespace_cache_host_updates])
+      end
+    end
+
+    test 'namespace list caches namespaces and watch updates' do
+      orig_env_val = ENV['K8S_NODE_NAME']
+      ENV['K8S_NODE_NAME'] = 'aNodeName'
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [@modified] do
+          process_namespace_watcher_notices(start_namespace_watch)
+          assert_equal(2, @stats[:namespace_cache_host_updates])
+          assert_equal(1, @stats[:namespace_cache_watch_updates])
+        end
+      end
+      ENV['K8S_NODE_NAME'] = orig_env_val
+    end
+
     test 'namespace watch ignores CREATED' do
       @client.stub :watch_namespaces, [@created] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(false, @namespace_cache.key?('created_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_ignored])
       end
@@ -64,7 +104,7 @@ class WatchNamespacesTestTest < WatchTest
 
     test 'namespace watch ignores MODIFIED when info not in cache' do
       @client.stub :watch_namespaces, [@modified] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(false, @namespace_cache.key?('modified_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_misses])
       end
@@ -73,7 +113,7 @@ class WatchNamespacesTestTest < WatchTest
     test 'namespace watch updates cache when MODIFIED is received and info is cached' do
       @namespace_cache['modified_uid'] = {}
       @client.stub :watch_namespaces, [@modified] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(true, @namespace_cache.key?('modified_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_updates])
       end
@@ -82,10 +122,22 @@ class WatchNamespacesTestTest < WatchTest
     test 'namespace watch ignores DELETED' do
       @namespace_cache['deleted_uid'] = {}
       @client.stub :watch_namespaces, [@deleted] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(true, @namespace_cache.key?('deleted_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
       end
     end
 
+    test 'namespace watch retries when exceptions are encountered' do
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [[@created, @exception_raised]] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_namespace_thread
+          end
+          assert_equal(3, @stats[:namespace_watch_failures])
+          assert_equal(2, Thread.current[:namespace_watch_retry_count])
+          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
+        end
+      end
+    end
 end

data/test/plugin/test_watch_pods.rb

@@ -142,7 +142,7 @@ class DefaultPodWatchStrategyTest < WatchTest
       orig_env_val = ENV['K8S_NODE_NAME']
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :get_pods, @initial do
-        start_pod_watch
+        process_pod_watcher_notices(start_pod_watch)
         assert_equal(true, @cache.key?('initial_uid'))
         assert_equal(true, @cache.key?('modified_uid'))
         assert_equal(2, @stats[:pod_cache_host_updates])
@@ -155,7 +155,7 @@ class DefaultPodWatchStrategyTest < WatchTest
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :get_pods, @initial do
         @client.stub :watch_pods, [@modified] do
-          start_pod_watch
+          process_pod_watcher_notices(start_pod_watch)
           assert_equal(2, @stats[:pod_cache_host_updates])
           assert_equal(1, @stats[:pod_cache_watch_updates])
         end
@@ -166,7 +166,7 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice ignores CREATED' do
       @client.stub :get_pods, @initial do
         @client.stub :watch_pods, [@created] do
-          start_pod_watch
+          process_pod_watcher_notices(start_pod_watch)
           assert_equal(false, @cache.key?('created_uid'))
           assert_equal(1, @stats[:pod_cache_watch_ignored])
         end
@@ -175,7 +175,7 @@ class DefaultPodWatchStrategyTest < WatchTest
 
     test 'pod watch notice is ignored when info not cached and MODIFIED is received' do
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(false, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_watch_misses])
       end
@@ -185,7 +185,7 @@ class DefaultPodWatchStrategyTest < WatchTest
       orig_env_val = ENV['K8S_NODE_NAME']
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_host_updates])
       end
@@ -195,7 +195,7 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice is updated when MODIFIED is received' do
       @cache['modified_uid'] = {}
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_watch_updates])
       end
@@ -204,10 +204,22 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice is ignored when delete is received' do
       @cache['deleted_uid'] = {}
       @client.stub :watch_pods, [@deleted] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('deleted_uid'))
        assert_equal(1, @stats[:pod_cache_watch_delete_ignored])
       end
     end
 
+    test 'pod watch retries when exceptions are encountered' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [[@created, @exception_raised]] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_pod_thread
+          end
+          assert_equal(3, @stats[:pod_watch_failures])
+          assert_equal(2, Thread.current[:pod_watch_retry_count])
+          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
+        end
+      end
+    end
 end

data/test/plugin/watch_test.rb

@@ -20,38 +20,54 @@ require_relative '../helper'
 require 'ostruct'
 
 class WatchTest < Test::Unit::TestCase
-   
-    setup do
-      @annotations_regexps = []
-      @namespace_cache = {}
-      @cache = {}
-      @stats = KubernetesMetadata::Stats.new
-      @client = OpenStruct.new
-      def @client.resourceVersion
-        '12345'
-      end
-      def @client.watch_pods(options = {})
-         []
-      end
-      def @client.watch_namespaces(options = {})
-         []
-      end
-      def @client.get_namespaces(options = {})
-          self
-      end
-      def @client.get_pods(options = {})
-          self
-      end
-    end
 
-    def watcher=(value)
+  def thread_current_running?
+    true
+  end
+
+  setup do
+    @annotations_regexps = []
+    @namespace_cache = {}
+    @watch_retry_max_times = 2
+    @watch_retry_interval = 1
+    @watch_retry_exponential_backoff_base = 2
+    @cache = {}
+    @stats = KubernetesMetadata::Stats.new
+
+    @client = OpenStruct.new
+    def @client.resourceVersion
+      '12345'
+    end
+    def @client.watch_pods(options = {})
+      []
+    end
+    def @client.watch_namespaces(options = {})
+      []
+    end
+    def @client.get_namespaces(options = {})
+      self
+    end
+    def @client.get_pods(options = {})
+      self
     end
 
-    def log
-        logger = {}
-        def logger.debug(message)
-        end
-        logger
+    @exception_raised = OpenStruct.new
+    def @exception_raised.each
+      raise Exception
     end
+  end
+
+  def watcher=(value)
+  end
 
+  def log
+    logger = {}
+    def logger.debug(message)
+    end
+    def logger.info(message, error)
+    end
+    def logger.error(message, error)
+    end
+    logger
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.4.2
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-09 00:00:00.000000000 Z
+date: 2020-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -64,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -258,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata