fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -18,6 +20,11 @@
 #
 module KubernetesMetadata
   module Common
+    class GoneError < StandardError
+      def initialize(msg = '410 Gone')
+        super
+      end
+    end
 
     def match_annotations(annotations)
       result = {}
@@ -32,61 +39,81 @@ module KubernetesMetadata
     end
 
     def parse_namespace_metadata(namespace_object)
-      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h)
-      annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
+      labels = ''
+      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
+
+      annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
       if @de_dot
-        self.de_dot!(labels)
-        self.de_dot!(annotations)
+        de_dot!(labels) unless @skip_labels
+        de_dot!(annotations)
+      end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
       end
       kubernetes_metadata = {
-        'namespace_id' => namespace_object['metadata']['uid'],
-        'creation_timestamp' => namespace_object['metadata']['creationTimestamp']
+        'namespace_id' => namespace_object[:metadata][:uid],
+        'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
       }
       kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
       kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
-      return kubernetes_metadata
+      kubernetes_metadata
     end
 
     def parse_pod_metadata(pod_object)
-      labels = syms_to_strs(pod_object['metadata']['labels'].to_h)
-      annotations = match_annotations(syms_to_strs(pod_object['metadata']['annotations'].to_h))
+      labels = ''
+      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
+
+      annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
       if @de_dot
-        self.de_dot!(labels)
-        self.de_dot!(annotations)
+        de_dot!(labels) unless @skip_labels
+        de_dot!(annotations)
+      end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
       end
 
-      # collect container informations
+      # collect container information
       container_meta = {}
       begin
-        pod_object['status']['containerStatuses'].each do|container_status|
-          # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
-          container_meta[container_id] = {
-              'name' => container_status['name'],
-              'image' => container_status['image'],
-              'image_id' => container_status['imageID']
-          }
-        end
-      rescue
-        log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
+        pod_object[:status][:containerStatuses].each do |container_status|
+          container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
+          key = container_status[:name]
+          container_meta[key] = if @skip_container_metadata
+                                           {
+                                             'name' => container_status[:name]
+                                           }
+                                         else
+                                           {
+                                             'name' => container_status[:name],
+                                             'image' => container_status[:image],
+                                             'image_id' => container_status[:imageID],
+                                             :containerID => container_id
+                                           }
+                                         end
+        end if pod_object[:status] && pod_object[:status][:containerStatuses]
+      rescue StandardError=>e
+        log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end
 
       kubernetes_metadata = {
-          'namespace_name' => pod_object['metadata']['namespace'],
-          'pod_id'         => pod_object['metadata']['uid'],
-          'pod_name'       => pod_object['metadata']['name'],
-          'containers'     => syms_to_strs(container_meta),
-          'labels'         => labels,
-          'host'           => pod_object['spec']['nodeName'],
-          'master_url'     => @kubernetes_url
+        'namespace_name' => pod_object[:metadata][:namespace],
+        'pod_id' => pod_object[:metadata][:uid],
+        'pod_name' => pod_object[:metadata][:name],
+        'pod_ip' => pod_object[:status][:podIP],
+        'containers' => syms_to_strs(container_meta),
+        'host' => pod_object[:spec][:nodeName]
       }
       kubernetes_metadata['annotations'] = annotations unless annotations.empty?
-      return kubernetes_metadata
+      kubernetes_metadata['labels'] = labels unless labels.empty?
+      kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
+      kubernetes_metadata
     end
 
     def syms_to_strs(hsh)
       newhsh = {}
-      hsh.each_pair do |kk,vv|
+      hsh.each_pair do |kk, vv|
         if vv.is_a?(Hash)
           vv = syms_to_strs(vv)
         end
@@ -98,6 +125,5 @@ module KubernetesMetadata
       end
       newhsh
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_stats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -19,17 +21,16 @@
 require 'lru_redux'
 module KubernetesMetadata
   class Stats
-
     def initialize
       @stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600)
     end
 
     def bump(key)
-        @stats[key] = @stats.getset(key) { 0 } + 1
+      @stats[key] = @stats.getset(key) { 0 } + 1
     end
 
     def set(key, value)
-       @stats[key] = value
+      @stats[key] = value
     end
 
     def [](key)
@@ -37,10 +38,9 @@ module KubernetesMetadata
     end
 
     def to_s
-      "stats - " + [].tap do |a|
-          @stats.each {|k,v| a << "#{k.to_s}: #{v}"}
+      'stats - ' + [].tap do |a|
+        @stats.each { |k, v| a << "#{k}: #{v}" }
       end.join(', ')
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2021 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'kubeclient'
+
+module KubernetesMetadata
+    module TestApiAdapter
+      
+      def api_valid?
+        true
+      end
+      def get_namespace(namespace_name)
+        return {
+          metadata: {
+            name: namespace_name,
+            uid: namespace_name + 'uuid',
+            labels: {
+              foo_ns: 'bar_ns'
+            }
+          }
+        }
+      end
+
+      def get_pod(pod_name, namespace_name)
+        return {
+          metadata: {
+            name: pod_name,
+            namespace: namespace_name,
+            uid: namespace_name + namespace_name + "uuid",
+            labels: {
+              foo: 'bar'
+            }
+          },
+          spec: {
+            nodeName: 'aNodeName',
+            containers: [{
+              name: 'foo',
+              image: 'bar'
+            }, {
+              name: 'bar',
+              image: 'foo'
+            }]
+          },
+          status: {
+            podIP: '172.17.0.8'
+          }
+        }
+      end
+
+    end
+end

data/lib/fluent/plugin/kubernetes_metadata_util.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2021 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module KubernetesMetadata
+  module Util
+    def create_time_from_record(record, internal_time)
+      time_key = @time_fields.detect { |ii| record.key?(ii) }
+      time = record[time_key]
+      if time.nil? || time.is_a?(String) && time.chop.empty?
+        # `internal_time` is a Fluent::EventTime, it can't compare with Time.
+        return Time.at(internal_time.to_f)
+      end
+
+      if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
+        timei = time.to_i
+        return Time.at(timei / 1_000_000, timei % 1_000_000)
+      end
+      return Time.at(time) if time.is_a?(Numeric)
+
+      Time.parse(time)
+    end
+  end
+end
+
+#https://stackoverflow.com/questions/5622435/how-do-i-convert-a-ruby-class-name-to-a-underscore-delimited-symbol
+class String
+  def underscore
+    word = self.dup
+    word.gsub!(/::/, '_')
+    word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
+    word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+    word.tr!("-", "_")
+    word.downcase!
+    word
+  end
+end

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,45 +18,137 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
   module WatchNamespaces
-
     include ::KubernetesMetadata::Common
 
+    def set_up_namespace_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      namespace_watcher = start_namespace_watch
+      Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:namespace_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      loop do
+        namespace_watcher ||= get_namespaces_and_start_watcher
+        process_namespace_watcher_notices(namespace_watcher)
+      rescue GoneError => e
+        # Expected error. Quietly go back through the loop in order to
+        # start watching from the latest resource versions
+        @stats.bump(:namespace_watch_gone_errors)
+        log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
+        namespace_watcher = nil
+      rescue StandardError => e
+        @stats.bump(:namespace_watch_failures)
+        if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
+          # Instead of raising exceptions and crashing Fluentd, swallow
+          # the exception and reset the watcher.
+          log.info(
+            'Exception encountered parsing namespace watch event. ' \
+            'The connection might have been closed. Sleeping for ' \
+            "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
+            'seconds and resetting the namespace watcher.', e
+          )
+          sleep(Thread.current[:namespace_watch_retry_backoff_interval])
+          Thread.current[:namespace_watch_retry_count] += 1
+          Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+          namespace_watcher = nil
+        else
+          # Since retries failed for many times, log as errors instead
+          # of info and raise exceptions and trigger Fluentd to restart.
+          message =
+            'Exception encountered parsing namespace watch event. The ' \
+            'connection might have been closed. Retried ' \
+            "#{@watch_retry_max_times} times yet still failing. Restarting."
+          log.error(message, e)
+          raise Fluent::UnrecoverableError, message
+        end
+      end
+    end
+
     def start_namespace_watch
-      begin
-        resource_version = @client.get_namespaces.resourceVersion
-        watcher          = @client.watch_namespaces(resource_version)
-      rescue Exception=>e
-        message = "start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
-        log.debug(message)
-        raise Fluent::ConfigError, message
+      get_namespaces_and_start_watcher
+    rescue StandardError => e
+      message = 'start_namespace_watch: Exception encountered setting up ' \
+                "namespace watch from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all namespaces, record the resourceVersion and return a watcher
+    # starting from that resourceVersion.
+    def get_namespaces_and_start_watcher
+      options = {
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
+      }
+      namespaces = @client.get_namespaces(options)
+      namespaces[:items].each do |namespace|
+        cache_key = namespace[:metadata][:uid]
+        @namespace_cache[cache_key] = parse_namespace_metadata(namespace)
+        @stats.bump(:namespace_cache_host_updates)
       end
+
+      # continue watching from most recent resourceVersion
+      options[:resource_version] = namespaces[:metadata][:resourceVersion]
+
+      watcher = @client.watch_namespaces(options)
+      reset_namespace_watch_retry_stats
+      watcher
+    end
+
+    # Reset namespace watch retry count and backoff interval as there is a
+    # successful watch notice.
+    def reset_namespace_watch_retry_stats
+      Thread.current[:namespace_watch_retry_count] = 0
+      Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
+    end
+
+    # Process a watcher notice and potentially raise an exception.
+    def process_namespace_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
-          when 'MODIFIED'
-            cache_key = notice.object['metadata']['uid']
-            cached    = @namespace_cache[cache_key]
-            if cached
-              @namespace_cache[cache_key] = parse_namespace_metadata(notice.object)
-              @stats.bump(:namespace_cache_watch_updates)
-            else
-              @stats.bump(:namespace_cache_watch_misses)
-            end
-          when 'DELETED'
-            # ignore and let age out for cases where 
-            # deleted but still processing logs
-            @stats.bump(:namespace_cache_watch_deletes_ignored)
+        case notice[:type]
+        when 'MODIFIED'
+          reset_namespace_watch_retry_stats
+          cache_key = notice[:object][:metadata][:uid]
+          cached    = @namespace_cache[cache_key]
+          if cached
+            @namespace_cache[cache_key] = parse_namespace_metadata(notice[:object])
+            @stats.bump(:namespace_cache_watch_updates)
           else
-            # Don't pay attention to creations, since the created namespace may not
-            # be used by any pod on this node.
-            @stats.bump(:namespace_cache_watch_ignored)
+            @stats.bump(:namespace_cache_watch_misses)
+          end
+        when 'DELETED'
+          reset_namespace_watch_retry_stats
+          # ignore and let age out for cases where
+          # deleted but still processing logs
+          @stats.bump(:namespace_cache_watch_deletes_ignored)
+        when 'ERROR'
+          if notice[:object] && notice[:object][:code] == 410
+            @stats.bump(:namespace_watch_gone_notices)
+            raise GoneError
+          else
+            @stats.bump(:namespace_watch_error_type_notices)
+            message = notice[:object][:message] if notice[:object] && notice[:object][:message]
+            raise "Error while watching namespaces: #{message}"
+          end
+        else
+          reset_namespace_watch_retry_stats
+          # Don't pay attention to creations, since the created namespace may not
+          # be used by any namespace on this node.
+          @stats.bump(:namespace_cache_watch_ignored)
         end
       end
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,46 +18,153 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
   module WatchPods
-
     include ::KubernetesMetadata::Common
 
+    def set_up_pod_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      pod_watcher = start_pod_watch
+
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:pod_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      loop do
+        pod_watcher ||= get_pods_and_start_watcher
+        process_pod_watcher_notices(pod_watcher)
+      rescue GoneError => e
+        # Expected error. Quietly go back through the loop in order to
+        # start watching from the latest resource versions
+        @stats.bump(:pod_watch_gone_errors)
+        log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
+        pod_watcher = nil
+      rescue StandardError => e
+        @stats.bump(:pod_watch_failures)
+        if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+          # Instead of raising exceptions and crashing Fluentd, swallow
+          # the exception and reset the watcher.
+          log.info(
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Sleeping for ' \
+            "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+            'seconds and resetting the pod watcher.', e
+          )
+          sleep(Thread.current[:pod_watch_retry_backoff_interval])
+          Thread.current[:pod_watch_retry_count] += 1
+          Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+          pod_watcher = nil
+        else
+          # Since retries failed for many times, log as errors instead
+          # of info and raise exceptions and trigger Fluentd to restart.
+          message =
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Retried ' \
+            "#{@watch_retry_max_times} times yet still failing. Restarting."
+          log.error(message, e)
+          raise Fluent::UnrecoverableError, message
+        end
+      end
+    end
+
     def start_pod_watch
-      begin
-        resource_version = @client.get_pods.resourceVersion
-        watcher          = @client.watch_pods(resource_version)
-      rescue Exception => e
-        message = "Exception encountered fetching metadata from Kubernetes API endpoint: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
-
-        raise Fluent::ConfigError, message
+      get_pods_and_start_watcher
+    rescue StandardError => e
+      message = 'start_pod_watch: Exception encountered setting up pod watch ' \
+                "from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all pods, record the resourceVersion and return a watcher starting
+    # from that resourceVersion.
+    def get_pods_and_start_watcher
+      options = {
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
+      }
+      if ENV['K8S_NODE_NAME']
+        options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
       end
+      if @last_seen_resource_version
+        options[:resource_version] = @last_seen_resource_version
+      else
+        pods = @client.get_pods(options)
+        pods[:items].each do |pod|
+          cache_key = pod[:metadata][:uid]
+          @cache[cache_key] = parse_pod_metadata(pod)
+          @stats.bump(:pod_cache_host_updates)
+        end
+
+        # continue watching from most recent resourceVersion
+        options[:resource_version] = pods[:metadata][:resourceVersion]
+      end
+
+      watcher = @client.watch_pods(options)
+      reset_pod_watch_retry_stats
+      watcher
+    end
+
+    # Reset pod watch retry count and backoff interval as there is a
+    # successful watch notice.
+    def reset_pod_watch_retry_stats
+      Thread.current[:pod_watch_retry_count] = 0
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+    end
 
+    # Process a watcher notice and potentially raise an exception.
+    def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
-          when 'MODIFIED'
-            cache_key = notice.object['metadata']['uid']
-            cached    = @cache[cache_key]
-            if cached
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_host_updates)
-            else
-              @stats.bump(:pod_cache_watch_misses)
-            end
-          when 'DELETED'
-            # ignore and let age out for cases where pods
-            # deleted but still processing logs
-            @stats.bump(:pod_cache_watch_delete_ignored)
+        # store version we processed to not reprocess it ... do not unset when there is no version in response
+        version = ( # TODO: replace with &.dig once we are on ruby 2.5+
+          notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion]
+        )
+        @last_seen_resource_version = version if version
+
+        case notice[:type]
+        when 'MODIFIED'
+          reset_pod_watch_retry_stats
+          cache_key = notice.dig(:object, :metadata, :uid)
+          cached    = @cache[cache_key]
+          if cached
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_watch_updates)
+          elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName]
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_host_updates)
+          else
+            @stats.bump(:pod_cache_watch_misses)
+          end
+        when 'DELETED'
+          reset_pod_watch_retry_stats
+          # ignore and let age out for cases where pods
+          # deleted but still processing logs
+          @stats.bump(:pod_cache_watch_delete_ignored)
+        when 'ERROR'
+          if notice[:object] && notice[:object][:code] == 410
+            @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset
+            @stats.bump(:pod_watch_gone_notices)
+            raise GoneError
           else
-            # Don't pay attention to creations, since the created pod may not
-            # end up on this node.
-            @stats.bump(:pod_cache_watch_ignored)
+            @stats.bump(:pod_watch_error_type_notices)
+            message = notice[:object][:message] if notice[:object] && notice[:object][:message]
+            raise "Error while watching pods: #{message}"
+          end
+        else
+          reset_pod_watch_retry_stats
+          # Don't pay attention to creations, since the created pod may not
+          # end up on this node.
+          @stats.bump(:pod_cache_watch_ignored)
         end
       end
     end