fluent-plugin-kubernetes_metadata_filter 2.5.0 → 2.7.0

data/test/plugin/test_utils.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
+  include KubernetesMetadata::Util
+
+  def setup
+    @time_fields = ['time']
+    @internal_time = Time.now
+  end
+
+  test '#create_time_from_record when time is empty' do
+    record = { 'time' => ' ' }
+    assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
+  end
+  test '#create_time_from_record when time is nil' do
+    record = {}
+    assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
+  end
+
+  test '#create_time_from_record when time is an integer' do
+    exp_time = Time.now
+    record = { 'time' => exp_time.to_i }
+    assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
+  end
+
+  test '#create_time_from_record when time is a string' do
+    exp_time = Time.now
+    record = { 'time' => exp_time.to_s }
+    assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
+  end
+
+  test '#create_time_from_record when timefields include journal time fields' do
+    @time_fields = ['_SOURCE_REALTIME_TIMESTAMP']
+    exp_time = Time.now
+    record = { '_SOURCE_REALTIME_TIMESTAMP' => exp_time.to_i.to_s }
+    assert_equal(Time.at(exp_time.to_i / 1_000_000, exp_time.to_i % 1_000_000).to_i, create_time_from_record(record, @internal_time).to_i)
+  end
+end

data/test/plugin/test_watch_namespaces.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -17,166 +19,227 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'ostruct'
 require_relative 'watch_test'
 
 class WatchNamespacesTestTest < WatchTest
+  include KubernetesMetadata::WatchNamespaces
+
+  setup do
+    @initial = {
+      kind: 'NamespaceList',
+      metadata: { resourceVersion: '123' },
+      items: [
+        {
+          metadata: {
+            name: 'initial',
+            uid: 'initial_uid'
+          }
+        },
+        {
+          metadata: {
+            name: 'modified',
+            uid: 'modified_uid'
+          }
+        }
+      ]
+    }
+
+    @created = {
+      type: 'CREATED',
+      object: {
+        metadata: {
+          name: 'created',
+          uid: 'created_uid'
+        }
+      }
+    }
+    @modified = {
+      type: 'MODIFIED',
+      object: {
+        metadata: {
+          name: 'foo',
+          uid: 'modified_uid'
+        }
+      }
+    }
+    @deleted = {
+      type: 'DELETED',
+      object: {
+        metadata: {
+          name: 'deleteme',
+          uid: 'deleted_uid'
+        }
+      }
+    }
+    @error = {
+      type: 'ERROR',
+      object: {
+        message: 'some error message'
+      }
+    }
+    @gone = {
+      type: 'ERROR',
+      object: {
+        code: 410,
+        kind: 'Status',
+        message: 'too old resource version: 123 (391079)',
+        metadata: {
+          name: 'gone',
+          namespace: 'gone',
+          uid: 'gone_uid'
+        },
+        reason: 'Gone'
+      }
+    }
+  end
 
-     include KubernetesMetadata::WatchNamespaces
-
-     setup do
-       @initial = Kubeclient::Common::EntityList.new(
-         'NamespaceList',
-         '123',
-         [
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'initial',
-                                        'uid' => 'initial_uid'
-                                      }
-                                    }),
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'modified',
-                                        'uid' => 'modified_uid'
-                                      }
-                                    })
-         ])
-
-       @created = OpenStruct.new(
-         type: 'CREATED',
-         object: {
-           'metadata' => {
-                'name' => 'created',
-                'uid' => 'created_uid'
-            }
-         }
-       )
-       @modified = OpenStruct.new(
-         type: 'MODIFIED',
-         object: {
-           'metadata' => {
-                'name' => 'foo',
-                'uid' => 'modified_uid'
-            }
-         }
-       )
-       @deleted = OpenStruct.new(
-         type: 'DELETED',
-         object: {
-           'metadata' => {
-                'name' => 'deleteme',
-                'uid' => 'deleted_uid'
-            }
-         }
-       )
-       @error = OpenStruct.new(
-         type: 'ERROR',
-         object: {
-           'message' => 'some error message'
-         }
-       )
-     end
-
-    test 'namespace list caches namespaces' do
-      @client.stub :get_namespaces, @initial do
+  test 'namespace list caches namespaces' do
+    @client.stub :get_namespaces, @initial do
+      process_namespace_watcher_notices(start_namespace_watch)
+      assert_equal(true, @namespace_cache.key?('initial_uid'))
+      assert_equal(true, @namespace_cache.key?('modified_uid'))
+      assert_equal(2, @stats[:namespace_cache_host_updates])
+    end
+  end
+
+  test 'namespace list caches namespaces and watch updates' do
+    orig_env_val = ENV['K8S_NODE_NAME']
+    ENV['K8S_NODE_NAME'] = 'aNodeName'
+    @client.stub :get_namespaces, @initial do
+      @client.stub :watch_namespaces, [@modified] do
         process_namespace_watcher_notices(start_namespace_watch)
-        assert_equal(true, @namespace_cache.key?('initial_uid'))
-        assert_equal(true, @namespace_cache.key?('modified_uid'))
         assert_equal(2, @stats[:namespace_cache_host_updates])
+        assert_equal(1, @stats[:namespace_cache_watch_updates])
       end
     end
+    ENV['K8S_NODE_NAME'] = orig_env_val
+  end
 
-    test 'namespace list caches namespaces and watch updates' do
-      orig_env_val = ENV['K8S_NODE_NAME']
-      ENV['K8S_NODE_NAME'] = 'aNodeName'
-      @client.stub :get_namespaces, @initial do
-        @client.stub :watch_namespaces, [@modified] do
-          process_namespace_watcher_notices(start_namespace_watch)
-          assert_equal(2, @stats[:namespace_cache_host_updates])
-          assert_equal(1, @stats[:namespace_cache_watch_updates])
-        end
-      end
-      ENV['K8S_NODE_NAME'] = orig_env_val
+  test 'namespace watch ignores CREATED' do
+    @client.stub :watch_namespaces, [@created] do
+      process_namespace_watcher_notices(start_namespace_watch)
+      assert_equal(false, @namespace_cache.key?('created_uid'))
+      assert_equal(1, @stats[:namespace_cache_watch_ignored])
     end
+  end
 
-    test 'namespace watch ignores CREATED' do
-      @client.stub :watch_namespaces, [@created] do
-       process_namespace_watcher_notices(start_namespace_watch)
-       assert_equal(false, @namespace_cache.key?('created_uid'))
-       assert_equal(1, @stats[:namespace_cache_watch_ignored])
-      end
+  test 'namespace watch ignores MODIFIED when info not in cache' do
+    @client.stub :watch_namespaces, [@modified] do
+      process_namespace_watcher_notices(start_namespace_watch)
+      assert_equal(false, @namespace_cache.key?('modified_uid'))
+      assert_equal(1, @stats[:namespace_cache_watch_misses])
     end
+  end
 
-    test 'namespace watch ignores MODIFIED when info not in cache' do
-      @client.stub :watch_namespaces, [@modified] do
-       process_namespace_watcher_notices(start_namespace_watch)
-       assert_equal(false, @namespace_cache.key?('modified_uid'))
-       assert_equal(1, @stats[:namespace_cache_watch_misses])
-      end
+  test 'namespace watch updates cache when MODIFIED is received and info is cached' do
+    @namespace_cache['modified_uid'] = {}
+    @client.stub :watch_namespaces, [@modified] do
+      process_namespace_watcher_notices(start_namespace_watch)
+      assert_equal(true, @namespace_cache.key?('modified_uid'))
+      assert_equal(1, @stats[:namespace_cache_watch_updates])
     end
+  end
 
-    test 'namespace watch updates cache when MODIFIED is received and info is cached' do
-      @namespace_cache['modified_uid'] = {}
-      @client.stub :watch_namespaces, [@modified] do
-       process_namespace_watcher_notices(start_namespace_watch)
-       assert_equal(true, @namespace_cache.key?('modified_uid'))
-       assert_equal(1, @stats[:namespace_cache_watch_updates])
+  test 'namespace watch ignores DELETED' do
+    @namespace_cache['deleted_uid'] = {}
+    @client.stub :watch_namespaces, [@deleted] do
+      process_namespace_watcher_notices(start_namespace_watch)
+      assert_equal(true, @namespace_cache.key?('deleted_uid'))
+      assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
+    end
+  end
+
+  test 'namespace watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do
+    # Stub start_namespace_watch to simulate initial successful connection to API server
+    stub(self).start_namespace_watch
+    # Stub watch_namespaces to simluate not being able to set up watch connection to API server
+    stub(@client).watch_namespaces { raise }
+    @client.stub :get_namespaces, @initial do
+      assert_raise Fluent::UnrecoverableError do
+        set_up_namespace_thread
       end
     end
+    assert_equal(3, @stats[:namespace_watch_failures])
+    assert_equal(2, Thread.current[:namespace_watch_retry_count])
+    assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
+    assert_nil(@stats[:namespace_watch_error_type_notices])
+  end
 
-    test 'namespace watch ignores DELETED' do
-      @namespace_cache['deleted_uid'] = {}
-      @client.stub :watch_namespaces, [@deleted] do
-       process_namespace_watcher_notices(start_namespace_watch)
-       assert_equal(true, @namespace_cache.key?('deleted_uid'))
-       assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
+  test 'namespace watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do
+    @client.stub :get_namespaces, @initial do
+      @client.stub :watch_namespaces, [[@created, @exception_raised]] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
+            set_up_namespace_thread
+          end
+        end
+        assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
       end
     end
+  end
 
-    test 'namespace watch retries when exceptions are encountered' do
-      @client.stub :get_namespaces, @initial do
-        @client.stub :watch_namespaces, [[@created, @exception_raised]] do
-          assert_raise Fluent::UnrecoverableError do
+  test 'namespace watch resets watch retry count when error is received and connection to k8s API server is re-established' do
+    @client.stub :get_namespaces, @initial do
+      @client.stub :watch_namespaces, [@error] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
             set_up_namespace_thread
           end
-          assert_equal(3, @stats[:namespace_watch_failures])
-          assert_equal(2, Thread.current[:namespace_watch_retry_count])
-          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
-          assert_nil(@stats[:namespace_watch_error_type_notices])
         end
+        assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
       end
     end
+  end
 
-    test 'namespace watch retries when error is received' do
-      @client.stub :get_namespaces, @initial do
-        @client.stub :watch_namespaces, [@error] do
-          assert_raise Fluent::UnrecoverableError do
+  test 'namespace watch continues after retries succeed' do
+    @client.stub :get_namespaces, @initial do
+      @client.stub :watch_namespaces, [@modified, @error, @modified] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
             set_up_namespace_thread
           end
-          assert_equal(3, @stats[:namespace_watch_failures])
-          assert_equal(2, Thread.current[:namespace_watch_retry_count])
-          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
-          assert_equal(3, @stats[:namespace_watch_error_type_notices])
         end
+        assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
+        assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3)
       end
     end
+  end
 
-    test 'namespace watch continues after retries succeed' do
-      @client.stub :get_namespaces, @initial do
-        @client.stub :watch_namespaces, [@modified, @error, @modified] do
-          # Force the infinite watch loop to exit after 3 seconds. Verifies that
-          # no unrecoverable error was thrown during this period of time.
-          assert_raise Timeout::Error.new('execution expired') do
-            Timeout.timeout(3) do
-              set_up_namespace_thread
-            end
+  test 'namespace watch raises a GoneError when a 410 Gone error is received' do
+    @cache['gone_uid'] = {}
+    @client.stub :watch_namespaces, [@gone] do
+      assert_raise KubernetesMetadata::Common::GoneError do
+        process_namespace_watcher_notices(start_namespace_watch)
+      end
+      assert_equal(1, @stats[:namespace_watch_gone_notices])
+    end
+  end
+
+  test 'namespace watch retries when 410 Gone errors are encountered' do
+    @client.stub :get_namespaces, @initial do
+      @client.stub :watch_namespaces, [@created, @gone, @modified] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
+            set_up_namespace_thread
           end
-          assert_operator(@stats[:namespace_watch_failures], :>=, 3)
-          assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
-          assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
-          assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3)
         end
+        assert_operator(@stats[:namespace_watch_gone_errors], :>=, 3)
+        assert_operator(@stats[:namespace_watch_gone_notices], :>=, 3)
       end
     end
+  end
 end

data/test/plugin/test_watch_pods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -17,248 +19,326 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'ostruct'
 require_relative 'watch_test'
 
 class DefaultPodWatchStrategyTest < WatchTest
+  include KubernetesMetadata::WatchPods
 
-     include KubernetesMetadata::WatchPods
-
-     setup do
-       @initial = Kubeclient::Common::EntityList.new(
-         'PodList',
-         '123',
-         [
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'initial',
-                                        'namespace' => 'initial_ns',
-                                        'uid' => 'initial_uid',
-                                        'labels' => {},
-                                      },
-                                      'spec' => {
-                                        'nodeName' => 'aNodeName',
-                                        'containers' => [{
-                                                           'name' => 'foo',
-                                                           'image' => 'bar',
-                                                         }, {
-                                                           'name' => 'bar',
-                                                           'image' => 'foo',
-                                                         }]
-                                      }
-                                    }),
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'modified',
-                                        'namespace' => 'create',
-                                        'uid' => 'modified_uid',
-                                        'labels' => {},
-                                      },
-                                      'spec' => {
-                                        'nodeName' => 'aNodeName',
-                                        'containers' => [{
-                                                           'name' => 'foo',
-                                                           'image' => 'bar',
-                                                         }, {
-                                                           'name' => 'bar',
-                                                           'image' => 'foo',
-                                                         }]
-                                      }
-                                    }),
-         ])
-       @created = OpenStruct.new(
-         type: 'CREATED',
-         object: {
-           'metadata' => {
-                'name' => 'created',
-                'namespace' => 'create',
-                'uid' => 'created_uid',
-                'labels' => {},
-            },
-            'spec' => {
-                'nodeName' => 'aNodeName',
-                'containers' => [{
-                     'name' => 'foo',
-                     'image' => 'bar',
-                 }, {
-                     'name' => 'bar',
-                     'image' => 'foo',
-                 }]
+  setup do
+    @initial = {
+      kind: 'PodList',
+      metadata: { resourceVersion: '123' },
+      items: [
+        {
+          metadata: {
+            name: 'initial',
+            namespace: 'initial_ns',
+            uid: 'initial_uid',
+            labels: {}
+          },
+          spec: {
+            nodeName: 'aNodeName',
+            containers: [{
+              name: 'foo',
+              image: 'bar'
+            }, {
+              name: 'bar',
+              image: 'foo'
+            }]
+          },
+          status: {
+            podIP: '172.17.0.8'
+          }
+        },
+        {
+          metadata: {
+            name: 'modified',
+            namespace: 'create',
+            uid: 'modified_uid',
+            labels: {}
+          },
+          spec: {
+            nodeName: 'aNodeName',
+            containers: [{
+              name: 'foo',
+              image: 'bar'
+            }, {
+              name: 'bar',
+              image: 'foo'
+            }]
+          },
+          status: {
+            podIP: '172.17.0.8'
+          }
+        }
+      ]
+    }
+    @created = {
+      type: 'CREATED',
+      object: {
+        metadata: {
+          name: 'created',
+          namespace: 'create',
+          uid: 'created_uid',
+          resourceVersion: '122',
+          labels: {}
+        },
+        spec: {
+          nodeName: 'aNodeName',
+          containers: [{
+            name: 'foo',
+            image: 'bar'
+          }, {
+            name: 'bar',
+            image: 'foo'
+          }]
+        },
+        status: {
+          podIP: '172.17.0.8'
+        }
+      }
+    }
+    @modified = {
+      type: 'MODIFIED',
+      object: {
+        metadata: {
+          name: 'foo',
+          namespace: 'modified',
+          uid: 'modified_uid',
+          resourceVersion: '123',
+          labels: {}
+        },
+        spec: {
+          nodeName: 'aNodeName',
+          containers: [{
+            name: 'foo',
+            image: 'bar'
+          }, {
+            name: 'bar',
+            image: 'foo'
+          }]
+        },
+        status: {
+          podIP: '172.17.0.8',
+          containerStatuses: [
+            {
+              name: 'fabric8-console-container',
+              state: {
+                running: {
+                  startedAt: '2015-05-08T09:22:44Z'
+                }
+              },
+              lastState: {},
+              ready: true,
+              restartCount: 0,
+              image: 'fabric8/hawtio-kubernetes:latest',
+              imageID: 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+              containerID: 'docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
             }
-         }
-       )
-       @modified = OpenStruct.new(
-         type: 'MODIFIED',
-         object: {
-           'metadata' => {
-                'name' => 'foo',
-                'namespace' => 'modified',
-                'uid' => 'modified_uid',
-                'labels' => {},
-            },
-            'spec' => {
-                'nodeName' => 'aNodeName',
-                'containers' => [{
-                    'name' => 'foo',
-                    'image' => 'bar',
-                 }, {
-                    'name' => 'bar',
-                    'image' => 'foo',
-                 }]
-            },
-           'status' => {
-               'containerStatuses' => [
-                   {
-                       'name' => 'fabric8-console-container',
-                       'state' => {
-                           'running' => {
-                               'startedAt' => '2015-05-08T09:22:44Z'
-                           }
-                       },
-                       'lastState' => {},
-                       'ready' => true,
-                       'restartCount' => 0,
-                       'image' => 'fabric8/hawtio-kubernetes:latest',
-                       'imageID' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-                       'containerID' => 'docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
-                   }
-               ]
-           }
-         }
-       )
-       @deleted = OpenStruct.new(
-         type: 'DELETED',
-         object: {
-           'metadata' => {
-                'name' => 'deleteme',
-                'namespace' => 'deleted',
-                'uid' => 'deleted_uid'
-            }
-         }
-       )
-       @error = OpenStruct.new(
-         type: 'ERROR',
-         object: {
-           'message' => 'some error message'
-         }
-       )
-     end
+          ]
+        }
+      }
+    }
+    @deleted = {
+      type: 'DELETED',
+      object: {
+        metadata: {
+          name: 'deleteme',
+          namespace: 'deleted',
+          uid: 'deleted_uid',
+          resourceVersion: '124'
+        }
+      }
+    }
+    @error = {
+      type: 'ERROR',
+      object: {
+        message: 'some error message'
+      }
+    }
+    @gone = {
+      type: 'ERROR',
+      object: {
+        code: 410,
+        kind: 'Status',
+        message: 'too old resource version: 123 (391079)',
+        metadata: {
+          name: 'gone',
+          namespace: 'gone',
+          uid: 'gone_uid'
+        },
+        reason: 'Gone'
+      }
+    }
+  end
 
-    test 'pod list caches pods' do
-      orig_env_val = ENV['K8S_NODE_NAME']
-      ENV['K8S_NODE_NAME'] = 'aNodeName'
-      @client.stub :get_pods, @initial do
+  test 'pod list caches pods' do
+    orig_env_val = ENV['K8S_NODE_NAME']
+    ENV['K8S_NODE_NAME'] = 'aNodeName'
+    @client.stub :get_pods, @initial do
+      process_pod_watcher_notices(start_pod_watch)
+      assert_equal(true, @cache.key?('initial_uid'))
+      assert_equal(true, @cache.key?('modified_uid'))
+      assert_equal(2, @stats[:pod_cache_host_updates])
+    end
+    ENV['K8S_NODE_NAME'] = orig_env_val
+  end
+
+  test 'pod list caches pods and watch updates' do
+    orig_env_val = ENV['K8S_NODE_NAME']
+    ENV['K8S_NODE_NAME'] = 'aNodeName'
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [@modified] do
         process_pod_watcher_notices(start_pod_watch)
-        assert_equal(true, @cache.key?('initial_uid'))
-        assert_equal(true, @cache.key?('modified_uid'))
         assert_equal(2, @stats[:pod_cache_host_updates])
+        assert_equal(1, @stats[:pod_cache_watch_updates])
       end
-      ENV['K8S_NODE_NAME'] = orig_env_val
     end
+    ENV['K8S_NODE_NAME'] = orig_env_val
+    assert_equal('123', @last_seen_resource_version) # from @modified
+  end
 
-    test 'pod list caches pods and watch updates' do
-      orig_env_val = ENV['K8S_NODE_NAME']
-      ENV['K8S_NODE_NAME'] = 'aNodeName'
-      @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [@modified] do
-          process_pod_watcher_notices(start_pod_watch)
-          assert_equal(2, @stats[:pod_cache_host_updates])
-          assert_equal(1, @stats[:pod_cache_watch_updates])
-        end
+  test 'pod watch notice ignores CREATED' do
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [@created] do
+        process_pod_watcher_notices(start_pod_watch)
+        assert_equal(false, @cache.key?('created_uid'))
+        assert_equal(1, @stats[:pod_cache_watch_ignored])
       end
-      ENV['K8S_NODE_NAME'] = orig_env_val
     end
+  end
 
-    test 'pod watch notice ignores CREATED' do
-      @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [@created] do
-          process_pod_watcher_notices(start_pod_watch)
-          assert_equal(false, @cache.key?('created_uid'))
-          assert_equal(1, @stats[:pod_cache_watch_ignored])
-        end
-      end
+  test 'pod watch notice is ignored when info not cached and MODIFIED is received' do
+    @client.stub :watch_pods, [@modified] do
+      process_pod_watcher_notices(start_pod_watch)
+      assert_equal(false, @cache.key?('modified_uid'))
+      assert_equal(1, @stats[:pod_cache_watch_misses])
     end
+  end
 
-    test 'pod watch notice is ignored when info not cached and MODIFIED is received' do
-      @client.stub :watch_pods, [@modified] do
-       process_pod_watcher_notices(start_pod_watch)
-       assert_equal(false, @cache.key?('modified_uid'))
-       assert_equal(1, @stats[:pod_cache_watch_misses])
-      end
+  test 'pod MODIFIED cached when hostname matches' do
+    orig_env_val = ENV['K8S_NODE_NAME']
+    ENV['K8S_NODE_NAME'] = 'aNodeName'
+    @client.stub :watch_pods, [@modified] do
+      process_pod_watcher_notices(start_pod_watch)
+      assert_equal(true, @cache.key?('modified_uid'))
+      assert_equal(1, @stats[:pod_cache_host_updates])
     end
+    ENV['K8S_NODE_NAME'] = orig_env_val
+  end
 
-    test 'pod MODIFIED cached when hostname matches' do
-      orig_env_val = ENV['K8S_NODE_NAME']
-      ENV['K8S_NODE_NAME'] = 'aNodeName'
-      @client.stub :watch_pods, [@modified] do
-       process_pod_watcher_notices(start_pod_watch)
-       assert_equal(true, @cache.key?('modified_uid'))
-       assert_equal(1, @stats[:pod_cache_host_updates])
-      end
-      ENV['K8S_NODE_NAME'] = orig_env_val
+  test 'pod watch notice is updated when MODIFIED is received' do
+    @cache['modified_uid'] = {}
+    @client.stub :watch_pods, [@modified] do
+      process_pod_watcher_notices(start_pod_watch)
+      assert_equal(true, @cache.key?('modified_uid'))
+      assert_equal(1, @stats[:pod_cache_watch_updates])
     end
+  end
 
-    test 'pod watch notice is updated when MODIFIED is received' do
-      @cache['modified_uid'] = {}
-      @client.stub :watch_pods, [@modified] do
-       process_pod_watcher_notices(start_pod_watch)
-       assert_equal(true, @cache.key?('modified_uid'))
-       assert_equal(1, @stats[:pod_cache_watch_updates])
+  test 'pod watch notice is ignored when delete is received' do
+    @cache['deleted_uid'] = {}
+    @client.stub :watch_pods, [@deleted] do
+      process_pod_watcher_notices(start_pod_watch)
+      assert_equal(true, @cache.key?('deleted_uid'))
+      assert_equal(1, @stats[:pod_cache_watch_delete_ignored])
+    end
+  end
+
+  test 'pod watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do
+    # Stub start_pod_watch to simulate initial successful connection to API server
+    stub(self).start_pod_watch
+    # Stub watch_pods to simluate not being able to set up watch connection to API server
+    stub(@client).watch_pods { raise }
+    @client.stub :get_pods, @initial do
+      assert_raise Fluent::UnrecoverableError do
+        set_up_pod_thread
       end
     end
+    assert_equal(3, @stats[:pod_watch_failures])
+    assert_equal(2, Thread.current[:pod_watch_retry_count])
+    assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
+    assert_nil(@stats[:pod_watch_error_type_notices])
+  end
 
-    test 'pod watch notice is ignored when delete is received' do
-      @cache['deleted_uid'] = {}
-      @client.stub :watch_pods, [@deleted] do
-       process_pod_watcher_notices(start_pod_watch)
-       assert_equal(true, @cache.key?('deleted_uid'))
-       assert_equal(1, @stats[:pod_cache_watch_delete_ignored])
+  test 'pod watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [[@created, @exception_raised]] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
+            set_up_pod_thread
+          end
+        end
+        assert_operator(@stats[:pod_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
       end
     end
+  end
 
-    test 'pod watch retries when exceptions are encountered' do
-      @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [[@created, @exception_raised]] do
-          assert_raise Fluent::UnrecoverableError do
+  test 'pod watch resets watch retry count when error is received and connection to k8s API server is re-established' do
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [@error] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
             set_up_pod_thread
           end
-          assert_equal(3, @stats[:pod_watch_failures])
-          assert_equal(2, Thread.current[:pod_watch_retry_count])
-          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
-          assert_nil(@stats[:pod_watch_error_type_notices])
         end
+        assert_operator(@stats[:pod_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
+        assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
       end
     end
+  end
 
-    test 'pod watch retries when error is received' do
-      @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [@error] do
-          assert_raise Fluent::UnrecoverableError do
+  test 'pod watch continues after retries succeed' do
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [@modified, @error, @modified] do
+        # Force the infinite watch loop to exit after 3 seconds. Verifies that
+        # no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
             set_up_pod_thread
           end
-          assert_equal(3, @stats[:pod_watch_failures])
-          assert_equal(2, Thread.current[:pod_watch_retry_count])
-          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
-          assert_equal(3, @stats[:pod_watch_error_type_notices])
         end
+        assert_operator(@stats[:pod_watch_failures], :>=, 3)
+        assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+        assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
+        assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
+      end
+    end
+  end
+
+  test 'pod watch raises a GoneError when a 410 Gone error is received' do
+    @cache['gone_uid'] = {}
+    @client.stub :watch_pods, [@gone] do
+      @last_seen_resource_version = '100'
+      assert_raise KubernetesMetadata::Common::GoneError do
+        process_pod_watcher_notices(start_pod_watch)
       end
+      assert_equal(1, @stats[:pod_watch_gone_notices])
+      assert_nil @last_seen_resource_version # forced restart
     end
+  end
 
-    test 'pod watch continues after retries succeed' do
-      @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [@modified, @error, @modified] do
-          # Force the infinite watch loop to exit after 3 seconds. Verifies that
-          # no unrecoverable error was thrown during this period of time.
-          assert_raise Timeout::Error.new('execution expired') do
-            Timeout.timeout(3) do
-              set_up_pod_thread
-            end
+  test 'pod watch retries when 410 Gone errors are encountered' do
+    @client.stub :get_pods, @initial do
+      @client.stub :watch_pods, [@created, @gone, @modified] do
+        # Force the infinite watch loop to exit after 3 seconds because the code sleeps 3 times.
+        # Verifies that no unrecoverable error was thrown during this period of time.
+        assert_raise Timeout::Error.new('execution expired') do
+          Timeout.timeout(3) do
+            set_up_pod_thread
           end
-          assert_operator(@stats[:pod_watch_failures], :>=, 3)
-          assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
-          assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
-          assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
         end
+        assert_operator(@stats[:pod_watch_gone_errors], :>=, 3)
+        assert_operator(@stats[:pod_watch_gone_notices], :>=, 3)
       end
     end
+  end
 end