fluent-plugin-kubernetes_metadata_filter 2.4.7 → 2.6.0

data/test/plugin/test_watch_pods.rb

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'ostruct'
 require_relative 'watch_test'
 
 class DefaultPodWatchStrategyTest < WatchTest
@@ -25,137 +24,141 @@ class DefaultPodWatchStrategyTest < WatchTest
      include KubernetesMetadata::WatchPods
 
      setup do
-       @initial = Kubeclient::Common::EntityList.new(
-         'PodList',
-         '123',
-         [
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'initial',
-                                        'namespace' => 'initial_ns',
-                                        'uid' => 'initial_uid',
-                                        'labels' => {},
-                                      },
-                                      'spec' => {
-                                        'nodeName' => 'aNodeName',
-                                        'containers' => [{
-                                                           'name' => 'foo',
-                                                           'image' => 'bar',
-                                                         }, {
-                                                           'name' => 'bar',
-                                                           'image' => 'foo',
-                                                         }]
-                                      }
-                                    }),
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'modified',
-                                        'namespace' => 'create',
-                                        'uid' => 'modified_uid',
-                                        'labels' => {},
-                                      },
-                                      'spec' => {
-                                        'nodeName' => 'aNodeName',
-                                        'containers' => [{
-                                                           'name' => 'foo',
-                                                           'image' => 'bar',
-                                                         }, {
-                                                           'name' => 'bar',
-                                                           'image' => 'foo',
-                                                         }]
-                                      }
-                                    }),
-         ])
-       @created = OpenStruct.new(
+       @initial = {
+         kind: 'PodList',
+         metadata: {resourceVersion: '123'},
+         items: [
+           {
+             metadata: {
+               name: 'initial',
+               namespace: 'initial_ns',
+               uid: 'initial_uid',
+               labels: {},
+             },
+             spec: {
+               nodeName: 'aNodeName',
+               containers: [{
+                 name: 'foo',
+                 image: 'bar',
+               }, {
+                 name: 'bar',
+                 image: 'foo',
+               }]
+             }
+           },
+           {
+             metadata: {
+               name: 'modified',
+               namespace: 'create',
+               uid: 'modified_uid',
+               labels: {},
+             },
+             spec: {
+               nodeName: 'aNodeName',
+               containers: [{
+                 name: 'foo',
+                 image: 'bar',
+               }, {
+                 name: 'bar',
+                 image: 'foo',
+               }]
+             }
+           }
+         ]
+       }
+       @created = {
          type: 'CREATED',
          object: {
-           'metadata' => {
-                'name' => 'created',
-                'namespace' => 'create',
-                'uid' => 'created_uid',
-                'labels' => {},
-            },
-            'spec' => {
-                'nodeName' => 'aNodeName',
-                'containers' => [{
-                     'name' => 'foo',
-                     'image' => 'bar',
-                 }, {
-                     'name' => 'bar',
-                     'image' => 'foo',
-                 }]
-            }
+           metadata: {
+             name: 'created',
+             namespace: 'create',
+             uid: 'created_uid',
+             resourceVersion: '122',
+             labels: {},
+           },
+           spec: {
+             nodeName: 'aNodeName',
+             containers: [{
+               name: 'foo',
+               image: 'bar',
+             }, {
+               name: 'bar',
+               image: 'foo',
+             }]
+           }
          }
-       )
-       @modified = OpenStruct.new(
+       }
+       @modified = {
          type: 'MODIFIED',
          object: {
-           'metadata' => {
-                'name' => 'foo',
-                'namespace' => 'modified',
-                'uid' => 'modified_uid',
-                'labels' => {},
-            },
-            'spec' => {
-                'nodeName' => 'aNodeName',
-                'containers' => [{
-                    'name' => 'foo',
-                    'image' => 'bar',
-                 }, {
-                    'name' => 'bar',
-                    'image' => 'foo',
-                 }]
-            },
-           'status' => {
-               'containerStatuses' => [
-                   {
-                       'name' => 'fabric8-console-container',
-                       'state' => {
-                           'running' => {
-                               'startedAt' => '2015-05-08T09:22:44Z'
-                           }
-                       },
-                       'lastState' => {},
-                       'ready' => true,
-                       'restartCount' => 0,
-                       'image' => 'fabric8/hawtio-kubernetes:latest',
-                       'imageID' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-                       'containerID' => 'docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+           metadata: {
+             name: 'foo',
+             namespace: 'modified',
+             uid: 'modified_uid',
+             resourceVersion: '123',
+             labels: {},
+           },
+           spec: {
+             nodeName: 'aNodeName',
+             containers: [{
+               name: 'foo',
+               image: 'bar',
+             }, {
+               name: 'bar',
+               image: 'foo',
+             }]
+           },
+           status: {
+             containerStatuses: [
+               {
+                 name: 'fabric8-console-container',
+                 state: {
+                   running: {
+                     startedAt: '2015-05-08T09:22:44Z'
                    }
-               ]
+                 },
+                 lastState: {},
+                 ready: true,
+                 restartCount: 0,
+                 image: 'fabric8/hawtio-kubernetes:latest',
+                 imageID: 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+                 containerID: 'docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+               }
+             ]
            }
          }
-       )
-       @deleted = OpenStruct.new(
+       }
+       @deleted = {
          type: 'DELETED',
          object: {
-           'metadata' => {
-                'name' => 'deleteme',
-                'namespace' => 'deleted',
-                'uid' => 'deleted_uid'
-            }
+           metadata: {
+             name: 'deleteme',
+             namespace: 'deleted',
+             uid: 'deleted_uid',
+             resourceVersion: '124'
+           }
          }
-       )
-       @error = OpenStruct.new(
+       }
+       @error = {
          type: 'ERROR',
          object: {
-           'message' => 'some error message'
+           message: 'some error message'
          }
-       )
-       @gone = OpenStruct.new(
-           type: 'ERROR',
-           object: {
-               'code' => 410,
-               'kind' => 'Status',
-               'message' => 'too old resource version: 123 (391079)',
-               'metadata' => {
-                   'name' => 'gone',
-                   'namespace' => 'gone',
-                   'uid' => 'gone_uid'
-               },
-               'reason' => 'Gone'
-           }
-       )
+       }
+       @gone = {
+         type: 'ERROR',
+         object: {
+           code: 410,
+           kind: 'Status',
+           message: 'too old resource version: 123 (391079)',
+           metadata: {
+             name: 'gone',
+             namespace: 'gone',
+             uid: 'gone_uid'
+           },
+           reason: 'Gone'
+         }
+       }
      end
 
     test 'pod list caches pods' do
@@ -181,6 +184,7 @@ class DefaultPodWatchStrategyTest < WatchTest
         end
       end
       ENV['K8S_NODE_NAME'] = orig_env_val
+      assert_equal('123', @last_seen_resource_version) # from @modified
     end
 
     test 'pod watch notice ignores CREATED' do
@@ -230,40 +234,53 @@ class DefaultPodWatchStrategyTest < WatchTest
       end
     end
 
-    test 'pod watch retries when exceptions are encountered' do
+    test 'pod watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do
+      # Stub start_pod_watch to simulate initial successful connection to API server
+      stub(self).start_pod_watch
+      # Stub watch_pods to simluate not being able to set up watch connection to API server
+      stub(@client).watch_pods { raise }
       @client.stub :get_pods, @initial do
-        @client.stub :watch_pods, [[@created, @exception_raised]] do
-          assert_raise Fluent::UnrecoverableError do
-            set_up_pod_thread
-          end
-          assert_equal(3, @stats[:pod_watch_failures])
-          assert_equal(2, Thread.current[:pod_watch_retry_count])
-          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
-          assert_nil(@stats[:pod_watch_error_type_notices])
+        assert_raise Fluent::UnrecoverableError do
+          set_up_pod_thread
         end
       end
+      assert_equal(3, @stats[:pod_watch_failures])
+      assert_equal(2, Thread.current[:pod_watch_retry_count])
+      assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
+      assert_nil(@stats[:pod_watch_error_type_notices])
     end
 
-     test 'pod watch raises a GoneError when a 410 Gone error is received' do
-       @cache['gone_uid'] = {}
-       @client.stub :watch_pods, [@gone] do
-         assert_raise KubernetesMetadata::GoneError do
-           process_pod_watcher_notices(start_pod_watch)
-         end
-         assert_equal(1, @stats[:pod_watch_gone_notices])
-       end
-     end
+    test 'pod watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [[@created, @exception_raised]] do
+            # Force the infinite watch loop to exit after 3 seconds. Verifies that
+            # no unrecoverable error was thrown during this period of time.
+            assert_raise Timeout::Error.new('execution expired') do
+              Timeout.timeout(3) do
+                set_up_pod_thread
+              end
+            end
+            assert_operator(@stats[:pod_watch_failures], :>=, 3)
+            assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+            assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
+        end
+      end
+    end
 
-    test 'pod watch retries when error is received' do
+    test 'pod watch resets watch retry count when error is received and connection to k8s API server is re-established' do
       @client.stub :get_pods, @initial do
         @client.stub :watch_pods, [@error] do
-          assert_raise Fluent::UnrecoverableError do
-            set_up_pod_thread
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_pod_thread
+            end
           end
-          assert_equal(3, @stats[:pod_watch_failures])
-          assert_equal(2, Thread.current[:pod_watch_retry_count])
-          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
-          assert_equal(3, @stats[:pod_watch_error_type_notices])
+          assert_operator(@stats[:pod_watch_failures], :>=, 3)
+          assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+          assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
+          assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
         end
       end
     end
@@ -285,4 +302,32 @@ class DefaultPodWatchStrategyTest < WatchTest
         end
       end
     end
+
+    test 'pod watch raises a GoneError when a 410 Gone error is received' do
+      @cache['gone_uid'] = {}
+      @client.stub :watch_pods, [@gone] do
+        @last_seen_resource_version = '100'
+        assert_raise KubernetesMetadata::Common::GoneError do
+          process_pod_watcher_notices(start_pod_watch)
+        end
+        assert_equal(1, @stats[:pod_watch_gone_notices])
+        assert_nil @last_seen_resource_version # forced restart
+      end
+    end
+
+    test 'pod watch retries when 410 Gone errors are encountered' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [@created, @gone, @modified] do
+          # Force the infinite watch loop to exit after 3 seconds because the code sleeps 3 times.
+          # Verifies that no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_pod_thread
+            end
+          end
+          assert_operator(@stats[:pod_watch_gone_errors], :>=, 3)
+          assert_operator(@stats[:pod_watch_gone_notices], :>=, 3)
+        end
+      end
+    end
 end

data/test/plugin/watch_test.rb

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'ostruct'
 
 class WatchTest < Test::Unit::TestCase
 
@@ -37,9 +36,6 @@ class WatchTest < Test::Unit::TestCase
     Thread.current[:namespace_watch_retry_count] = 0
 
     @client = OpenStruct.new
-    def @client.resourceVersion
-      '12345'
-    end
     def @client.watch_pods(options = {})
       []
     end
@@ -47,16 +43,13 @@ class WatchTest < Test::Unit::TestCase
       []
     end
     def @client.get_namespaces(options = {})
-      self
+      {items: [], metadata: {resourceVersion: '12345'}}
     end
     def @client.get_pods(options = {})
-      self
+      {items: [], metadata: {resourceVersion: '12345'}}
     end
 
-    @exception_raised = OpenStruct.new
-    def @exception_raised.each
-      raise Exception
-    end
+    @exception_raised = :blow_up_when_used
   end
 
   def watcher=(value)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.4.7
+  version: 2.6.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-10 00:00:00.000000000 Z
+date: 2021-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -19,7 +19,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
@@ -252,7 +252,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -263,25 +263,4 @@ rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata
-test_files:
-- test/cassettes/invalid_api_server_config.yml
-- test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
-- test/cassettes/kubernetes_get_api_v1.yml
-- test/cassettes/kubernetes_get_api_v1_using_token.yml
-- test/cassettes/kubernetes_get_namespace_default.yml
-- test/cassettes/kubernetes_get_namespace_default_using_token.yml
-- test/cassettes/kubernetes_get_pod.yml
-- test/cassettes/kubernetes_get_pod_using_token.yml
-- test/cassettes/metadata_from_tag_and_journald_fields.yml
-- test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
-- test/cassettes/valid_kubernetes_api_server.yml
-- test/cassettes/valid_kubernetes_api_server_using_token.yml
-- test/helper.rb
-- test/plugin/test.token
-- test/plugin/test_cache_stats.rb
-- test/plugin/test_cache_strategy.rb
-- test/plugin/test_filter_kubernetes_metadata.rb
-- test/plugin/test_watch_namespaces.rb
-- test/plugin/test_watch_pods.rb
-- test/plugin/watch_test.rb
+test_files: []