fluent-plugin-kubernetes_metadata_filter 2.5.0 → 2.6.0

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
@@ -39,7 +40,13 @@ module KubernetesMetadata
         begin
           namespace_watcher ||= get_namespaces_and_start_watcher
           process_namespace_watcher_notices(namespace_watcher)
-        rescue Exception => e
+        rescue GoneError => e
+          # Expected error. Quietly go back through the loop in order to
+          # start watching from the latest resource versions
+          @stats.bump(:namespace_watch_gone_errors)
+          log.info("410 Gone encountered. Restarting namespace watch to reset resource versions.", e)
+          namespace_watcher = nil
+        rescue => e
           @stats.bump(:namespace_watch_failures)
           if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
             # Instead of raising exceptions and crashing Fluentd, swallow
@@ -68,8 +75,8 @@ module KubernetesMetadata
     end
 
     def start_namespace_watch
-      return get_namespaces_and_start_watcher
-    rescue Exception => e
+      get_namespaces_and_start_watcher
+    rescue => e
       message = "start_namespace_watch: Exception encountered setting up " \
                 "namespace watch from Kubernetes API #{@apiVersion} endpoint " \
                 "#{@kubernetes_url}: #{e.message}"
@@ -83,16 +90,20 @@ module KubernetesMetadata
     # starting from that resourceVersion.
     def get_namespaces_and_start_watcher
       options = {
-        resource_version: '0'  # Fetch from API server.
+        resource_version: '0'  # Fetch from API server cache instead of etcd quorum read
       }
       namespaces = @client.get_namespaces(options)
-      namespaces.each do |namespace|
-        cache_key = namespace.metadata['uid']
+      namespaces[:items].each do |namespace|
+        cache_key = namespace[:metadata][:uid]
         @namespace_cache[cache_key] = parse_namespace_metadata(namespace)
         @stats.bump(:namespace_cache_host_updates)
       end
-      options[:resource_version] = namespaces.resourceVersion
+
+      # continue watching from most recent resourceVersion
+      options[:resource_version] = namespaces[:metadata][:resourceVersion]
+
       watcher = @client.watch_namespaces(options)
+      reset_namespace_watch_retry_stats
       watcher
     end
 
@@ -106,13 +117,13 @@ module KubernetesMetadata
     # Process a watcher notice and potentially raise an exception.
     def process_namespace_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
+        case notice[:type]
           when 'MODIFIED'
             reset_namespace_watch_retry_stats
-            cache_key = notice.object['metadata']['uid']
+            cache_key = notice[:object][:metadata][:uid]
             cached    = @namespace_cache[cache_key]
             if cached
-              @namespace_cache[cache_key] = parse_namespace_metadata(notice.object)
+              @namespace_cache[cache_key] = parse_namespace_metadata(notice[:object])
               @stats.bump(:namespace_cache_watch_updates)
             else
               @stats.bump(:namespace_cache_watch_misses)
@@ -123,9 +134,14 @@ module KubernetesMetadata
             # deleted but still processing logs
             @stats.bump(:namespace_cache_watch_deletes_ignored)
           when 'ERROR'
-            @stats.bump(:namespace_watch_error_type_notices)
-            message = notice['object']['message'] if notice['object'] && notice['object']['message']
-            raise "Error while watching namespaces: #{message}"
+            if notice[:object] && notice[:object][:code] == 410
+              @stats.bump(:namespace_watch_gone_notices)
+              raise GoneError
+            else
+              @stats.bump(:namespace_watch_error_type_notices)
+              message = notice[:object][:message] if notice[:object] && notice[:object][:message]
+              raise "Error while watching namespaces: #{message}"
+            end
           else
             reset_namespace_watch_retry_stats
             # Don't pay attention to creations, since the created namespace may not

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -16,9 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
+
   module WatchPods
 
     include ::KubernetesMetadata::Common
@@ -28,6 +30,7 @@ module KubernetesMetadata
       # Fluent:ConfigError, so that users can inspect potential errors in
       # the configuration.
       pod_watcher = start_pod_watch
+
       Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
       Thread.current[:pod_watch_retry_count] = 0
 
@@ -39,7 +42,13 @@ module KubernetesMetadata
         begin
           pod_watcher ||= get_pods_and_start_watcher
           process_pod_watcher_notices(pod_watcher)
-        rescue Exception => e
+        rescue GoneError => e
+          # Expected error. Quietly go back through the loop in order to
+          # start watching from the latest resource versions
+          @stats.bump(:pod_watch_gone_errors)
+          log.info("410 Gone encountered. Restarting pod watch to reset resource versions.", e)
+          pod_watcher = nil
+        rescue => e
           @stats.bump(:pod_watch_failures)
           if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
             # Instead of raising exceptions and crashing Fluentd, swallow
@@ -69,7 +78,7 @@ module KubernetesMetadata
 
     def start_pod_watch
       get_pods_and_start_watcher
-    rescue Exception => e
+    rescue => e
       message = "start_pod_watch: Exception encountered setting up pod watch " \
                 "from Kubernetes API #{@apiVersion} endpoint " \
                 "#{@kubernetes_url}: #{e.message}"
@@ -83,19 +92,27 @@ module KubernetesMetadata
     # from that resourceVersion.
     def get_pods_and_start_watcher
       options = {
-        resource_version: '0'  # Fetch from API server.
+        resource_version: '0'  # Fetch from API server cache instead of etcd quorum read
       }
       if ENV['K8S_NODE_NAME']
         options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
       end
-      pods = @client.get_pods(options)
-      pods.each do |pod|
-        cache_key = pod.metadata['uid']
-        @cache[cache_key] = parse_pod_metadata(pod)
-        @stats.bump(:pod_cache_host_updates)
+      if @last_seen_resource_version
+        options[:resource_version] = @last_seen_resource_version
+      else
+        pods = @client.get_pods(options)
+        pods[:items].each do |pod|
+          cache_key = pod[:metadata][:uid]
+          @cache[cache_key] = parse_pod_metadata(pod)
+          @stats.bump(:pod_cache_host_updates)
+        end
+
+        # continue watching from most recent resourceVersion
+        options[:resource_version] = pods[:metadata][:resourceVersion]
       end
-      options[:resource_version] = pods.resourceVersion
+
       watcher = @client.watch_pods(options)
+      reset_pod_watch_retry_stats
       watcher
     end
 
@@ -109,16 +126,22 @@ module KubernetesMetadata
     # Process a watcher notice and potentially raise an exception.
     def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
+        # store version we processed to not reprocess it ... do not unset when there is no version in response
+        version = ( # TODO: replace with &.dig once we are on ruby 2.5+
+          notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion]
+        )
+        @last_seen_resource_version = version if version
+
+        case notice[:type]
           when 'MODIFIED'
             reset_pod_watch_retry_stats
-            cache_key = notice.object['metadata']['uid']
+            cache_key = notice.dig(:object, :metadata, :uid)
             cached    = @cache[cache_key]
             if cached
-              @cache[cache_key] = parse_pod_metadata(notice.object)
+              @cache[cache_key] = parse_pod_metadata(notice[:object])
               @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
+            elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName] then
+              @cache[cache_key] = parse_pod_metadata(notice[:object])
               @stats.bump(:pod_cache_host_updates)
             else
               @stats.bump(:pod_cache_watch_misses)
@@ -129,9 +152,15 @@ module KubernetesMetadata
             # deleted but still processing logs
             @stats.bump(:pod_cache_watch_delete_ignored)
           when 'ERROR'
-            @stats.bump(:pod_watch_error_type_notices)
-            message = notice['object']['message'] if notice['object'] && notice['object']['message']
-            raise "Error while watching pods: #{message}"
+            if notice[:object] && notice[:object][:code] == 410
+              @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset
+              @stats.bump(:pod_watch_gone_notices)
+              raise GoneError
+            else
+              @stats.bump(:pod_watch_error_type_notices)
+              message = notice[:object][:message] if notice[:object] && notice[:object][:message]
+              raise "Error while watching pods: #{message}"
+            end
           else
             reset_pod_watch_retry_stats
             # Don't pay attention to creations, since the created pod may not

data/test/helper.rb

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'bundler/setup'
 require 'codeclimate-test-reporter'
 SimpleCov.start do
   formatter SimpleCov::Formatter::MultiFormatter.new [
@@ -31,8 +32,14 @@ require 'fileutils'
 require 'fluent/log'
 require 'fluent/test'
 require 'minitest/autorun'
-require 'webmock/test_unit'
 require 'vcr'
+require 'ostruct'
+require 'fluent/plugin/filter_kubernetes_metadata'
+require 'fluent/test/driver/filter'
+require 'kubeclient'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!
 
 VCR.configure do |config|
   config.cassette_library_dir = 'test/cassettes'
@@ -62,3 +69,12 @@ def ipv6_enabled?
     false
   end
 end
+
+# TEST_NAME='foo' ruby test_file.rb to run a single test case
+if ENV["TEST_NAME"]
+  (class << Test::Unit::TestCase; self; end).prepend(Module.new do
+    def test(name)
+      super if name == ENV["TEST_NAME"]
+    end
+  end)
+end

data/test/plugin/test_cache_stats.rb

@@ -17,12 +17,9 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'fluent/plugin/kubernetes_metadata_stats'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
-  
+
     test 'watch stats' do
       require 'lru_redux'
       stats = KubernetesMetadata::Stats.new
@@ -32,5 +29,5 @@ class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
 
       assert_equal("stats - deleted: 2, missed: 1", stats.to_s)
     end
-    
+
 end

data/test/plugin/test_cache_strategy.rb

@@ -17,11 +17,6 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
-require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
-require 'lru_redux'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class TestCacheStrategy
     include KubernetesMetadata::CacheStrategy
@@ -38,9 +33,11 @@ class TestCacheStrategy
     attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
 
     def fetch_pod_metadata(namespace_name, pod_name)
+        {}
     end
 
     def fetch_namespace_metadata(namespace_name)
+        {}
     end
 
     def log
@@ -56,7 +53,7 @@ class TestCacheStrategy
 end
 
 class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
-  
+
     def setup
        @strategy = TestCacheStrategy.new
        @cache_key = 'some_long_container_id'
@@ -114,7 +111,7 @@ class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
         # we ever will have and should allow us to process all the deleted
         # pod records
         exp = {
-            'pod_id'=> @cache_key, 
+            'pod_id'=> @cache_key,
             'namespace_id'=> @namespace_uuid
         }
         @strategy.stub :fetch_pod_metadata, {} do
@@ -175,7 +172,7 @@ class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
         end
         assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
     end
-    
+
     test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
         # we should never see this since pod meta should not be retrievable
         # unless the namespace exists

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -17,11 +17,6 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'fluent/test/driver/filter'
-require 'fluent/plugin/filter_kubernetes_metadata'
-
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class KubernetesMetadataFilterTest < Test::Unit::TestCase
   include Fluent
@@ -122,8 +117,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               secret_dir #{dir}
             ")
             assert_equal(d.instance.kubernetes_url, "https://localhost:8443/api")
-            assert_false(d.instance.ca_file.present?)
-            assert_false(d.instance.bearer_token_file.present?)
+            assert_nil(d.instance.ca_file, nil)
+            assert_nil(d.instance.bearer_token_file)
           }
         ensure
           ENV['KUBERNETES_SERVICE_HOST'] = nil
@@ -769,9 +764,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
         'randomfield' => 'randomvalue'
       }
-      VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'},
-                         {name: 'kubernetes_get_namespace_default'},
-                         {name: 'metadata_from_tag_and_journald_fields'}]) do
+      VCR.use_cassettes([
+        {name: 'valid_kubernetes_api_server'},
+        {name: 'kubernetes_get_api_v1'},
+        {name: 'kubernetes_get_pod'},
+        {name: 'kubernetes_get_namespace_default'},
+        {name: 'metadata_from_tag_and_journald_fields'}
+      ]) do
         filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
@@ -873,6 +872,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+
     test 'with CONTAINER_NAME that does not match' do
       tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
       msg = {
@@ -897,6 +897,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+
     test 'with CONTAINER_NAME starts with k8s_ that does not match' do
       tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
       msg = {

data/test/plugin/test_watch_namespaces.rb

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'ostruct'
 require_relative 'watch_test'
 
 class WatchNamespacesTestTest < WatchTest
@@ -25,57 +24,72 @@ class WatchNamespacesTestTest < WatchTest
      include KubernetesMetadata::WatchNamespaces
 
      setup do
-       @initial = Kubeclient::Common::EntityList.new(
-         'NamespaceList',
-         '123',
-         [
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'initial',
-                                        'uid' => 'initial_uid'
-                                      }
-                                    }),
-           Kubeclient::Resource.new({
-                                      'metadata' => {
-                                        'name' => 'modified',
-                                        'uid' => 'modified_uid'
-                                      }
-                                    })
-         ])
-
-       @created = OpenStruct.new(
+       @initial = {
+         kind: 'NamespaceList',
+         metadata: {resourceVersion: '123'},
+         items: [
+           {
+             metadata: {
+               name: 'initial',
+               uid: 'initial_uid'
+             }
+           },
+           {
+             metadata: {
+               name: 'modified',
+               uid: 'modified_uid'
+             }
+           }
+         ]
+       }
+
+       @created = {
          type: 'CREATED',
          object: {
-           'metadata' => {
-                'name' => 'created',
-                'uid' => 'created_uid'
-            }
+           metadata: {
+             name: 'created',
+             uid: 'created_uid'
+           }
          }
-       )
-       @modified = OpenStruct.new(
+       }
+       @modified = {
          type: 'MODIFIED',
          object: {
-           'metadata' => {
-                'name' => 'foo',
-                'uid' => 'modified_uid'
-            }
+           metadata: {
+             name: 'foo',
+             uid: 'modified_uid'
+           }
          }
-       )
-       @deleted = OpenStruct.new(
+       }
+       @deleted = {
          type: 'DELETED',
          object: {
-           'metadata' => {
-                'name' => 'deleteme',
-                'uid' => 'deleted_uid'
-            }
+           metadata: {
+             name: 'deleteme',
+             uid: 'deleted_uid'
+           }
+         }
+       }
+       @error = {
+         type: 'ERROR',
+         object: {
+           message: 'some error message'
          }
-       )
-       @error = OpenStruct.new(
+       }
+       @gone = {
          type: 'ERROR',
          object: {
-           'message' => 'some error message'
+           code: 410,
+           kind: 'Status',
+           message: 'too old resource version: 123 (391079)',
+           metadata: {
+             name: 'gone',
+             namespace: 'gone',
+             uid: 'gone_uid'
+           },
+           reason: 'Gone'
          }
-       )
+       }
      end
 
     test 'namespace list caches namespaces' do
@@ -134,30 +148,52 @@ class WatchNamespacesTestTest < WatchTest
       end
     end
 
-    test 'namespace watch retries when exceptions are encountered' do
+    test 'namespace watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do
+      # Stub start_namespace_watch to simulate initial successful connection to API server
+      stub(self).start_namespace_watch
+      # Stub watch_namespaces to simluate not being able to set up watch connection to API server
+      stub(@client).watch_namespaces { raise }
+      @client.stub :get_namespaces, @initial do
+        assert_raise Fluent::UnrecoverableError do
+          set_up_namespace_thread
+        end
+      end
+      assert_equal(3, @stats[:namespace_watch_failures])
+      assert_equal(2, Thread.current[:namespace_watch_retry_count])
+      assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
+      assert_nil(@stats[:namespace_watch_error_type_notices])
+    end
+
+    test 'namespace watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do
       @client.stub :get_namespaces, @initial do
         @client.stub :watch_namespaces, [[@created, @exception_raised]] do
-          assert_raise Fluent::UnrecoverableError do
-            set_up_namespace_thread
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_namespace_thread
+            end
           end
-          assert_equal(3, @stats[:namespace_watch_failures])
-          assert_equal(2, Thread.current[:namespace_watch_retry_count])
-          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
-          assert_nil(@stats[:namespace_watch_error_type_notices])
+          assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+          assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+          assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
         end
       end
     end
 
-    test 'namespace watch retries when error is received' do
+    test 'namespace watch resets watch retry count when error is received and connection to k8s API server is re-established' do
       @client.stub :get_namespaces, @initial do
         @client.stub :watch_namespaces, [@error] do
-          assert_raise Fluent::UnrecoverableError do
-            set_up_namespace_thread
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_namespace_thread
+            end
           end
-          assert_equal(3, @stats[:namespace_watch_failures])
-          assert_equal(2, Thread.current[:namespace_watch_retry_count])
-          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
-          assert_equal(3, @stats[:namespace_watch_error_type_notices])
+          assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+          assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+          assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
         end
       end
     end
@@ -179,4 +215,30 @@ class WatchNamespacesTestTest < WatchTest
         end
       end
     end
+
+    test 'namespace watch raises a GoneError when a 410 Gone error is received' do
+      @cache['gone_uid'] = {}
+      @client.stub :watch_namespaces, [@gone] do
+        assert_raise KubernetesMetadata::Common::GoneError do
+          process_namespace_watcher_notices(start_namespace_watch)
+        end
+        assert_equal(1, @stats[:namespace_watch_gone_notices])
+      end
+    end
+
+    test 'namespace watch retries when 410 Gone errors are encountered' do
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [@created, @gone, @modified] do
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_namespace_thread
+            end
+          end
+          assert_operator(@stats[:namespace_watch_gone_errors], :>=, 3)
+          assert_operator(@stats[:namespace_watch_gone_notices], :>=, 3)
+        end
+      end
+    end
 end