fluent-plugin-kubernetes_metadata_filter 2.5.0 → 2.7.2

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,11 +18,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
   module WatchPods
-
     include ::KubernetesMetadata::Common
 
     def set_up_pod_thread
@@ -28,6 +30,7 @@ module KubernetesMetadata
       # Fluent:ConfigError, so that users can inspect potential errors in
       # the configuration.
       pod_watcher = start_pod_watch
+
       Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
       Thread.current[:pod_watch_retry_count] = 0
 
@@ -35,42 +38,47 @@ module KubernetesMetadata
       # processing will be swallowed and retried. These failures /
       # exceptions could be caused by Kubernetes API being temporarily
       # down. We assume the configuration is correct at this point.
-      while true
-        begin
-          pod_watcher ||= get_pods_and_start_watcher
-          process_pod_watcher_notices(pod_watcher)
-        rescue Exception => e
-          @stats.bump(:pod_watch_failures)
-          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
-            # Instead of raising exceptions and crashing Fluentd, swallow
-            # the exception and reset the watcher.
-            log.info(
-              "Exception encountered parsing pod watch event. The " \
-              "connection might have been closed. Sleeping for " \
-              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
-              "seconds and resetting the pod watcher.", e)
-            sleep(Thread.current[:pod_watch_retry_backoff_interval])
-            Thread.current[:pod_watch_retry_count] += 1
-            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
-            pod_watcher = nil
-          else
-            # Since retries failed for many times, log as errors instead
-            # of info and raise exceptions and trigger Fluentd to restart.
-            message =
-              "Exception encountered parsing pod watch event. The " \
-              "connection might have been closed. Retried " \
-              "#{@watch_retry_max_times} times yet still failing. Restarting."
-            log.error(message, e)
-            raise Fluent::UnrecoverableError.new(message)
-          end
+      loop do
+        pod_watcher ||= get_pods_and_start_watcher
+        process_pod_watcher_notices(pod_watcher)
+      rescue GoneError => e
+        # Expected error. Quietly go back through the loop in order to
+        # start watching from the latest resource versions
+        @stats.bump(:pod_watch_gone_errors)
+        log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
+        pod_watcher = nil
+      rescue StandardError => e
+        @stats.bump(:pod_watch_failures)
+        if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+          # Instead of raising exceptions and crashing Fluentd, swallow
+          # the exception and reset the watcher.
+          log.info(
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Sleeping for ' \
+            "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+            'seconds and resetting the pod watcher.', e
+          )
+          sleep(Thread.current[:pod_watch_retry_backoff_interval])
+          Thread.current[:pod_watch_retry_count] += 1
+          Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+          pod_watcher = nil
+        else
+          # Since retries failed for many times, log as errors instead
+          # of info and raise exceptions and trigger Fluentd to restart.
+          message =
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Retried ' \
+            "#{@watch_retry_max_times} times yet still failing. Restarting."
+          log.error(message, e)
+          raise Fluent::UnrecoverableError, message
         end
       end
     end
 
     def start_pod_watch
       get_pods_and_start_watcher
-    rescue Exception => e
-      message = "start_pod_watch: Exception encountered setting up pod watch " \
+    rescue StandardError => e
+      message = 'start_pod_watch: Exception encountered setting up pod watch ' \
                 "from Kubernetes API #{@apiVersion} endpoint " \
                 "#{@kubernetes_url}: #{e.message}"
       message += " (#{e.response})" if e.respond_to?(:response)
@@ -83,19 +91,27 @@ module KubernetesMetadata
     # from that resourceVersion.
     def get_pods_and_start_watcher
       options = {
-        resource_version: '0'  # Fetch from API server.
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       if ENV['K8S_NODE_NAME']
         options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
       end
-      pods = @client.get_pods(options)
-      pods.each do |pod|
-        cache_key = pod.metadata['uid']
-        @cache[cache_key] = parse_pod_metadata(pod)
-        @stats.bump(:pod_cache_host_updates)
+      if @last_seen_resource_version
+        options[:resource_version] = @last_seen_resource_version
+      else
+        pods = @client.get_pods(options)
+        pods[:items].each do |pod|
+          cache_key = pod[:metadata][:uid]
+          @cache[cache_key] = parse_pod_metadata(pod)
+          @stats.bump(:pod_cache_host_updates)
+        end
+
+        # continue watching from most recent resourceVersion
+        options[:resource_version] = pods[:metadata][:resourceVersion]
       end
-      options[:resource_version] = pods.resourceVersion
+
       watcher = @client.watch_pods(options)
+      reset_pod_watch_retry_stats
       watcher
     end
 
@@ -109,34 +125,46 @@ module KubernetesMetadata
     # Process a watcher notice and potentially raise an exception.
     def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
-          when 'MODIFIED'
-            reset_pod_watch_retry_stats
-            cache_key = notice.object['metadata']['uid']
-            cached    = @cache[cache_key]
-            if cached
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_host_updates)
-            else
-              @stats.bump(:pod_cache_watch_misses)
-            end
-          when 'DELETED'
-            reset_pod_watch_retry_stats
-            # ignore and let age out for cases where pods
-            # deleted but still processing logs
-            @stats.bump(:pod_cache_watch_delete_ignored)
-          when 'ERROR'
+        # store version we processed to not reprocess it ... do not unset when there is no version in response
+        version = ( # TODO: replace with &.dig once we are on ruby 2.5+
+          notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion]
+        )
+        @last_seen_resource_version = version if version
+
+        case notice[:type]
+        when 'MODIFIED'
+          reset_pod_watch_retry_stats
+          cache_key = notice.dig(:object, :metadata, :uid)
+          cached    = @cache[cache_key]
+          if cached
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_watch_updates)
+          elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName]
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_host_updates)
+          else
+            @stats.bump(:pod_cache_watch_misses)
+          end
+        when 'DELETED'
+          reset_pod_watch_retry_stats
+          # ignore and let age out for cases where pods
+          # deleted but still processing logs
+          @stats.bump(:pod_cache_watch_delete_ignored)
+        when 'ERROR'
+          if notice[:object] && notice[:object][:code] == 410
+            @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset
+            @stats.bump(:pod_watch_gone_notices)
+            raise GoneError
+          else
             @stats.bump(:pod_watch_error_type_notices)
-            message = notice['object']['message'] if notice['object'] && notice['object']['message']
+            message = notice[:object][:message] if notice[:object] && notice[:object][:message]
             raise "Error while watching pods: #{message}"
-          else
-            reset_pod_watch_retry_stats
-            # Don't pay attention to creations, since the created pod may not
-            # end up on this node.
-            @stats.bump(:pod_cache_watch_ignored)
+          end
+        else
+          reset_pod_watch_retry_stats
+          # Don't pay attention to creations, since the created pod may not
+          # end up on this node.
+          @stats.bump(:pod_cache_watch_ignored)
         end
       end
     end

data/test/helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,6 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'bundler/setup'
 require 'codeclimate-test-reporter'
 SimpleCov.start do
   formatter SimpleCov::Formatter::MultiFormatter.new [
@@ -31,8 +34,14 @@ require 'fileutils'
 require 'fluent/log'
 require 'fluent/test'
 require 'minitest/autorun'
-require 'webmock/test_unit'
 require 'vcr'
+require 'ostruct'
+require 'fluent/plugin/filter_kubernetes_metadata'
+require 'fluent/test/driver/filter'
+require 'kubeclient'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!
 
 VCR.configure do |config|
   config.cassette_library_dir = 'test/cassettes'
@@ -58,7 +67,16 @@ def ipv6_enabled?
   begin
     TCPServer.open('::1', 0)
     true
-  rescue
+  rescue StandardError
     false
   end
 end
+
+# TEST_NAME='foo' ruby test_file.rb to run a single test case
+if ENV['TEST_NAME']
+  (class << Test::Unit::TestCase; self; end).prepend(Module.new do
+    def test(name)
+      super if name == ENV['TEST_NAME']
+    end
+  end)
+end

data/test/plugin/test_cache_stats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -17,20 +19,15 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'fluent/plugin/kubernetes_metadata_stats'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
-  
-    test 'watch stats' do
-      require 'lru_redux'
-      stats = KubernetesMetadata::Stats.new
-      stats.bump(:missed)
-      stats.bump(:deleted)
-      stats.bump(:deleted)
+  test 'watch stats' do
+    require 'lru_redux'
+    stats = KubernetesMetadata::Stats.new
+    stats.bump(:missed)
+    stats.bump(:deleted)
+    stats.bump(:deleted)
 
-      assert_equal("stats - deleted: 2, missed: 1", stats.to_s)
-    end
-    
+    assert_equal('stats - deleted: 2, missed: 1', stats.to_s)
+  end
 end

data/test/plugin/test_cache_strategy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -17,180 +19,176 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
-require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
-require 'lru_redux'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class TestCacheStrategy
-    include KubernetesMetadata::CacheStrategy
-
-    def initialize
-      @stats = KubernetesMetadata::Stats.new
-      @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
-      @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
-      @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
-      @orphaned_namespace_name = '.orphaned'
-      @orphaned_namespace_id = 'orphaned'
+  include KubernetesMetadata::CacheStrategy
+
+  def initialize
+    @stats = KubernetesMetadata::Stats.new
+    @cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
+    @id_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
+    @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
+    @orphaned_namespace_name = '.orphaned'
+    @orphaned_namespace_id = 'orphaned'
+  end
+
+  attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
+
+  def fetch_pod_metadata(_namespace_name, _pod_name)
+    {}
+  end
+
+  def fetch_namespace_metadata(_namespace_name)
+    {}
+  end
+
+  def log
+    logger = {}
+    def logger.trace?
+      true
     end
 
-    attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
-
-    def fetch_pod_metadata(namespace_name, pod_name)
+    def logger.trace(message)
     end
-
-    def fetch_namespace_metadata(namespace_name)
-    end
-
-    def log
-        logger = {}
-        def logger.trace?
-          true
-        end
-        def logger.trace(message)
-        end
-        logger
-    end
-
+    logger
+  end
 end
 
 class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
-  
-    def setup
-       @strategy = TestCacheStrategy.new
-       @cache_key = 'some_long_container_id'
-       @namespace_name = 'some_namespace_name'
-       @namespace_uuid = 'some_namespace_uuid'
-       @pod_name = 'some_pod_name'
-       @pod_uuid = 'some_pod_uuid'
-       @time = Time.now
-       @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}}
-       @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s}
+  def setup
+    @strategy = TestCacheStrategy.new
+    @cache_key = 'some_long_container_id'
+    @namespace_name = 'some_namespace_name'
+    @namespace_uuid = 'some_namespace_uuid'
+    @pod_name = 'some_pod_name'
+    @pod_uuid = 'some_pod_uuid'
+    @time = Time.now
+    @pod_meta = { 'pod_id' => @pod_uuid, 'labels' => { 'meta' => 'pod' } }
+    @namespace_meta = { 'namespace_id' => @namespace_uuid, 'creation_timestamp' => @time.to_s }
+  end
+
+  test 'when cached metadata is found' do
+    exp = @pod_meta.merge(@namespace_meta)
+    exp.delete('creation_timestamp')
+    @strategy.id_cache[@cache_key] = {
+      pod_id: @pod_uuid,
+      namespace_id: @namespace_uuid
+    }
+    @strategy.cache[@pod_uuid] = @pod_meta
+    @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
+    assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
+  end
+
+  test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
+    exp = {
+      'pod_id' => @pod_uuid,
+      'namespace_id' => @namespace_uuid
+    }
+    @strategy.id_cache[@cache_key] = {
+      pod_id: @pod_uuid,
+      namespace_id: @namespace_uuid
+    }
+    @strategy.stub :fetch_pod_metadata, {} do
+      @strategy.stub :fetch_namespace_metadata, nil do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
+      end
     end
-
-    test 'when cached metadata is found' do
-        exp = @pod_meta.merge(@namespace_meta)
-        exp.delete('creation_timestamp')
-        @strategy.id_cache[@cache_key] = {
-            pod_id: @pod_uuid,
-            namespace_id: @namespace_uuid
-        }
-        @strategy.cache[@pod_uuid] = @pod_meta
-        @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
-        assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
+  end
+
+  test 'when metadata is not cached and is fetched' do
+    exp = @pod_meta.merge(@namespace_meta)
+    exp.delete('creation_timestamp')
+    @strategy.stub :fetch_pod_metadata, @pod_meta do
+      @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
+        assert_true(@strategy.id_cache.key?(@cache_key))
+      end
     end
-
-    test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
-        exp = {
-            'pod_id'=> @pod_uuid,
-            'namespace_id'=> @namespace_uuid
-        }
-        @strategy.id_cache[@cache_key] = {
-            pod_id: @pod_uuid,
-            namespace_id: @namespace_uuid
-        }
-        @strategy.stub :fetch_pod_metadata, {} do
-            @strategy.stub :fetch_namespace_metadata, nil do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
-            end
-        end
+  end
+
+  test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
+    # this is the case for a record from a deleted pod where no other
+    # records were read.  using the container hash since that is all
+    # we ever will have and should allow us to process all the deleted
+    # pod records
+    exp = {
+      'pod_id' => @cache_key,
+      'namespace_id' => @namespace_uuid
+    }
+    @strategy.stub :fetch_pod_metadata, {} do
+      @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
+        assert_true(@strategy.id_cache.key?(@cache_key))
+      end
     end
-
-    test 'when metadata is not cached and is fetched' do
-        exp = @pod_meta.merge(@namespace_meta)
-        exp.delete('creation_timestamp')
-        @strategy.stub :fetch_pod_metadata, @pod_meta do
-            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
-                assert_true(@strategy.id_cache.key?(@cache_key))
-            end
-        end
-    end
-
-    test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
-        # this is the case for a record from a deleted pod where no other
-        # records were read.  using the container hash since that is all
-        # we ever will have and should allow us to process all the deleted
-        # pod records
-        exp = {
-            'pod_id'=> @cache_key, 
-            'namespace_id'=> @namespace_uuid
-        }
-        @strategy.stub :fetch_pod_metadata, {} do
-            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
-                assert_true(@strategy.id_cache.key?(@cache_key))
-            end
-        end
-    end
-
-    test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
-        # this is the case for a record from a deleted pod from a deleted namespace
-        # where new namespace was created with the same name
-        exp = {
-            'namespace_id'=> @namespace_uuid
-        }
-        @strategy.stub :fetch_pod_metadata, {} do
-            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
-                assert_true(@strategy.id_cache.key?(@cache_key))
-            end
-        end
+  end
+
+  test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
+    # this is the case for a record from a deleted pod from a deleted namespace
+    # where new namespace was created with the same name
+    exp = {
+      'namespace_id' => @namespace_uuid
+    }
+    @strategy.stub :fetch_pod_metadata, {} do
+      @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
+        assert_true(@strategy.id_cache.key?(@cache_key))
+      end
     end
-
-    test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
-        # we should never see this since pod meta should not be retrievable
-        # unless the namespace exists
-        @strategy.stub :fetch_pod_metadata, @pod_meta do
-            @strategy.stub :fetch_namespace_metadata, {} do
-                assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
-            end
-        end
+  end
+
+  test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
+    # we should never see this since pod meta should not be retrievable
+    # unless the namespace exists
+    @strategy.stub :fetch_pod_metadata, @pod_meta do
+      @strategy.stub :fetch_namespace_metadata, {} do
+        assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
+      end
     end
-
-    test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
-        # we should never see this since pod meta should not be retrievable
-        # unless the namespace exists
-        @strategy.allow_orphans = true
-        exp = {
-            'orphaned_namespace' => 'namespace',
-            'namespace_name' => '.orphaned',
-            'namespace_id' => 'orphaned'
-        }
-        @strategy.stub :fetch_pod_metadata, @pod_meta do
-            @strategy.stub :fetch_namespace_metadata, {} do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
-            end
-        end
+  end
+
+  test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
+    # we should never see this since pod meta should not be retrievable
+    # unless the namespace exists
+    @strategy.allow_orphans = true
+    exp = {
+      'orphaned_namespace' => 'namespace',
+      'namespace_name' => '.orphaned',
+      'namespace_id' => 'orphaned'
+    }
+    @strategy.stub :fetch_pod_metadata, @pod_meta do
+      @strategy.stub :fetch_namespace_metadata, {} do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
+      end
     end
-
-    test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
-        # processing a batch of records with no meta. ideally we only hit the api server once
-        batch_miss_cache = {}
-        @strategy.stub :fetch_pod_metadata, {} do
-            @strategy.stub :fetch_namespace_metadata, {} do
-                assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
-            end
-        end
-        assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+  end
+
+  test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
+    # processing a batch of records with no meta. ideally we only hit the api server once
+    batch_miss_cache = {}
+    @strategy.stub :fetch_pod_metadata, {} do
+      @strategy.stub :fetch_namespace_metadata, {} do
+        assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
+      end
     end
-    
-    test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
-        # we should never see this since pod meta should not be retrievable
-        # unless the namespace exists
-        @strategy.allow_orphans = true
-        exp = {
-            'orphaned_namespace' => 'namespace',
-            'namespace_name' => '.orphaned',
-            'namespace_id' => 'orphaned'
-        }
-        batch_miss_cache = {}
-        @strategy.stub :fetch_pod_metadata, {}  do
-            @strategy.stub :fetch_namespace_metadata, {} do
-                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
-            end
-        end
-        assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+    assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
+  end
+
+  test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
+    # we should never see this since pod meta should not be retrievable
+    # unless the namespace exists
+    @strategy.allow_orphans = true
+    exp = {
+      'orphaned_namespace' => 'namespace',
+      'namespace_name' => '.orphaned',
+      'namespace_id' => 'orphaned'
+    }
+    batch_miss_cache = {}
+    @strategy.stub :fetch_pod_metadata, {} do
+      @strategy.stub :fetch_namespace_metadata, {} do
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
+      end
     end
+    assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
+  end
 end