RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 1.2.2 → 2.0.0 - Mend

fluent-plugin-kubernetes_metadata_filter 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (15) hide show

checksums.yaml +5 -5
data/Gemfile +1 -5
data/README.md +3 -35
data/circle.yml +2 -0
data/fluent-plugin-kubernetes_metadata_filter.gemspec +4 -13
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +116 -79
data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +1 -1
data/lib/fluent/plugin/kubernetes_metadata_common.rb +0 -46
data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +0 -3
data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml +1 -1
data/test/plugin/test_filter_kubernetes_metadata.rb +247 -345
data/test/plugin/test_watch_pods.rb +3 -46
metadata +13 -10
data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -408
data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -540

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb CHANGED

@@ -18,6 +18,7 @@
 #
 module KubernetesMetadata
   module CacheStrategy
     def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
       metadata = {}
       ids = @id_cache[key]
@@ -83,7 +84,6 @@ module KubernetesMetadata
         end
         @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
       end
       # remove namespace info that is only used for comparison
       metadata.delete('creation_timestamp')
       metadata.delete_if{|k,v| v.nil?}

data/lib/fluent/plugin/kubernetes_metadata_common.rb CHANGED

@@ -54,30 +54,10 @@ module KubernetesMetadata
         self.de_dot!(labels)
         self.de_dot!(annotations)
       end
-      # collect container informations
-      container_meta = {}
-      begin
-        pod_object['status']['containerStatuses'].each do|container_status|
-          # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
-          container_meta[container_id] = {
-              'name' => container_status['name'],
-              'image' => container_status['image'],
-              'image_id' => container_status['imageID']
-          }
-        end
-      rescue
-        log.on_debug do
-          log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
-        end
-      end
       kubernetes_metadata = {
           'namespace_name' => pod_object['metadata']['namespace'],
           'pod_id'         => pod_object['metadata']['uid'],
           'pod_name'       => pod_object['metadata']['name'],
-          'containers'     => syms_to_strs(container_meta),
           'labels'         => labels,
           'host'           => pod_object['spec']['nodeName'],
           'master_url'     => @kubernetes_url
@@ -103,29 +83,3 @@ module KubernetesMetadata
   end
 end
-# copied from activesupport
-class Object
-  # An object is blank if it's false, empty, or a whitespace string.
-  # For example, +nil+, '', '   ', [], {}, and +false+ are all blank.
-  #
-  # This simplifies
-  #
-  #   !address || address.empty?
-  #
-  # to
-  #
-  #   address.blank?
-  #
-  # @return [true, false]
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-  # An object is present if it's not blank.
-  #
-  # @return [true, false]
-  def present?
-    !blank?
-  end
-end

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb CHANGED

@@ -42,9 +42,6 @@ module KubernetesMetadata
             if cached
               @cache[cache_key] = parse_pod_metadata(notice.object)
               @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_host_updates)
             else
               @stats.bump(:pod_cache_watch_misses)
             end

data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml CHANGED

@@ -174,7 +174,7 @@ http_interactions:
                 "restartCount": 2,
                 "image": "fabric8/hawtio-kubernetes:latest",
                 "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
-                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+                "containerID": "docker://1b1d1f61c1205fe73328c75b2945e2ce05acfba2fde16299a8103fb22e9ec58a"
               },
               {
                 "name": "POD",

data/test/plugin/test_filter_kubernetes_metadata.rb CHANGED

@@ -17,6 +17,7 @@
 # limitations under the License.
 #
 require_relative '../helper'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_kubernetes_metadata'
 require 'webmock/test_unit'
@@ -30,8 +31,10 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     @time = Fluent::Engine.now
   end
+  DEFAULT_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(KubernetesMetadataFilter, 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log').configure(conf, true)
+    Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf)
   end
   sub_test_case 'configure' do
@@ -84,8 +87,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           Dir.mktmpdir { |dir|
             # Fake token file and CA crt.
-            expected_cert_path = File.join(dir, KubernetesMetadataFilter::K8_POD_CA_CERT)
-            expected_token_path = File.join(dir, KubernetesMetadataFilter::K8_POD_TOKEN)
+            expected_cert_path = File.join(dir, Plugin::KubernetesMetadataFilter::K8_POD_CA_CERT)
+            expected_token_path = File.join(dir, Plugin::KubernetesMetadataFilter::K8_POD_TOKEN)
             File.open(expected_cert_path, "w") {}
             File.open(expected_token_path, "w") {}
@@ -137,9 +140,16 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ', d: nil)
       d = create_driver(config) if d.nil?
-      d.run {
-        d.emit(msg, @time)
-      }.filtered
+      if ENV['LOGLEVEL']
+        logger = Logger.new(STDOUT)
+        logger.level = eval("Logger::#{ENV['LOGLEVEL'].upcase}")
+        instance = d.instance
+        instance.instance_variable_set(:@log,logger)
+      end
+      d.run(default_tag: DEFAULT_TAG) {
+        d.feed(@time, msg)
+      }
+      d.filtered.map{|e| e.last}
     end
     def emit_with_tag(tag, msg={}, config='
@@ -148,45 +158,10 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ')
       d = create_driver(config)
-      d.run {
-        d.emit_with_tag(tag, msg, @time)
-      }.filtered
-    end
-    test 'nil event stream' do
-     #not certain how this is possible but adding test to properly
-     #guard against this condition we have seen - test for nil,
-     #empty, no empty method, not an event stream
-     plugin = create_driver.instance
-     plugin.filter_stream('tag', nil)
-     plugin.filter_stream('tag', Fluent::MultiEventStream.new)
-    end
-    test 'inability to connect to the api server handles exception and doensnt block pipeline' do
-      VCR.use_cassette('kubernetes_docker_metadata') do
-        driver = create_driver('
-          kubernetes_url https://localhost:8443
-          watch false
-          cache_size 1
-        ')
-        stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_raise(SocketError.new('error from pod fetch'))
-        stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_raise(SocketError.new('socket error from namespace fetch'))
-        es = emit({'time'=>'2015-05-08T09:22:01Z'}, '', :d => driver)
-        expected_kube_metadata = {
-          'time'=>'2015-05-08T09:22:01Z',
-          'docker' => {
-              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
-          },
-          'kubernetes' => {
-            'pod_name'       => 'fabric8-console-controller-98rqc',
-            'container_name' => 'fabric8-console-container',
-            "namespace_id"=>"orphaned",
-            'namespace_name' => '.orphaned',
-            "orphaned_namespace"=>"default"
-          }
-        }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
-      end
+      d.run(default_tag: tag) {
+        d.feed(@time, msg)
+      }
+      d.filtered.map{|e| e.last}
     end
     test 'with docker & kubernetes metadata where id cache hit and metadata miss' do
@@ -203,22 +178,22 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         }
         stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_timeout
         stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
-        es = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver)
+        filtered = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver)
         expected_kube_metadata = {
           'time'=>'2015-05-08T09:22:01Z',
           'docker' => {
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'pod_name'        => 'fabric8-console-controller-98rqc',
-            'container_name'  => 'fabric8-console-container',
-            'namespace_name'  => 'default',
-            'namespace_id'    => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'          => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
@@ -234,63 +209,59 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             :pod_id       =>'c76927af-f563-11e4-b32d-54ee7527188d',
             :namespace_id =>'898268c8-4a36-11e5-9d81-42010af0194c'
         }
-        es = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver)
+        filtered = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver)
         expected_kube_metadata = {
           'time'=>'2015-05-08T09:22:01Z',
           'docker' => {
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with docker & kubernetes metadata' do
       VCR.use_cassette('kubernetes_docker_metadata') do
-        es = emit({'time'=>'2015-05-08T09:22:01Z'})
+        filtered = emit({'time'=>'2015-05-08T09:22:01Z'})
         expected_kube_metadata = {
           'time'=>'2015-05-08T09:22:01Z',
           'docker' => {
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with docker & kubernetes metadata & namespace_id enabled' do
       VCR.use_cassette('metadata_with_namespace_id') do
-        es = emit({}, '
+        filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -300,27 +271,25 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with docker & kubernetes metadata using bearer token' do
       VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
-        es = emit({}, '
+        filtered = emit({}, '
           kubernetes_url https://localhost:8443
           verify_ssl false
           watch false
@@ -331,37 +300,35 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with docker & kubernetes metadata but no configured api server' do
-      es = emit({}, '')
+      filtered = emit({}, '')
       expected_kube_metadata = {
           'docker' => {
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-              'pod_name'        => 'fabric8-console-controller-98rqc',
-              'container_name'  => 'fabric8-console-container',
-              'namespace_name'  => 'default',
+              'pod_name'       => 'fabric8-console-controller-98rqc',
+              'container_name' => 'fabric8-console-container',
+              'namespace_name' => 'default',
           }
       }
-      assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      assert_equal(expected_kube_metadata, filtered[0])
     end
     test 'with docker & inaccessible kubernetes metadata' do
@@ -372,20 +339,20 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       )
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_timeout
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
-      es = emit()
+      filtered = emit()
       expected_kube_metadata = {
         'docker' => {
           'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
-          'pod_name'           => 'fabric8-console-controller-98rqc',
-          'container_name'     => 'fabric8-console-container',
-          'namespace_name'     => '.orphaned',
+          'pod_name'       => 'fabric8-console-controller-98rqc',
+          'container_name' => 'fabric8-console-container',
+          'namespace_name' => '.orphaned',
           'orphaned_namespace' => 'default',
-          'namespace_id'       => 'orphaned'
+          'namespace_id' => 'orphaned'
         }
       }
-      assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      assert_equal(expected_kube_metadata, filtered[0])
     end
     test 'with dot in pod name' do
@@ -395,23 +362,62 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         }.to_json
       )
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout
-      es = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
+      filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
       expected_kube_metadata = {
         'docker' => {
           'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
-          'pod_name'        => 'fabric8-console-controller.98rqc',
-          'container_name'  => 'fabric8-console-container',
-          'namespace_name'  => 'default'
+          'pod_name'       => 'fabric8-console-controller.98rqc',
+          'container_name' => 'fabric8-console-container',
+          'namespace_name' => 'default'
         }
       }
-      assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      assert_equal(expected_kube_metadata, filtered[0])
     end
     test 'with docker metadata, non-kubernetes' do
-      es = emit_with_tag('non-kubernetes', {}, '')
-      assert_false(es.instance_variable_get(:@record_array)[0].has_key?(:kubernetes))
+      filtered = emit_with_tag('non-kubernetes', {}, '')
+      assert_false(filtered[0].has_key?(:kubernetes))
+    end
+    test 'merges json log data' do
+      json_log = {
+        'hello' => 'world'
+      }
+      msg = {
+        'log' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, '')
+      assert_equal(msg.merge(json_log), filtered[0])
+    end
+    test 'merges json log data in MESSAGE' do
+      json_log = {
+        'hello' => 'world'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
+      assert_equal(msg.merge(json_log), filtered[0])
+    end
+    test 'merges json log data with message field' do
+      json_log = {
+        'timeMillis' => 1459853347608,
+        'thread' => 'main',
+        'level' => 'INFO',
+        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
+        'message' => 'Total 1 routes, of which 1 is started.',
+        'endOfBatch' => false,
+        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
+      }
+      msg = {
+        'log' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, '')
+      assert_equal(msg.merge(json_log), filtered[0])
     end
     test 'ignores invalid json in log field' do
@@ -419,13 +425,57 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       msg = {
           'log' => json_log
       }
-      es = emit_with_tag('non-kubernetes', msg, '')
-      assert_equal(msg, es.instance_variable_get(:@record_array)[0])
+      filtered = emit_with_tag('non-kubernetes', msg, '')
+      assert_equal(msg, filtered[0])
+    end
+    test 'merges json log data with message field in MESSAGE' do
+      json_log = {
+        'timeMillis' => 1459853347608,
+        'thread' => 'main',
+        'level' => 'INFO',
+        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
+        'message' => 'Total 1 routes, of which 1 is started.',
+        'endOfBatch' => false,
+        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
+      assert_equal(msg.merge(json_log), filtered[0])
+    end
+    test 'emit individual fields from json, throw out whole original string' do
+      json_log = {
+        'hello' => 'world',
+        'more' => 'data'
+      }
+      msg = {
+        'log' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, 'preserve_json_log false')
+      assert_equal(json_log, filtered[0])
+    end
+    test 'emit individual fields from json, throw out whole original string in MESSAGE' do
+      json_log = {
+        'hello' => 'world',
+        'more' => 'data'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      filtered = emit_with_tag('non-kubernetes', msg, '
+preserve_json_log false
+use_journal true
+')
+      assert_equal(json_log, filtered[0])
     end
     test 'with kubernetes dotted labels, de_dot enabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
-        es = emit({}, '
+        filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -435,30 +485,28 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'namespace_labels'   => {
+            'host'             => 'jimmi-redhat.localnet',
+            'pod_name'         => 'fabric8-console-controller-98rqc',
+            'container_name'   => 'fabric8-console-container',
+            'namespace_id'     => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
               'kubernetes_io/namespacetest' => 'somevalue'
             },
-            'namespace_name'     => 'default',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'namespace_name'   => 'default',
+            'pod_id'           => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'       => 'https://localhost:8443',
             'labels' => {
               'kubernetes_io/test' => 'somevalue'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with kubernetes dotted labels, de_dot disabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
-        es = emit({}, '
+        filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -469,24 +517,22 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'namespace_labels'   => {
+            'host'             => 'jimmi-redhat.localnet',
+            'pod_name'         => 'fabric8-console-controller-98rqc',
+            'container_name'   => 'fabric8-console-container',
+            'namespace_id'     => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'namespace_labels' => {
               'kubernetes.io/namespacetest' => 'somevalue'
             },
-            'namespace_name'     => 'default',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'namespace_name'   => 'default',
+            'pod_id'           => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'       => 'https://localhost:8443',
             'labels' => {
               'kubernetes.io/test' => 'somevalue'
             }
           }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
@@ -507,7 +553,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'randomfield' => 'randomvalue'
       }
       VCR.use_cassette('kubernetes_docker_metadata') do
-        es = emit_with_tag(tag, msg, '
+        filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -518,21 +564,19 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }.merge(msg)
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
@@ -545,7 +589,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'randomfield' => 'randomvalue'
       }
       VCR.use_cassette('metadata_with_namespace_id') do
-        es = emit_with_tag(tag, msg, '
+        filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -556,159 +600,25 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }.merge(msg)
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
-      end
-    end
-    test 'with records from journald and docker & kubernetes metadata with use_journal unset' do
-      # with use_journal unset, should still use the journal fields instead of tag fields
-      tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
-      msg = {
-        'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
-        'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
-        'kubernetes' => {
-          'namespace_name' => 'k8s-namespace-name',
-          'pod_name' => 'k8s-pod-name',
-          'container_name' => 'k8s-container-name'
-        },
-        'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'},
-        'randomfield' => 'randomvalue'
-      }
-      VCR.use_cassette('metadata_from_tag_journald_and_kubernetes_fields') do
-        es = emit_with_tag(tag, msg, '
-          kubernetes_url https://localhost:8443
-          watch false
-          cache_size 1
-        ')
-        expected_kube_metadata = {
-          'docker' => {
-              'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'
-          },
-          'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'k8s-pod-name',
-            'container_name'     => 'k8s-container-name',
-            'container_image'    => 'k8s-container-image:latest',
-            'container_image_id' => 'docker://d78c5217c41e9af08d37d9ae2cb070afa1fe3da6bc77bfb18879a8b4bfdf8a34',
-            'namespace_name'     => 'k8s-namespace-name',
-            'namespace_id'       => '8e0dc8fc-59f2-49f7-a3e2-ed0913e19d9f',
-            'pod_id'             => 'ebabf749-5fcd-4750-a3f0-aedd89476da8',
-            'master_url'         => 'https://localhost:8443',
-            'labels' => {
-              'component' => 'k8s-test'
-            }
-          }
-        }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval}
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
-      end
-    end
-    test 'with records from journald and docker & kubernetes metadata with lookup_from_k8s_field false' do
-      # with use_journal unset, should still use the journal fields instead of tag fields
-      tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
-      msg = {
-        'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
-        'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
-        'kubernetes' => {
-          'namespace_name' => 'k8s-namespace-name',
-          'pod_name' => 'k8s-pod-name',
-          'container_name' => 'k8s-container-name'
-        },
-        'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'},
-        'randomfield' => 'randomvalue'
-      }
-      VCR.use_cassette('metadata_from_tag_and_journald_fields') do
-        es = emit_with_tag(tag, msg, '
-          kubernetes_url https://localhost:8443
-          watch false
-          cache_size 1
-          lookup_from_k8s_field false
-        ')
-        expected_kube_metadata = {
-          'docker' => {
-              'container_id' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b'
-          },
-          'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'journald-pod-name',
-            'container_name'     => 'journald-container-name',
-            'container_image'    => 'journald-container-image:latest',
-            'container_image_id' => 'docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47',
-            'namespace_name'     => 'journald-namespace-name',
-            'namespace_id'       => '8282888f-733f-4f23-a3d3-1fdfa3bdacf2',
-            'pod_id'             => '5e1c1e27-b637-4e81-80b6-6d8a8c404d3b',
-            'master_url'         => 'https://localhost:8443',
-            'labels' => {
-              'component' => 'journald-test'
-            }
-          }
-        }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval}
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
-      end
-    end
-    test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
-      # with use_journal unset, should still use the journal fields instead of tag fields
-      tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
-      msg = {
-        'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
-        'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
-        'kubernetes' => {
-          'namespace_name' => 'k8s-namespace-name',
-          'pod_name' => 'k8s-pod-name',
-          'container_name' => 'k8s-container-name'
-        },
-        'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'},
-        'randomfield' => 'randomvalue'
-      }
-      VCR.use_cassette('metadata_from_tag_and_journald_fields') do
-        es = emit_with_tag(tag, msg, '
-          kubernetes_url https://localhost:8443
-          watch false
-          cache_size 1
-          lookup_from_k8s_field false
-          use_journal false
-        ')
-        expected_kube_metadata = {
-          'docker' => {
-              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
-          },
-          'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
-            'labels' => {
-              'component' => 'fabric8Console'
-            }
-          }
-        }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval}
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with kubernetes annotations' do
       VCR.use_cassette('kubernetes_docker_metadata_annotations') do
-        es = emit({},'
+        filtered = emit({},'
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -719,16 +629,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
                 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
             },
             'kubernetes' => {
-                'host'               => 'jimmi-redhat.localnet',
-                'pod_name'           => 'fabric8-console-controller-98rqc',
-                'container_name'     => 'fabric8-console-container',
-                'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-                'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-                'namespace_name'     => 'default',
-                'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-                'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-                'master_url'         => 'https://localhost:8443',
-                'labels'             => {
+                'host'           => 'jimmi-redhat.localnet',
+                'pod_name'       => 'fabric8-console-controller-98rqc',
+                'container_name' => 'fabric8-console-container',
+                'namespace_name' => 'default',
+                'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+                'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+                'master_url'     => 'https://localhost:8443',
+                'labels'         => {
                     'component' => 'fabric8Console'
                 },
                 'annotations'    => {
@@ -737,7 +645,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
                 }
             }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
@@ -750,7 +658,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'randomfield' => 'randomvalue'
       }
       VCR.use_cassette('kubernetes_docker_metadata') do
-        es = emit_with_tag(tag, msg, '
+        filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -761,27 +669,25 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
               'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
-            'host'               => 'jimmi-redhat.localnet',
-            'pod_name'           => 'fabric8-console-controller-98rqc',
-            'container_name'     => 'fabric8-console-container',
-            'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-            'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-            'namespace_name'     => 'default',
-            'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-            'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-            'master_url'         => 'https://localhost:8443',
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'master_url'     => 'https://localhost:8443',
             'labels' => {
               'component' => 'fabric8Console'
             }
           }
         }.merge(msg)
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with kubernetes namespace annotations' do
       VCR.use_cassette('kubernetes_docker_metadata_annotations') do
-        es = emit({},'
+        filtered = emit({},'
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -792,16 +698,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
                 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
             },
             'kubernetes' => {
-                'host'               => 'jimmi-redhat.localnet',
-                'pod_name'           => 'fabric8-console-controller-98rqc',
-                'container_name'     => 'fabric8-console-container',
-                'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-                'namespace_name'     => 'default',
-                'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-                'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-                'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-                'master_url'         => 'https://localhost:8443',
-                'labels'             => {
+                'host'           => 'jimmi-redhat.localnet',
+                'pod_name'       => 'fabric8-console-controller-98rqc',
+                'container_name' => 'fabric8-console-container',
+                'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+                'namespace_name' => 'default',
+                'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+                'master_url'     => 'https://localhost:8443',
+                'labels'         => {
                     'component' => 'fabric8Console'
                 },
                 'annotations'    => {
@@ -813,13 +717,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
                 }
             }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with kubernetes namespace annotations no match' do
       VCR.use_cassette('kubernetes_docker_metadata_annotations') do
-        es = emit({},'
+        filtered = emit({},'
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -830,21 +734,19 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
                 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
             },
             'kubernetes' => {
-                'host'               => 'jimmi-redhat.localnet',
-                'pod_name'           => 'fabric8-console-controller-98rqc',
-                'container_name'     => 'fabric8-console-container',
-                'container_image'    => 'fabric8/hawtio-kubernetes:latest',
-                'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
-                'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
-                'namespace_name'     => 'default',
-                'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
-                'master_url'         => 'https://localhost:8443',
-                'labels'             => {
+                'host'           => 'jimmi-redhat.localnet',
+                'pod_name'       => 'fabric8-console-controller-98rqc',
+                'container_name' => 'fabric8-console-container',
+                'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+                'namespace_name' => 'default',
+                'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+                'master_url'     => 'https://localhost:8443',
+                'labels'         => {
                     'component' => 'fabric8Console'
                 }
             }
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with CONTAINER_NAME that does not match' do
@@ -855,7 +757,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'randomfield' => 'randomvalue'
       }
       VCR.use_cassette('kubernetes_docker_metadata_annotations') do
-        es = emit_with_tag(tag, msg, '
+        filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -866,7 +768,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
           'randomfield' => 'randomvalue'
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
     test 'with CONTAINER_NAME starts with k8s_ that does not match' do
@@ -877,7 +779,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         'randomfield' => 'randomvalue'
       }
       VCR.use_cassette('kubernetes_docker_metadata_annotations') do
-        es = emit_with_tag(tag, msg, '
+        filtered = emit_with_tag(tag, msg, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -888,7 +790,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
           'randomfield' => 'randomvalue'
         }
-        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, filtered[0])
       end
     end
   end