fluent-plugin-kubernetes_metadata_filter 2.1.2 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe17406dc7a359a7bb8b52271680b041397bf567
-  data.tar.gz: 449541c91baaa0a96ee27adf6a74464941c96916
+  metadata.gz: 8e478872fe7fd194bb1746e4e44a917a7d07067b
+  data.tar.gz: dfc10c0b81a8a3a8a805e53fb914bf5525f4e328
 SHA512:
-  metadata.gz: 7f8d366a2eb39f8c43ce9d240817cc251524ec58c2cde4186d85824b7d1e861b8396443d2c4a148ffc84892632f3ff4c87f4ca8396ddfb2c46d4f5bf55ee5caa
-  data.tar.gz: fdc11df9a3d428929e2aa986b48848cc7204273a83b212f77277a295990afd6fd968d24105c4ebb30be4abdbd629e899478a76f6a03be6582492324b6f26cb28
+  metadata.gz: 27ac4b15b101d557411aebc943d5d1e5b464ec94d4e5e9318b960f0c022b09232f6112936af84b30a17834ee67562a3d0f12cea477639672089e1d3afd3541ae
+  data.tar.gz: bd79142fc9afde71d94ab772e65cc36cb118e0d6624014275fe2a151d5353e3e25a214ee8fc8a74e8e0d519e475cc0aa8ee364ace1f25091e0af88498685de26

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.1.2"
+  gem.version       = "2.1.3"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -299,22 +299,24 @@ module Fluent::Plugin
     end
 
     def filter_stream_from_files(tag, es)
-      return es if es.nil? || es.empty?
+      return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
       new_es = Fluent::MultiEventStream.new
 
       match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
       batch_miss_cache = {}
-      if match_data
-        container_id = match_data['docker_id']
-        metadata = {
-          'docker' => {
-            'container_id' => container_id
-          },
-          'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]), batch_miss_cache)
-        }
-      end
+      metadata = nil
 
       es.each do |time, record|
+        if match_data && metadata.nil?
+          container_id = match_data['docker_id']
+          metadata = {
+              'docker' => {
+                  'container_id' => container_id
+              },
+              'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record), batch_miss_cache)
+          }
+        end
+
         record = record.merge(Marshal.load(Marshal.dump(metadata))) if metadata
         new_es.add(time, record)
       end
@@ -323,7 +325,7 @@ module Fluent::Plugin
     end
 
     def filter_stream_from_journal(tag, es)
-      return es if es.nil? || es.empty?
+      return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
       new_es = Fluent::MultiEventStream.new
       batch_miss_cache = {}
       es.each do |time, record|

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -160,11 +160,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
     test 'nil event stream from journal' do
      #not certain how this is possible but adding test to properly
-     #guard against this condition we have seen
-
+     #guard against this condition we have seen - test for nil,
+     #empty, no empty method, not an event stream
      plugin = create_driver.instance
-     plugin.filter_stream_from_journal('tag', nil)
-     plugin.filter_stream_from_journal('tag', Fluent::MultiEventStream.new)
+     [nil, Fluent::MultiEventStream.new, 1, [1]].each do |es|
+       assert_equal es, plugin.filter_stream_from_journal('tag', es)
+     end
+     # and make sure OneEventStream works
+     ts = Time.now()
+     rec = {"message"=>"hello"}
+     es = Fluent::OneEventStream.new(ts, rec)
+     newes = plugin.filter_stream_from_journal('tag', es)
+     newes.each do |newts, newrec|
+      assert_equal ts, newts
+      assert_equal rec, newrec
+     end
     end
 
     test 'nil event stream from files' do
@@ -172,8 +182,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
      #guard against this condition we have seen
 
      plugin = create_driver.instance
-     plugin.filter_stream_from_files('tag', nil)
-     plugin.filter_stream_from_files('tag', Fluent::MultiEventStream.new)
+     [nil, Fluent::MultiEventStream.new, 1, [1]].each do |es|
+       assert_equal es, plugin.filter_stream_from_files('tag', es)
+     end
+     # and make sure OneEventStream works
+     ts = Time.now()
+     rec = {"message"=>"hello"}
+     es = Fluent::OneEventStream.new(ts, rec)
+     newes = plugin.filter_stream_from_journal('tag', es)
+     newes.each do |newts, newrec|
+      assert_equal ts, newts
+      assert_equal rec, newrec
+     end
     end
 
     test 'inability to connect to the api server handles exception and doensnt block pipeline' do
@@ -773,5 +793,47 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(expected_kube_metadata, filtered[0])
       end
     end
+
+    test 'processes all events when reading from MessagePackEventStream' do
+      VCR.use_cassette('kubernetes_docker_metadata') do
+        entries = [[@time, {'time'=>'2015-05-08T09:22:01Z'}], [@time, {'time'=>'2015-05-08T09:22:01Z'}]]
+        array_stream = Fluent::ArrayEventStream.new(entries)
+        msgpack_stream = Fluent::MessagePackEventStream.new(array_stream.to_msgpack_stream)
+
+        d = create_driver('
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+        ')
+        d.run {
+          d.feed(DEFAULT_TAG, msgpack_stream)
+        }
+        filtered = d.filtered.map{|e| e.last}
+
+        expected_kube_metadata = {
+            'time'=>'2015-05-08T09:22:01Z',
+            'docker' => {
+                'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+                'host'               => 'jimmi-redhat.localnet',
+                'pod_name'           => 'fabric8-console-controller-98rqc',
+                'container_name'     => 'fabric8-console-container',
+                'container_image'    => 'fabric8/hawtio-kubernetes:latest',
+                'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
+                'namespace_name'     => 'default',
+                'namespace_id'       => '898268c8-4a36-11e5-9d81-42010af0194c',
+                'pod_id'             => 'c76927af-f563-11e4-b32d-54ee7527188d',
+                'master_url'         => 'https://localhost:8443',
+                'labels' => {
+                    'component' => 'fabric8Console'
+                }
+            }
+        }
+
+        assert_equal(expected_kube_metadata, filtered[0])
+        assert_equal(expected_kube_metadata, filtered[1])
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.3
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-17 00:00:00.000000000 Z
+date: 2018-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd