fluent-plugin-kubernetes_metadata_filter 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8917464fdb9accec2996bb272aa99b5e5c78b8e1
-  data.tar.gz: 015f9f5e3ece31d4b279f0a536fa562fea6f8787
+  metadata.gz: 51343e9798a88739c3959e9cf6e350ec1b213ac3
+  data.tar.gz: d361fbbaa5f7f43832f4ae261746c5715c2e3aa0
 SHA512:
-  metadata.gz: 2d0c7cb0ddf63a3a3e529c3bd02ad6895a8a58f72b3f06917cd116591eea5b13ef73f7b736ac0725c308ff511a30b19cb71e8f1e772971fc391049120794b0a5
-  data.tar.gz: b579ec0cead5c9fcdaba5746e1f9c1f241ebfbe156e2bc36b609d17cac0bf7bff4653a6e072b7d565ec5d97d0edea80c4e1459a4b04627d133b2c094d3a02522
+  metadata.gz: 4307c9fc6195ed933cca79e12dbedb6698266943632814c9f25ad669cb909ebab2957317694ce764cc5abed0eabd4366622f2446e0e45f68566ad15f4f3082ce
+  data.tar.gz: 62f89e376091e5d4f856d034a8f47f0b15839e346912840cd3fafa2e7654e91bb4aa990030b22d87cc9056496b9c2aa4f893cbebbf8379993d9616d9aecb634a

data/README.md

@@ -104,6 +104,22 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
 </match>
 ```
 
+## Environment variables for Kubernetes
+
+If the name of the Kubernetes node the plugin is running on is set as
+an environment variable with the name `K8S_NODE_NAME`, it will reduce cache
+misses and needless calls to the Kubernetes API.
+
+In the Kubernetes container definition, this is easily accomplished by:
+
+```yaml
+env:
+- name: K8S_NODE_NAME
+  valueFrom:
+    fieldRef:
+      fieldPath: spec.nodeName
+```
+
 ## Example input/output
 
 Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "1.0.2"
+  gem.version       = "1.0.3"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -90,12 +90,12 @@ module Fluent
           rescue Exception=>e
             log.debug(e)
             @stats.bump(:pod_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error") if log.trace?
+            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
           end
         end
-      rescue KubeException=>e
+      rescue Exception=>e
         @stats.bump(:pod_cache_api_nil_error)
-        log.debug "Exception encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
+        log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       end
       {}
     end
@@ -132,12 +132,12 @@ module Fluent
           rescue Exception => e
             log.debug(e)
             @stats.bump(:namespace_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name} due to error") if log.trace?
+            log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
           end
         end
-      rescue KubeException => kube_error
+      rescue Exception => kube_error
         @stats.bump(:namespace_cache_api_nil_error)
-        log.debug "Exception encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
+        log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       end
       {}
     end

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -18,7 +18,6 @@
 #
 module KubernetesMetadata
   module CacheStrategy
-
     def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
       metadata = {}
       ids = @id_cache[key]

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -42,6 +42,9 @@ module KubernetesMetadata
             if cached
               @cache[cache_key] = parse_pod_metadata(notice.object)
               @stats.bump(:pod_cache_watch_updates)
+            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
+              @cache[cache_key] = parse_pod_metadata(notice.object)
+              @stats.bump(:pod_cache_host_updates)
             else
               @stats.bump(:pod_cache_watch_misses)
             end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -137,12 +137,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ', d: nil)
       d = create_driver(config) if d.nil?
-      if ENV['LOGLEVEL'] 
-        logger = Logger.new(STDOUT)
-        logger.level = eval("Logger::#{ENV['LOGLEVEL'].upcase}")
-        instance = d.instance
-        instance.instance_variable_set(:@log,logger)
-      end
       d.run {
         d.emit(msg, @time)
       }.filtered
@@ -159,6 +153,33 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       }.filtered
     end
 
+    test 'inability to connect to the api server handles exception and doensnt block pipeline' do
+      VCR.use_cassette('kubernetes_docker_metadata') do
+        driver = create_driver('
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+        ')
+        stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_raise(SocketError.new('error from pod fetch'))
+        stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_raise(SocketError.new('socket error from namespace fetch'))
+        es = emit({'time'=>'2015-05-08T09:22:01Z'}, '', :d => driver)
+        expected_kube_metadata = {
+          'time'=>'2015-05-08T09:22:01Z',
+          'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            "namespace_id"=>"orphaned",
+            'namespace_name' => '.orphaned',
+            "orphaned_namespace"=>"default"
+          }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
     test 'with docker & kubernetes metadata where id cache hit and metadata miss' do
       VCR.use_cassette('kubernetes_docker_metadata') do
         driver = create_driver('

data/test/plugin/test_watch_pods.rb

@@ -81,6 +81,17 @@ class DefaultPodWatchStrategyTest < WatchTest
       end
     end
 
+    test 'pod MODIFIED cached when hostname matches' do
+      orig_env_val = ENV['K8S_NODE_NAME']
+      ENV['K8S_NODE_NAME'] = 'aNodeName'
+      @client.stub :watch_pods, [@modified] do
+       start_pod_watch
+       assert_equal(true, @cache.key?('modified_uid'))
+       assert_equal(1, @stats[:pod_cache_host_updates])
+      end
+      ENV['K8S_NODE_NAME'] = orig_env_val
+    end
+
     test 'pod watch notice is updated when MODIFIED is received' do
       @cache['modified_uid'] = {}
       @client.stub :watch_pods, [@modified] do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2018-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd