fluent-plugin-kubernetes_metadata_filter 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b64ea97cd4df7be745b06c0f46e82f152e0503ea
-  data.tar.gz: b026a7a6039cd7f15aa938be5969fa7e5a818c7b
+  metadata.gz: ef179686ef1088fb46084410bdeb0a2c4b07c3f6
+  data.tar.gz: 02b2dfe0880adb892922876f5dfd123a54922d76
 SHA512:
-  metadata.gz: 53f23402783e6a0b7857698b99a3a7dfd0e343f4ccd5b7ea2c440d8c3fc624d61c4ef89846aa2c91b90995e8df7feb408df7cd5f933d577b3bcfe76e5ced5174
-  data.tar.gz: 912ffc8b2ba2e376bb0ffffe9bd9a881cf2ce6f535328e2c76a53941558d0c7e2209a2359ba319d3d8deb0278252673c15b84c169464bfd8dfc8aa5101adacf8
+  metadata.gz: 5972e28a29443b1e11fd7071ee1b7df5ee5b93be0f406a3bac51979058c49651777515acdc12c63ea2d4f9f84f632668a7d2fef592b9aa4d0249def6b54bdf2b
+  data.tar.gz: b079681eceb4fd3e1e182c133fd529e9820137cb7b139ffcf63a85b082b8fdeda0ee735ff271e28fa081f348451f2684279aeee8565928d5bd33ba4fc178b851

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "1.0.0"
+  gem.version       = "1.0.1"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -17,6 +17,8 @@
 # limitations under the License.
 #
 
+require_relative 'kubernetes_metadata_cache_strategy'
+require_relative 'kubernetes_metadata_common'
 require_relative 'kubernetes_metadata_stats'
 require_relative 'kubernetes_metadata_watch_namespaces'
 require_relative 'kubernetes_metadata_watch_pods'
@@ -26,6 +28,7 @@ module Fluent
     K8_POD_CA_CERT = 'ca.crt'
     K8_POD_TOKEN = 'token'
 
+    include KubernetesMetadata::CacheStrategy
     include KubernetesMetadata::Common
     include KubernetesMetadata::WatchNamespaces
     include KubernetesMetadata::WatchPods
@@ -252,7 +255,7 @@ module Fluent
 
     end
 
-    def get_metadata_for_record(match_data, cache_key, create_time)
+    def get_metadata_for_record(match_data, cache_key, create_time, batch_miss_cache)
       namespace_name = match_data['namespace']
       pod_name = match_data['pod_name']
       metadata = {
@@ -261,58 +264,12 @@ module Fluent
         'pod_name'       => pod_name
       }
       if @kubernetes_url.present?
-        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time)
+        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
         metadata.merge!(pod_metadata) if pod_metadata
       end
       metadata
     end
 
-    def get_pod_metadata(key, namespace_name, pod_name, record_create_time)
-        ids = @id_cache.fetch(key) do
-          #no metadata found
-          @stats.bump(:id_cache_miss)
-          pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
-          namespace_metadata = fetch_namespace_metadata(namespace_name)
-          ids = {:pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
-          # pod not found or namespace not found
-          if ids[:pod_id].nil? || ids[:namespace_id].nil?
-             # pod notfound & namespace found
-             if ids[:pod_id].nil? && !ids[:namespace_id].nil?
-               ns_time = Time.parse(namespace_metadata['creation_timestamp'])
-               ids[:pod_id] = key if ns_time <= record_create_time #ns is older then record
-               ids
-             else
-               # nothing found
-               @stats.bump(:pod_cache_orphaned_record)
-               ids = :orphaned
-             end
-          end
-          ids
-        end
-
-        if ids == :orphaned
-          return {} unless @allow_orphans
-          log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
-          return {
-            'orphaned_namespace' => namespace_name,
-            'namespace_name' => @orphaned_namespace_name,
-            'namespace_id' => @orphaned_namespace_id
-          }
-        end
-        metadata =  @cache.fetch(ids[:pod_id]) do
-            @stats.bump(:pod_cache_miss)
-            m = fetch_pod_metadata(namespace_name, pod_name)
-            (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
-        end
-        metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
-           @stats.bump(:namespace_cache_miss)
-           m = fetch_namespace_metadata(namespace_name)
-           (m.nil? || m.empty?) ?  {'namespace_id'=>ids[:namespace_id]} : m
-        end)
-        metadata.delete('creation_timestamp') #remove namespace info thats only used for comparison
-        metadata
-    end
-
     def create_time_from_record(record)
         time = if @use_journal
            record['_SOURCE_REALTIME_TIMESTAMP'].nil? ? record['_SOURCE_REALTIME_TIMESTAMP'] : record['__REALTIME_TIMESTAMP']
@@ -330,14 +287,14 @@ module Fluent
       new_es = MultiEventStream.new
 
       match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
-
+      batch_miss_cache = {}
       if match_data
         container_id = match_data['docker_id']
         metadata = {
           'docker' => {
             'container_id' => container_id
           },
-          'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]))
+          'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]), batch_miss_cache)
         }
       end
 
@@ -354,7 +311,7 @@ module Fluent
 
     def filter_stream_from_journal(tag, es)
       new_es = MultiEventStream.new
-
+      batch_miss_cache = {}
       es.each { |time, record|
         record = merge_json_log(record) if @merge_json_log
         metadata = nil
@@ -365,7 +322,7 @@ module Fluent
               'docker' => {
                 'container_id' => container_id
               },
-              'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record))
+              'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record), batch_miss_cache)
             }
 
             metadata

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -0,0 +1,93 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module KubernetesMetadata
+  module CacheStrategy
+
+    def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
+      metadata = {}
+      ids = @id_cache[key]
+      if !ids.nil?
+        # FAST PATH
+        # Cache hit, fetch metadata from the cache
+        metadata = @cache.fetch(ids[:pod_id]) do
+          @stats.bump(:pod_cache_miss)
+          m = fetch_pod_metadata(namespace_name, pod_name)
+          (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
+        end
+        metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
+          @stats.bump(:namespace_cache_miss)
+          m = fetch_namespace_metadata(namespace_name)
+          (m.nil? || m.empty?) ?  {'namespace_id'=>ids[:namespace_id]} : m
+        end)
+      else
+        # SLOW PATH
+        @stats.bump(:id_cache_miss)
+        return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+        pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
+        namespace_metadata = fetch_namespace_metadata(namespace_name)
+        ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
+        if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
+          # pod found and namespace found
+          metadata = pod_metadata
+          metadata.merge!(namespace_metadata)
+        else
+          if ids[:pod_id].nil? && !ids[:namespace_id].nil?
+            # pod not found, but namespace found
+            @stats.bump(:id_cache_pod_not_found_namespace)
+            ns_time = Time.parse(namespace_metadata['creation_timestamp'])
+            if ns_time <= record_create_time
+              # namespace is older then record for pod
+              ids[:pod_id] = key
+              metadata = @cache.fetch(ids[:pod_id]) do
+                m = { 'pod_id' => ids[:pod_id] }
+              end
+            end
+            metadata.merge!(namespace_metadata)
+          else
+            if !ids[:pod_id].nil? && ids[:namespace_id].nil?
+              # pod found, but namespace NOT found
+              # this should NEVER be possible since pod meta can
+              # only be retrieved with a namespace
+              @stats.bump(:id_cache_namespace_not_found_pod)
+            else
+              # nothing found
+              @stats.bump(:id_cache_orphaned_record)
+            end
+            if @allow_orphans
+              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
+              metadata = {
+                'orphaned_namespace' => namespace_name,
+                'namespace_name' => @orphaned_namespace_name,
+                'namespace_id' => @orphaned_namespace_id
+              }
+            else
+              metadata = {}
+            end
+            batch_miss_cache["#{namespace_name}_#{pod_name}"] = metadata
+          end
+        end
+        @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+      end
+      # remove namespace info that is only used for comparison
+      metadata.delete('creation_timestamp')
+      metadata.delete_if{|k,v| v.nil?}
+    end
+
+  end
+end

data/test/plugin/test_cache_strategy.rb

@@ -0,0 +1,196 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative '../helper'
+require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
+require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
+require 'lru_redux'
+require 'webmock/test_unit'
+WebMock.disable_net_connect!
+
+class TestCacheStrategy
+    include KubernetesMetadata::CacheStrategy
+
+    def initialize
+      @stats = KubernetesMetadata::Stats.new
+      @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
+      @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
+      @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
+      @orphaned_namespace_name = '.orphaned'
+      @orphaned_namespace_id = 'orphaned'
+    end
+
+    attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
+
+    def fetch_pod_metadata(namespace_name, pod_name)
+    end
+
+    def fetch_namespace_metadata(namespace_name)
+    end
+
+    def log
+        logger = {}
+        def logger.trace?
+          true
+        end
+        def logger.trace(message)
+        end
+        logger
+    end
+
+end
+
+class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
+  
+    def setup
+       @strategy = TestCacheStrategy.new
+       @cache_key = 'some_long_container_id'
+       @namespace_name = 'some_namespace_name'
+       @namespace_uuid = 'some_namespace_uuid'
+       @pod_name = 'some_pod_name'
+       @pod_uuid = 'some_pod_uuid'
+       @time = Time.now
+       @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}}
+       @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s}
+    end
+
+    test 'when cached metadata is found' do
+        exp = @pod_meta.merge(@namespace_meta)
+        exp.delete('creation_timestamp')
+        @strategy.id_cache[@cache_key] = {
+            pod_id: @pod_uuid,
+            namespace_id: @namespace_uuid
+        }
+        @strategy.cache[@pod_uuid] = @pod_meta
+        @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
+    end
+
+    test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
+        exp = {
+            'pod_id'=> @pod_uuid,
+            'namespace_id'=> @namespace_uuid
+        }
+        @strategy.id_cache[@cache_key] = {
+            pod_id: @pod_uuid,
+            namespace_id: @namespace_uuid
+        }
+        @strategy.stub :fetch_pod_metadata, {} do
+            @strategy.stub :fetch_namespace_metadata, nil do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and is fetched' do
+        exp = @pod_meta.merge(@namespace_meta)
+        exp.delete('creation_timestamp')
+        @strategy.stub :fetch_pod_metadata, @pod_meta do
+            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
+                assert_true(@strategy.id_cache.key?(@cache_key))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
+        # this is the case for a record from a deleted pod where no other
+        # records were read.  using the container hash since that is all
+        # we ever will have and should allow us to process all the deleted
+        # pod records
+        exp = {
+            'pod_id'=> @cache_key, 
+            'namespace_id'=> @namespace_uuid
+        }
+        @strategy.stub :fetch_pod_metadata, {} do
+            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
+                assert_true(@strategy.id_cache.key?(@cache_key))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
+        # this is the case for a record from a deleted pod from a deleted namespace
+        # where new namespace was created with the same name
+        exp = {
+            'namespace_id'=> @namespace_uuid
+        }
+        @strategy.stub :fetch_pod_metadata, {} do
+            @strategy.stub :fetch_namespace_metadata, @namespace_meta do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
+                assert_true(@strategy.id_cache.key?(@cache_key))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
+        # we should never see this since pod meta should not be retrievable
+        # unless the namespace exists
+        @strategy.stub :fetch_pod_metadata, @pod_meta do
+            @strategy.stub :fetch_namespace_metadata, {} do
+                assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
+        # we should never see this since pod meta should not be retrievable
+        # unless the namespace exists
+        @strategy.allow_orphans = true
+        exp = {
+            'orphaned_namespace' => 'namespace',
+            'namespace_name' => '.orphaned',
+            'namespace_id' => 'orphaned'
+        }
+        @strategy.stub :fetch_pod_metadata, @pod_meta do
+            @strategy.stub :fetch_namespace_metadata, {} do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
+            end
+        end
+    end
+
+    test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
+        # processing a batch of records with no meta. ideally we only hit the api server once
+        batch_miss_cache = {}
+        @strategy.stub :fetch_pod_metadata, {} do
+            @strategy.stub :fetch_namespace_metadata, {} do
+                assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+            end
+        end
+        assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+    end
+    
+    test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
+        # we should never see this since pod meta should not be retrievable
+        # unless the namespace exists
+        @strategy.allow_orphans = true
+        exp = {
+            'orphaned_namespace' => 'namespace',
+            'namespace_name' => '.orphaned',
+            'namespace_id' => 'orphaned'
+        }
+        batch_miss_cache = {}
+        @strategy.stub :fetch_pod_metadata, {}  do
+            @strategy.stub :fetch_namespace_metadata, {} do
+                assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+            end
+        end
+        assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-04 00:00:00.000000000 Z
+date: 2018-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -207,6 +207,7 @@ files:
 - circle.yml
 - fluent-plugin-kubernetes_metadata_filter.gemspec
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
+- lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb
 - lib/fluent/plugin/kubernetes_metadata_common.rb
 - lib/fluent/plugin/kubernetes_metadata_stats.rb
 - lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb
@@ -222,6 +223,7 @@ files:
 - test/helper.rb
 - test/plugin/test.token
 - test/plugin/test_cache_stats.rb
+- test/plugin/test_cache_strategy.rb
 - test/plugin/test_filter_kubernetes_metadata.rb
 - test/plugin/test_watch_namespaces.rb
 - test/plugin/test_watch_pods.rb
@@ -262,6 +264,7 @@ test_files:
 - test/helper.rb
 - test/plugin/test.token
 - test/plugin/test_cache_stats.rb
+- test/plugin/test_cache_strategy.rb
 - test/plugin/test_filter_kubernetes_metadata.rb
 - test/plugin/test_watch_namespaces.rb
 - test/plugin/test_watch_pods.rb