fluent-plugin-kubernetes_metadata_filter 2.5.0 → 3.2.0

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -20,6 +22,7 @@
 require_relative 'kubernetes_metadata_cache_strategy'
 require_relative 'kubernetes_metadata_common'
 require_relative 'kubernetes_metadata_stats'
+require_relative 'kubernetes_metadata_util'
 require_relative 'kubernetes_metadata_watch_namespaces'
 require_relative 'kubernetes_metadata_watch_pods'
 
@@ -47,25 +50,21 @@ module Fluent::Plugin
     config_param :client_key, :string, default: nil
     config_param :ca_file, :string, default: nil
     config_param :verify_ssl, :bool, default: true
-    config_param :tag_to_kubernetes_name_regexp,
-                 :string,
-                 :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+    config_param :open_timeout, :integer, default: 3
+    config_param :read_timeout, :integer, default: 10
+
+    REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
+    REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+
+    #tag_to_kubernetes_name_regexp which must include named capture groups:
+    #  namespace            - The namespace in which the pod is deployed
+    #  pod_name             - The pod name
+    #  container_name       - The name of the container
+    #  pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
+    config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
+
     config_param :bearer_token_file, :string, default: nil
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
-    config_param :de_dot, :bool, default: true
-    config_param :de_dot_separator, :string, default: '_'
-    # if reading from the journal, the record will contain the following fields in the following
-    # format:
-    # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
-    # CONTAINER_FULL_ID=dockeridassha256hexvalue
-    config_param :use_journal, :bool, default: nil
-    # Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
-    # I would have included them as named groups, but you can't have named groups that are
-    # non-capturing :P
-    # parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
-    config_param :container_name_to_kubernetes_regexp,
-                 :string,
-                 :default => '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
 
     config_param :annotation_match, :array, default: []
     config_param :stats_interval, :integer, default: 30
@@ -81,6 +80,11 @@ module Fluent::Plugin
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
+
+    # A classname in the form of Test::APIAdapter which will try
+    # to be resolved from a relative named file 'test_api_adapter'
+    config_param :test_api_adapter, :string, default: nil
+
     # The time interval in seconds for retry backoffs when watch connections fail.
     config_param :watch_retry_interval, :integer, default: 1
     # The base number of exponential backoff for retries.
@@ -89,36 +93,39 @@ module Fluent::Plugin
     config_param :watch_retry_max_times, :integer, default: 10
 
     def fetch_pod_metadata(namespace_name, pod_name)
-      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
-      begin
-        metadata = @client.get_pod(pod_name, namespace_name)
-        unless metadata
-          log.trace("no metadata returned for: #{namespace_name}/#{pod_name}") if log.trace?
-          @stats.bump(:pod_cache_api_nil_not_found)
+      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
+      options = {
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
+      }
+      pod_object = @client.get_pod(pod_name, namespace_name, options)
+      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
+      metadata = parse_pod_metadata(pod_object)
+      @stats.bump(:pod_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
+      @cache[metadata['pod_id']] = metadata
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
+        elsif e.error_code == 404
+          log.debug "Encountered '404 Not Found' exception, pod not found"
+          @stats.bump(:pod_cache_api_nil_error)
         else
-          begin
-            log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            metadata = parse_pod_metadata(metadata)
-            @stats.bump(:pod_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            @cache[metadata['pod_id']] = metadata
-            return metadata
-          rescue Exception=>e
-            log.debug(e)
-            @stats.bump(:pod_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
-          end
+          log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:pod_cache_api_nil_error)
         end
-      rescue Exception=>e
+        {}
+      rescue StandardError => e
         @stats.bump(:pod_cache_api_nil_error)
-        log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
       end
-      {}
-    end
 
     def dump_stats
       @curr_time = Time.now
       return if @curr_time.to_i - @prev_time.to_i < @stats_interval
+
       @prev_time = @curr_time
       @stats.set(:pod_cache_size, @cache.count)
       @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
@@ -131,55 +138,53 @@ module Fluent::Plugin
     end
 
     def fetch_namespace_metadata(namespace_name)
-      log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
-      begin
-        metadata = @client.get_namespace(namespace_name)
-        unless metadata
-            log.trace("no metadata returned for: #{namespace_name}") if log.trace?
-            @stats.bump(:namespace_cache_api_nil_not_found)
+      log.trace("fetching namespace metadata: #{namespace_name}")
+      options = {
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
+      }
+      namespace_object = @client.get_namespace(namespace_name, nil, options)
+      log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
+      metadata = parse_namespace_metadata(namespace_object)
+      @stats.bump(:namespace_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}: #{metadata}")
+      @namespace_cache[metadata['namespace_id']] = metadata
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
+          create_client()
         else
-          begin
-            log.trace("raw metadata for #{namespace_name}: #{metadata}") if log.trace?
-            metadata = parse_namespace_metadata(metadata)
-            @stats.bump(:namespace_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
-            @namespace_cache[metadata['namespace_id']] = metadata
-            return metadata
-          rescue Exception => e
-            log.debug(e)
-            @stats.bump(:namespace_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
-          end
+          log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+          @stats.bump(:namespace_cache_api_nil_error)
         end
-      rescue Exception => kube_error
+        {}
+      rescue StandardError => e
         @stats.bump(:namespace_cache_api_nil_error)
-        log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      end
-      {}
+        log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        {}
     end
 
     def initialize
       super
       @prev_time = Time.now
+      @ssl_options = {}
+      @auth_options = {}
     end
 
     def configure(conf)
       super
 
-      def log.trace?
-        level == Fluent::Log::LEVEL_TRACE
-      end
-
       require 'kubeclient'
       require 'lru_redux'
-      @stats = KubernetesMetadata::Stats.new
 
-      if @de_dot && (@de_dot_separator =~ /\./).present?
-        raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
+      @stats = KubernetesMetadata::Stats.new
+      if @stats_interval <= 0
+        @stats = KubernetesMetadata::NoOpStats.new
+        self.define_singleton_method(:dump_stats) {}
       end
 
       if @cache_ttl < 0
-        log.info "Setting the cache TTL to :none because it was <= 0"
+        log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
       end
 
@@ -193,15 +198,14 @@ module Fluent::Plugin
       @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
 
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
-      @container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
-
+      
       # Use Kubernetes default service account if we're in a pod.
       if @kubernetes_url.nil?
-        log.debug "Kubernetes URL is not set - inspecting environ"
+        log.debug 'Kubernetes URL is not set - inspecting environ'
 
         env_host = ENV['KUBERNETES_SERVICE_HOST']
         env_port = ENV['KUBERNETES_SERVICE_PORT']
-        if env_host.present? && env_port.present?
+        if present?(env_host) && present?(env_port)
           if env_host =~ Resolv::IPv6::Regex
             # Brackets are needed around IPv6 addresses
             env_host = "[#{env_host}]"
@@ -209,7 +213,7 @@ module Fluent::Plugin
           @kubernetes_url = "https://#{env_host}:#{env_port}/api"
           log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
         else
-          log.debug "No Kubernetes URL could be found in config or environ"
+          log.debug 'No Kubernetes URL could be found in config or environ'
         end
       end
 
@@ -219,24 +223,23 @@ module Fluent::Plugin
         ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
         pod_token = File.join(@secret_dir, K8_POD_TOKEN)
 
-        if !@ca_file.present? and File.exist?(ca_cert)
+        if !present?(@ca_file) && File.exist?(ca_cert)
           log.debug "Found CA certificate: #{ca_cert}"
           @ca_file = ca_cert
         end
 
-        if !@bearer_token_file.present? and File.exist?(pod_token)
+        if !present?(@bearer_token_file) && File.exist?(pod_token)
           log.debug "Found pod token: #{pod_token}"
           @bearer_token_file = pod_token
         end
       end
 
-      if @kubernetes_url.present?
-
-        ssl_options = {
-            client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
-            client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
-            ca_file:     @ca_file,
-            verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      if present?(@kubernetes_url)
+        @ssl_options = {
+          client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+          client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+          ca_file: @ca_file,
+          verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         }
 
         if @ssl_partial_chain
@@ -244,157 +247,133 @@ module Fluent::Plugin
           require 'openssl'
           ssl_store = OpenSSL::X509::Store.new
           ssl_store.set_default_paths
-          if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
-            flagval = OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
-          else
-            # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
-            flagval = 0x80000
-          end
+          flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
+                      OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
+                    else
+                      # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
+                      0x80000
+                    end
           ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
-          ssl_options[:cert_store] = ssl_store
+          @ssl_options[:cert_store] = ssl_store
         end
 
-        auth_options = {}
-
-        if @bearer_token_file.present?
-          bearer_token = File.read(@bearer_token_file)
-          auth_options[:bearer_token] = bearer_token
+        if present?(@bearer_token_file)
+          @auth_options[:bearer_token_file] = @bearer_token_file
         end
 
-        log.debug "Creating K8S client"
-        @client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
-                                         ssl_options: ssl_options,
-                                         auth_options: auth_options
+        create_client()
+
+        if @test_api_adapter
+          log.info "Extending client with test api adaper #{@test_api_adapter}"
+          require_relative @test_api_adapter.underscore
+          @client.extend(eval(@test_api_adapter))
+        end
 
         begin
           @client.api_valid?
-        rescue KubeException => kube_error
-          raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
+        rescue KubeException => e
+          raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
         end
 
         if @watch
-          pod_thread = Thread.new(self) { |this| this.set_up_pod_thread }
+          if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
+            log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
+          end
+
+          pod_thread = Thread.new(self, &:set_up_pod_thread)
           pod_thread.abort_on_exception = true
 
-          namespace_thread = Thread.new(self) { |this| this.set_up_namespace_thread }
+          namespace_thread = Thread.new(self, &:set_up_namespace_thread)
           namespace_thread.abort_on_exception = true
         end
       end
-      @time_fields = []
-      @time_fields.push('_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP') if @use_journal || @use_journal.nil?
-      @time_fields.push('time') unless @use_journal
-      @time_fields.push('@timestamp') if @lookup_from_k8s_field
 
       @annotations_regexps = []
       @annotation_match.each do |regexp|
-        begin
-          @annotations_regexps << Regexp.compile(regexp)
-        rescue RegexpError => e
-          log.error "Error: invalid regular expression in annotation_match: #{e}"
-        end
+        @annotations_regexps << Regexp.compile(regexp)
+      rescue RegexpError => e
+        log.error "Error: invalid regular expression in annotation_match: #{e}"
       end
+    end
 
+    def create_client()
+      log.debug 'Creating K8S client'
+      @client = nil
+      @client = Kubeclient::Client.new(
+        @kubernetes_url,
+        @apiVersion,
+        ssl_options: @ssl_options,
+        auth_options: @auth_options,
+        timeouts: {
+          open: @open_timeout,
+          read: @read_timeout
+        },
+        as: :parsed_symbolized
+      )
     end
 
-    def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
+    def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
       metadata = {
-        'docker' => {'container_id' => container_id},
+        'docker' => { 'container_id' => "" },
         'kubernetes' => {
-          'container_name'  => container_name,
-          'namespace_name'  => namespace_name,
-          'pod_name'        => pod_name
+          'container_name' => container_name,
+          'namespace_name' => namespace_name,
+          'pod_name' => pod_name
         }
       }
-      if @kubernetes_url.present?
-        pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
-
-        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
-          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
-          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
+      metadata['docker']['container_id'] = docker_id unless docker_id.nil?
+      container_cache_key = container_name
+      if present?(@kubernetes_url)
+        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
+        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
+          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
+          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
+          metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
         end
 
         metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
         metadata['kubernetes'].delete('containers')
       end
-      metadata
-    end
-
-    def create_time_from_record(record, internal_time)
-      time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
-      time = record[time_key]
-      if time.nil? || time.chop.empty?
-        # `internal_time` is a Fluent::EventTime, it can't compare with Time.
-        return Time.at(internal_time.to_f)
-      end
-      if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
-        timei= time.to_i
-        return Time.at(timei / 1000000, timei % 1000000)
+      metadata['kubernetes'].tap do |kube|
+        kube.each_pair do |k,v|
+          kube[k.dup] = v.dup
+        end
       end
-      return Time.parse(time)
+      metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
+      metadata
     end
 
-    def filter_stream(tag, es)
-      return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
-      new_es = Fluent::MultiEventStream.new
-      tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
-      tag_metadata = nil
+    def filter(tag, time, record)
+      tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
       batch_miss_cache = {}
-      es.each do |time, record|
-        if tag_match_data && tag_metadata.nil?
-          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-            tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
-        end
-        metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
-        if (@use_journal || @use_journal.nil?) &&
-          (j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
-          metadata = j_metadata
-        end
-        if @lookup_from_k8s_field && record.has_key?('kubernetes') && record.has_key?('docker') &&
+      if tag_match_data
+        cache_key =  if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
+          tag_match_data['pod_uuid']
+        else
+          tag_match_data['docker_id']
+        end 
+        docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
+        metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
+                                                cache_key, time, batch_miss_cache, docker_id)
+      end
+      if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
           record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
-          record['kubernetes'].has_key?('namespace_name') &&
-          record['kubernetes'].has_key?('pod_name') &&
-          record['kubernetes'].has_key?('container_name') &&
-          record['docker'].has_key?('container_id') &&
+          record['kubernetes'].key?('namespace_name') &&
+          record['kubernetes'].key?('pod_name') &&
+          record['kubernetes'].key?('container_name') &&
+          record['docker'].key?('container_id') &&
           (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
-            record['kubernetes']['container_name'], record['docker']['container_id'],
-            create_time_from_record(record, time), batch_miss_cache))
-            metadata = k_metadata
-        end
-
-        record = record.merge(metadata) if metadata
-        new_es.add(time, record)
+                                                record['kubernetes']['container_name'], record['docker']['container_id'],
+                                                time, batch_miss_cache, record['docker']['container_id']))
+        metadata = k_metadata
       end
       dump_stats
-      new_es
-    end
-
-    def get_metadata_for_journal_record(record, time, batch_miss_cache)
-      metadata = nil
-      if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
-        metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
-          get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
-            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
-        end
-        unless metadata
-          log.debug "Error: could not match CONTAINER_NAME from record #{record}"
-          @stats.bump(:container_name_match_failed)
-        end
-      elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
-        log.debug "Error: no container name and id in record #{record}"
-        @stats.bump(:container_name_id_missing)
-      end
-      metadata
+      metadata ? record.merge(metadata) : record
     end
 
-    def de_dot!(h)
-      h.keys.each do |ref|
-        if h[ref] && ref =~ /\./
-          v = h.delete(ref)
-          newref = ref.to_s.gsub('.', @de_dot_separator)
-          h[newref] = v
-        end
-      end
+    # copied from activesupport
+    def present?(object)
+      object.respond_to?(:empty?) ? !object.empty? : !!object
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -18,34 +20,21 @@
 #
 module KubernetesMetadata
   module CacheStrategy
-    def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
+    def get_pod_metadata(key, namespace_name, pod_name, time, batch_miss_cache)
       metadata = {}
       ids = @id_cache[key]
-      if !ids.nil?
-        # FAST PATH
-        # Cache hit, fetch metadata from the cache
-        metadata = @cache.fetch(ids[:pod_id]) do
-          @stats.bump(:pod_cache_miss)
-          m = fetch_pod_metadata(namespace_name, pod_name)
-          (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
-        end
-        metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
-          @stats.bump(:namespace_cache_miss)
-          m = fetch_namespace_metadata(namespace_name) unless @skip_namespace_metadata
-          (m.nil? || m.empty?) ?  {'namespace_id'=>ids[:namespace_id]} : m
-        end)
-      else
-        # SLOW PATH
+      if ids.nil?
         @stats.bump(:id_cache_miss)
         return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+
         pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
         if @skip_namespace_metadata
-          ids = { :pod_id=> pod_metadata['pod_id'] }
+          ids = { pod_id: pod_metadata['pod_id'] }
           @id_cache[key] = ids
           return pod_metadata
         end
         namespace_metadata = fetch_namespace_metadata(namespace_name)
-        ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
+        ids = { pod_id: pod_metadata['pod_id'], namespace_id: namespace_metadata['namespace_id'] }
         if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
           # pod found and namespace found
           metadata = pod_metadata
@@ -55,11 +44,11 @@ module KubernetesMetadata
             # pod not found, but namespace found
             @stats.bump(:id_cache_pod_not_found_namespace)
             ns_time = Time.parse(namespace_metadata['creation_timestamp'])
-            if ns_time <= record_create_time
+            if ns_time <= Time.at(time.to_f)
               # namespace is older then record for pod
               ids[:pod_id] = key
               metadata = @cache.fetch(ids[:pod_id]) do
-                m = { 'pod_id' => ids[:pod_id] }
+                { 'pod_id' => ids[:pod_id] }
               end
             end
             metadata.merge!(namespace_metadata)
@@ -74,7 +63,7 @@ module KubernetesMetadata
               @stats.bump(:id_cache_orphaned_record)
             end
             if @allow_orphans
-              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
+              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
               metadata = {
                 'orphaned_namespace' => namespace_name,
                 'namespace_name' => @orphaned_namespace_name,
@@ -87,12 +76,25 @@ module KubernetesMetadata
           end
         end
         @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+      else
+        # SLOW PATH
+        metadata = @cache.fetch(ids[:pod_id]) do
+          @stats.bump(:pod_cache_miss)
+          m = fetch_pod_metadata(namespace_name, pod_name)
+          m.nil? || m.empty? ? { 'pod_id' => ids[:pod_id] } : m
+        end
+        metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
+          m = unless @skip_namespace_metadata
+                @stats.bump(:namespace_cache_miss)
+                fetch_namespace_metadata(namespace_name)
+              end
+          m.nil? || m.empty? ? { 'namespace_id' => ids[:namespace_id] } : m
+        end)
       end
 
       # remove namespace info that is only used for comparison
       metadata.delete('creation_timestamp')
-      metadata.delete_if{|k,v| v.nil?}
+      metadata.delete_if { |_k, v| v.nil? }
     end
-
   end
 end