fluent-plugin-kubernetes_metadata_filter 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1991a6b8cd09f65728588401e697dda36954b94cf75b625789c6e7773d50ccf
-  data.tar.gz: df46f28a113c17aa3e2174d3ad23edb0b5b93fb6d65d5f02787d6ac83ac129a2
+  metadata.gz: c891b663769f4927f2bc0a2524b8358d965c0962ec8d56e691e50a4011c4b6f4
+  data.tar.gz: cb8110159bd202df0fba96c477149f16836b54598be864bbb5bda9412d8ef563
 SHA512:
-  metadata.gz: 1a41d498af8a2e92723c05679608294d07acf2cf7a87dfdf5921fe3a385df6369003eef939dd5aeabf970c67f31eb319ac1485d566ac110bcbe6b79f5acc392a
-  data.tar.gz: e62c355dbdb0ac1e89e43534c5929c80e82b166f5c614b1967d6ff79658835881a9c6803d5072e85a41c83fbefc8db326ba95c6d2f1c5a20ca1b8a0f18ce3cf1
+  metadata.gz: a974a260ab2b56e5e941711ae0216c6da574429a35e12cd622f76cea8423b66fa09b71a58f7627c5f940ad175b8c45cf8fb1b3993cb825197dab0ef4b4d28443
+  data.tar.gz: 5988e42558432f8e067b50230490c2a128661ef0f6f7294ee0e3e219eab764c11d4fc86b33b8c13a2c2753afc8416e14af8afff625afa64858d9a59c5802bb97

data/.circleci/config.yml

@@ -14,21 +14,21 @@ missingdeps: &missingdeps
             sudo cp /tmp/sources.list /etc/apt/sources.list
             sudo apt-get update
             sudo apt-get install cmake libicu-dev libssl-dev
-  
+
 test: &test
   name: Test bundle
   command: bundle exec rake test
 
 executors:
-  ruby-2-4:
-    docker:
-      - image: circleci/ruby:2.4.6
   ruby-2-5:
     docker:
       - image: circleci/ruby:2.5.5
   ruby-2-6:
     docker:
       - image: circleci/ruby:2.6.3
+  ruby-2-7:
+    docker:
+      - image: circleci/ruby:2.7.1
 jobs:
   "ruby-test":
     parameters:
@@ -49,9 +49,9 @@ jobs:
 workflows:
   "test_multiple_ruby_versions":
     jobs:
-      - ruby-test:
-          ruby-version: ruby-2-4
       - ruby-test:
           ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
+      - ruby-test:
+          ruby-version: ruby-2-7

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.5.0)
-      fluentd (>= 0.14.0, < 1.12)
+    fluent-plugin-kubernetes_metadata_filter (2.6.0)
+      fluentd (>= 0.14.0, < 1.13)
       kubeclient (< 5)
       lru_redux
 
@@ -12,36 +12,37 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.1)
-    bump (0.9.0)
+    bump (0.10.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.0)
     copyright-header (1.0.22)
       github-linguist
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    docile (1.3.2)
+    crack (0.4.5)
+      rexml
+    docile (1.3.5)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.13.1)
+    ffi (1.14.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.11.1)
+    fluentd (1.12.0)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.9.0)
-      charlock_holmes (~> 0.7.6)
+    github-linguist (7.12.2)
+      charlock_holmes (~> 0.7.7)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
       rugged (>= 0.25.1)
@@ -55,30 +56,34 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.1)
+    http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.6.0)
-    kubeclient (4.7.0)
+    jsonpath (1.1.0)
+      multi_json
+    kubeclient (4.9.1)
       http (>= 3.0, < 5.0)
+      jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
     lru_redux (1.1.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
+    mime-types-data (3.2020.1104)
     mini_mime (1.0.2)
     minitest (4.7.5)
     msgpack (1.3.3)
+    multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
     power_assert (1.2.0)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
     rainbow (3.0.0)
-    rake (13.0.1)
-    recursive-open-struct (1.1.2)
-    regexp_parser (1.7.1)
+    rake (13.0.3)
+    recursive-open-struct (1.1.3)
+    regexp_parser (2.0.3)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -86,43 +91,44 @@ GEM
       netrc (~> 0.8)
     rexml (3.2.4)
     rr (1.2.1)
-    rubocop (0.86.0)
+    rubocop (1.8.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.1.0)
-      parser (>= 2.7.0.1)
-    ruby-progressbar (1.10.1)
-    rugged (1.0.1)
-    safe_yaml (1.0.5)
-    serverengine (2.2.1)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.0)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
+    rugged (1.1.0)
+    serverengine (2.2.2)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    simplecov (0.18.5)
+    simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-    simplecov-html (0.12.2)
-    strptime (0.2.4)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
+    strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
     test-unit-rr (1.0.5)
       rr (>= 1.1.1)
       test-unit (>= 2.5.2)
-    tzinfo (2.0.2)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
+    tzinfo-data (1.2020.6)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unicode-display_width (1.7.0)
+    unicode-display_width (2.0.0)
     vcr (6.0.0)
-    webmock (3.8.3)
+    webmock (3.11.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.5.0"
+  gem.version       = "2.6.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -13,13 +13,10 @@ Gem::Specification.new do |gem|
   gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 2.1.0'
+  gem.required_ruby_version = '>= 2.5.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.12']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.13']
   gem.add_runtime_dependency "lru_redux"
   gem.add_runtime_dependency "kubeclient", '< 5'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -90,29 +90,15 @@ module Fluent::Plugin
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
-      begin
-        metadata = @client.get_pod(pod_name, namespace_name)
-        unless metadata
-          log.trace("no metadata returned for: #{namespace_name}/#{pod_name}") if log.trace?
-          @stats.bump(:pod_cache_api_nil_not_found)
-        else
-          begin
-            log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            metadata = parse_pod_metadata(metadata)
-            @stats.bump(:pod_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            @cache[metadata['pod_id']] = metadata
-            return metadata
-          rescue Exception=>e
-            log.debug(e)
-            @stats.bump(:pod_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
-          end
-        end
-      rescue Exception=>e
-        @stats.bump(:pod_cache_api_nil_error)
-        log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      end
+      pod_object = @client.get_pod(pod_name, namespace_name)
+      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
+      metadata = parse_pod_metadata(pod_object)
+      @stats.bump(:pod_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
+      @cache[metadata['pod_id']] = metadata
+    rescue => e
+      @stats.bump(:pod_cache_api_nil_error)
+      log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       {}
     end
 
@@ -132,29 +118,15 @@ module Fluent::Plugin
 
     def fetch_namespace_metadata(namespace_name)
       log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
-      begin
-        metadata = @client.get_namespace(namespace_name)
-        unless metadata
-            log.trace("no metadata returned for: #{namespace_name}") if log.trace?
-            @stats.bump(:namespace_cache_api_nil_not_found)
-        else
-          begin
-            log.trace("raw metadata for #{namespace_name}: #{metadata}") if log.trace?
-            metadata = parse_namespace_metadata(metadata)
-            @stats.bump(:namespace_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
-            @namespace_cache[metadata['namespace_id']] = metadata
-            return metadata
-          rescue Exception => e
-            log.debug(e)
-            @stats.bump(:namespace_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
-          end
-        end
-      rescue Exception => kube_error
-        @stats.bump(:namespace_cache_api_nil_error)
-        log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      end
+      namespace_object = @client.get_namespace(namespace_name)
+      log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
+      metadata = parse_namespace_metadata(namespace_object)
+      @stats.bump(:namespace_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
+      @namespace_cache[metadata['namespace_id']] = metadata
+    rescue => kube_error
+      @stats.bump(:namespace_cache_api_nil_error)
+      log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       {}
     end
 
@@ -174,7 +146,7 @@ module Fluent::Plugin
       require 'lru_redux'
       @stats = KubernetesMetadata::Stats.new
 
-      if @de_dot && (@de_dot_separator =~ /\./).present?
+      if @de_dot && @de_dot_separator.include?(".")
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
@@ -201,7 +173,7 @@ module Fluent::Plugin
 
         env_host = ENV['KUBERNETES_SERVICE_HOST']
         env_port = ENV['KUBERNETES_SERVICE_PORT']
-        if env_host.present? && env_port.present?
+        if present?(env_host) && present?(env_port)
           if env_host =~ Resolv::IPv6::Regex
             # Brackets are needed around IPv6 addresses
             env_host = "[#{env_host}]"
@@ -219,22 +191,21 @@ module Fluent::Plugin
         ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
         pod_token = File.join(@secret_dir, K8_POD_TOKEN)
 
-        if !@ca_file.present? and File.exist?(ca_cert)
+        if !present?(@ca_file) and File.exist?(ca_cert)
           log.debug "Found CA certificate: #{ca_cert}"
           @ca_file = ca_cert
         end
 
-        if !@bearer_token_file.present? and File.exist?(pod_token)
+        if !present?(@bearer_token_file) and File.exist?(pod_token)
           log.debug "Found pod token: #{pod_token}"
           @bearer_token_file = pod_token
         end
       end
 
-      if @kubernetes_url.present?
-
+      if present?(@kubernetes_url)
         ssl_options = {
-            client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
-            client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+            client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+            client_key:  present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
             ca_file:     @ca_file,
             verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         }
@@ -256,15 +227,19 @@ module Fluent::Plugin
 
         auth_options = {}
 
-        if @bearer_token_file.present?
+        if present?(@bearer_token_file)
           bearer_token = File.read(@bearer_token_file)
           auth_options[:bearer_token] = bearer_token
         end
 
         log.debug "Creating K8S client"
-        @client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
-                                         ssl_options: ssl_options,
-                                         auth_options: auth_options
+        @client = Kubeclient::Client.new(
+          @kubernetes_url,
+          @apiVersion,
+          ssl_options: ssl_options,
+          auth_options: auth_options,
+          as: :parsed_symbolized
+        )
 
         begin
           @client.api_valid?
@@ -305,7 +280,7 @@ module Fluent::Plugin
           'pod_name'        => pod_name
         }
       }
-      if @kubernetes_url.present?
+      if present?(@kubernetes_url)
         pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
 
         if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
@@ -396,5 +371,9 @@ module Fluent::Plugin
       end
     end
 
+    # copied from activesupport
+    def present?(object)
+      object.respond_to?(:empty?) ? !object.empty? : !!object
+    end
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -19,6 +19,12 @@
 module KubernetesMetadata
   module Common
 
+    class GoneError < StandardError
+      def initialize(msg="410 Gone")
+        super
+      end
+    end
+
     def match_annotations(annotations)
       result = {}
       @annotations_regexps.each do |regexp|
@@ -33,65 +39,65 @@ module KubernetesMetadata
 
     def parse_namespace_metadata(namespace_object)
       labels = String.new
-      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h) unless @skip_labels
+      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
 
-      annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
+      annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
       if @de_dot
         self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
       kubernetes_metadata = {
-        'namespace_id' => namespace_object['metadata']['uid'],
-        'creation_timestamp' => namespace_object['metadata']['creationTimestamp']
+        'namespace_id' => namespace_object[:metadata][:uid],
+        'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
       }
       kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
       kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
-      return kubernetes_metadata
+      kubernetes_metadata
     end
 
     def parse_pod_metadata(pod_object)
       labels = String.new
-      labels = syms_to_strs(pod_object['metadata']['labels'].to_h) unless @skip_labels
+      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
 
-      annotations = match_annotations(syms_to_strs(pod_object['metadata']['annotations'].to_h))
+      annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
       if @de_dot
         self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
 
-      # collect container informations
+      # collect container information
       container_meta = {}
       begin
-        pod_object['status']['containerStatuses'].each do|container_status|
+        pod_object[:status][:containerStatuses].each do|container_status|
           # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
+          container_id = container_status[:containerID].sub /^[-_a-zA-Z0-9]+:\/\//, ''
           unless @skip_container_metadata
             container_meta[container_id] = {
-                'name' => container_status['name'],
-                'image' => container_status['image'],
-                'image_id' => container_status['imageID']
+                'name' => container_status[:name],
+                'image' => container_status[:image],
+                'image_id' => container_status[:imageID]
             }
           else
             container_meta[container_id] = {
-                'name' => container_status['name']
+                'name' => container_status[:name]
             }
           end
         end
       rescue
-        log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
+        log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ")
       end
 
       kubernetes_metadata = {
-          'namespace_name' => pod_object['metadata']['namespace'],
-          'pod_id'         => pod_object['metadata']['uid'],
-          'pod_name'       => pod_object['metadata']['name'],
+          'namespace_name' => pod_object[:metadata][:namespace],
+          'pod_id'         => pod_object[:metadata][:uid],
+          'pod_name'       => pod_object[:metadata][:name],
           'containers'     => syms_to_strs(container_meta),
-          'host'           => pod_object['spec']['nodeName']
+          'host'           => pod_object[:spec][:nodeName]
       }
       kubernetes_metadata['annotations'] = annotations unless annotations.empty?
       kubernetes_metadata['labels'] = labels unless labels.empty?
       kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
-      return kubernetes_metadata
+      kubernetes_metadata
     end
 
     def syms_to_strs(hsh)
@@ -111,29 +117,3 @@ module KubernetesMetadata
 
   end
 end
-
-# copied from activesupport
-class Object
-  # An object is blank if it's false, empty, or a whitespace string.
-  # For example, +nil+, '', '   ', [], {}, and +false+ are all blank.
-  #
-  # This simplifies
-  #
-  #   !address || address.empty?
-  #
-  # to
-  #
-  #   address.blank?
-  #
-  # @return [true, false]
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-
-  # An object is present if it's not blank.
-  #
-  # @return [true, false]
-  def present?
-    !blank?
-  end
-end