fluent-plugin-kubernetes_metadata_filter 2.4.6 → 2.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 674303b83893faf1b8188fba478414257e85e78ef3ea92945849950ea56381bf
-  data.tar.gz: db56e05f338366b9204f4e94b19062edf230c7357b21ff385da0c3575ad91953
+  metadata.gz: 8eada7adb2f8305fc4f14d91c04a2f8d7293f4f622713b0e7d463d85b2ed15d3
+  data.tar.gz: 890266f08cda7757d459d99c728f7638c17f1c01ffa042c89c53d74da6c97c0e
 SHA512:
-  metadata.gz: f8c9d1ae8db09db5fe19f7f53aaff882dbad2c7f88a0da88a31f999fb396d642c8315d6f3ca7de6f7df0c88efdf68b615610d6fed629edaa7eb9ce121c1150b4
-  data.tar.gz: a0a7e76ad0ad6230c5913d394378a7df7c51ea790f20bbbaa8d507247b5cb0f70d4a6a3d778421c12f501ba2189f0b96f5792bd64376c715d9084c5d77638160
+  metadata.gz: 5874033154af1f111f41e6197295134b66ab1ac94777fe2158e112880ae15343a69449fb78d2958cc0ad7a99e8db5249bd91e074fd17993d31fd48a859fe370d
+  data.tar.gz: 6dfc784c4f9696ec7618fffaf6b95e0fe797266ea8fe13e1bc0ee9fb0e51e48f1da77b6a0d1b7f0fd3e407608ac467f58bcc9081ddaf56daaccf8fb1eef572e0

data/.circleci/config.yml

@@ -14,21 +14,21 @@ missingdeps: &missingdeps
             sudo cp /tmp/sources.list /etc/apt/sources.list
             sudo apt-get update
             sudo apt-get install cmake libicu-dev libssl-dev
-  
+
 test: &test
   name: Test bundle
   command: bundle exec rake test
 
 executors:
-  ruby-2-4:
-    docker:
-      - image: circleci/ruby:2.4.6
   ruby-2-5:
     docker:
       - image: circleci/ruby:2.5.5
   ruby-2-6:
     docker:
       - image: circleci/ruby:2.6.3
+  ruby-2-7:
+    docker:
+      - image: circleci/ruby:2.7.1
 jobs:
   "ruby-test":
     parameters:
@@ -49,9 +49,9 @@ jobs:
 workflows:
   "test_multiple_ruby_versions":
     jobs:
-      - ruby-test:
-          ruby-version: ruby-2-4
       - ruby-test:
           ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
+      - ruby-test:
+          ruby-version: ruby-2-7

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.4.6)
-      fluentd (>= 0.14.0, < 1.10)
+    fluent-plugin-kubernetes_metadata_filter (2.5.3)
+      fluentd (>= 0.14.0, < 1.12)
       kubeclient (< 5)
       lru_redux
 
@@ -11,13 +11,13 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.0)
+    ast (2.4.1)
     bump (0.9.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.0)
     copyright-header (1.0.22)
       github-linguist
     crack (0.4.3)
@@ -26,21 +26,21 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.12.2)
+    ffi (1.14.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.9.3)
+    fluentd (1.11.5)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.8.0)
+    github-linguist (7.9.0)
       charlock_holmes (~> 0.7.6)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
@@ -55,30 +55,34 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.1)
+    http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.6.0)
-    jaro_winkler (1.5.4)
-    kubeclient (4.6.0)
+    jsonpath (1.1.0)
+      multi_json
+    kubeclient (4.9.1)
       http (>= 3.0, < 5.0)
-      recursive-open-struct (~> 1.0, >= 1.0.4)
+      jsonpath (~> 1.0)
+      recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
     lru_redux (1.1.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0425)
+    mime-types-data (3.2020.1104)
     mini_mime (1.0.2)
     minitest (4.7.5)
     msgpack (1.3.3)
+    multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.19.1)
-    parser (2.7.0.4)
-      ast (~> 2.4.0)
-    power_assert (1.1.6)
-    public_suffix (4.0.3)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    power_assert (1.2.0)
+    public_suffix (4.0.5)
     rainbow (3.0.0)
     rake (13.0.1)
-    recursive-open-struct (1.1.1)
+    recursive-open-struct (1.1.3)
+    regexp_parser (1.7.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -86,39 +90,42 @@ GEM
       netrc (~> 0.8)
     rexml (3.2.4)
     rr (1.2.1)
-    rubocop (0.80.1)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.86.0)
       parallel (~> 1.10)
       parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.0.3, < 1.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.1.0)
+      parser (>= 2.7.0.1)
     ruby-progressbar (1.10.1)
-    rugged (0.99.0)
+    rugged (1.0.1)
     safe_yaml (1.0.5)
-    serverengine (2.2.1)
+    serverengine (2.2.2)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.18.5)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
     simplecov-html (0.12.2)
-    strptime (0.2.3)
+    strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
     test-unit-rr (1.0.5)
       rr (>= 1.1.1)
       test-unit (>= 2.5.2)
-    tzinfo (2.0.2)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
+    tzinfo-data (1.2020.6)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unicode-display_width (1.6.1)
-    vcr (5.1.0)
+    unicode-display_width (1.7.0)
+    vcr (6.0.0)
     webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/README.md

@@ -15,6 +15,7 @@ that rely on the authenticity of the namespace for proper log isolation.
 
 | fluent-plugin-kubernetes_metadata_filter  | fluentd | ruby |
 |-------------------|---------|------|
+| >= 2.5.0 | >= v1.10.0 | >= 2.5 |
 | >= 2.0.0 | >= v0.14.20 | >= 2.1 |
 |  < 2.0.0 | >= v0.12.0 | >= 1.9 |
 

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.4.6"
+  gem.version       = "2.5.3"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -13,13 +13,10 @@ Gem::Specification.new do |gem|
   gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 2.1.0'
+  gem.required_ruby_version = '>= 2.5.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.10']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.12']
   gem.add_runtime_dependency "lru_redux"
   gem.add_runtime_dependency "kubeclient", '< 5'
 

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -82,37 +82,23 @@ module Fluent::Plugin
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
     # The time interval in seconds for retry backoffs when watch connections fail.
-    config_param :watch_retry_interval, :bool, default: 1
+    config_param :watch_retry_interval, :integer, default: 1
     # The base number of exponential backoff for retries.
-    config_param :watch_retry_exponential_backoff_base, :bool, default: 2
+    config_param :watch_retry_exponential_backoff_base, :integer, default: 2
     # The maximum number of times to retry pod and namespace watches.
-    config_param :watch_retry_max_times, :bool, default: 10
+    config_param :watch_retry_max_times, :integer, default: 10
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
-      begin
-        metadata = @client.get_pod(pod_name, namespace_name)
-        unless metadata
-          log.trace("no metadata returned for: #{namespace_name}/#{pod_name}") if log.trace?
-          @stats.bump(:pod_cache_api_nil_not_found)
-        else
-          begin
-            log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            metadata = parse_pod_metadata(metadata)
-            @stats.bump(:pod_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
-            @cache[metadata['pod_id']] = metadata
-            return metadata
-          rescue Exception=>e
-            log.debug(e)
-            @stats.bump(:pod_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
-          end
-        end
-      rescue Exception=>e
-        @stats.bump(:pod_cache_api_nil_error)
-        log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      end
+      pod_object = @client.get_pod(pod_name, namespace_name)
+      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
+      metadata = parse_pod_metadata(pod_object)
+      @stats.bump(:pod_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
+      @cache[metadata['pod_id']] = metadata
+    rescue => e
+      @stats.bump(:pod_cache_api_nil_error)
+      log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       {}
     end
 
@@ -132,29 +118,15 @@ module Fluent::Plugin
 
     def fetch_namespace_metadata(namespace_name)
       log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
-      begin
-        metadata = @client.get_namespace(namespace_name)
-        unless metadata
-            log.trace("no metadata returned for: #{namespace_name}") if log.trace?
-            @stats.bump(:namespace_cache_api_nil_not_found)
-        else
-          begin
-            log.trace("raw metadata for #{namespace_name}: #{metadata}") if log.trace?
-            metadata = parse_namespace_metadata(metadata)
-            @stats.bump(:namespace_cache_api_updates)
-            log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
-            @namespace_cache[metadata['namespace_id']] = metadata
-            return metadata
-          rescue Exception => e
-            log.debug(e)
-            @stats.bump(:namespace_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
-          end
-        end
-      rescue Exception => kube_error
-        @stats.bump(:namespace_cache_api_nil_error)
-        log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
-      end
+      namespace_object = @client.get_namespace(namespace_name)
+      log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
+      metadata = parse_namespace_metadata(namespace_object)
+      @stats.bump(:namespace_cache_api_updates)
+      log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
+      @namespace_cache[metadata['namespace_id']] = metadata
+    rescue => kube_error
+      @stats.bump(:namespace_cache_api_nil_error)
+      log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
       {}
     end
 
@@ -174,7 +146,7 @@ module Fluent::Plugin
       require 'lru_redux'
       @stats = KubernetesMetadata::Stats.new
 
-      if @de_dot && (@de_dot_separator =~ /\./).present?
+      if @de_dot && @de_dot_separator.include?(".")
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
@@ -201,7 +173,7 @@ module Fluent::Plugin
 
         env_host = ENV['KUBERNETES_SERVICE_HOST']
         env_port = ENV['KUBERNETES_SERVICE_PORT']
-        if env_host.present? && env_port.present?
+        if present?(env_host) && present?(env_port)
           if env_host =~ Resolv::IPv6::Regex
             # Brackets are needed around IPv6 addresses
             env_host = "[#{env_host}]"
@@ -219,22 +191,21 @@ module Fluent::Plugin
         ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
         pod_token = File.join(@secret_dir, K8_POD_TOKEN)
 
-        if !@ca_file.present? and File.exist?(ca_cert)
+        if !present?(@ca_file) and File.exist?(ca_cert)
           log.debug "Found CA certificate: #{ca_cert}"
           @ca_file = ca_cert
         end
 
-        if !@bearer_token_file.present? and File.exist?(pod_token)
+        if !present?(@bearer_token_file) and File.exist?(pod_token)
           log.debug "Found pod token: #{pod_token}"
           @bearer_token_file = pod_token
         end
       end
 
-      if @kubernetes_url.present?
-
+      if present?(@kubernetes_url)
         ssl_options = {
-            client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
-            client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+            client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+            client_key:  present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
             ca_file:     @ca_file,
             verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         }
@@ -256,15 +227,19 @@ module Fluent::Plugin
 
         auth_options = {}
 
-        if @bearer_token_file.present?
+        if present?(@bearer_token_file)
           bearer_token = File.read(@bearer_token_file)
           auth_options[:bearer_token] = bearer_token
         end
 
         log.debug "Creating K8S client"
-        @client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
-                                         ssl_options: ssl_options,
-                                         auth_options: auth_options
+        @client = Kubeclient::Client.new(
+          @kubernetes_url,
+          @apiVersion,
+          ssl_options: ssl_options,
+          auth_options: auth_options,
+          as: :parsed_symbolized
+        )
 
         begin
           @client.api_valid?
@@ -305,7 +280,7 @@ module Fluent::Plugin
           'pod_name'        => pod_name
         }
       }
-      if @kubernetes_url.present?
+      if present?(@kubernetes_url)
         pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
 
         if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
@@ -396,5 +371,9 @@ module Fluent::Plugin
       end
     end
 
+    # copied from activesupport
+    def present?(object)
+      object.respond_to?(:empty?) ? !object.empty? : !!object
+    end
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -19,6 +19,12 @@
 module KubernetesMetadata
   module Common
 
+    class GoneError < StandardError
+      def initialize(msg="410 Gone")
+        super
+      end
+    end
+
     def match_annotations(annotations)
       result = {}
       @annotations_regexps.each do |regexp|
@@ -33,65 +39,65 @@ module KubernetesMetadata
 
     def parse_namespace_metadata(namespace_object)
       labels = String.new
-      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h) unless @skip_labels
+      labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
 
-      annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
+      annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
       if @de_dot
         self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
       kubernetes_metadata = {
-        'namespace_id' => namespace_object['metadata']['uid'],
-        'creation_timestamp' => namespace_object['metadata']['creationTimestamp']
+        'namespace_id' => namespace_object[:metadata][:uid],
+        'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
       }
       kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
       kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
-      return kubernetes_metadata
+      kubernetes_metadata
     end
 
     def parse_pod_metadata(pod_object)
       labels = String.new
-      labels = syms_to_strs(pod_object['metadata']['labels'].to_h) unless @skip_labels
+      labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
 
-      annotations = match_annotations(syms_to_strs(pod_object['metadata']['annotations'].to_h))
+      annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
       if @de_dot
         self.de_dot!(labels) unless @skip_labels
         self.de_dot!(annotations)
       end
 
-      # collect container informations
+      # collect container information
       container_meta = {}
       begin
-        pod_object['status']['containerStatuses'].each do|container_status|
+        pod_object[:status][:containerStatuses].each do|container_status|
           # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
+          container_id = container_status[:containerID].sub /^[-_a-zA-Z0-9]+:\/\//, ''
           unless @skip_container_metadata
             container_meta[container_id] = {
-                'name' => container_status['name'],
-                'image' => container_status['image'],
-                'image_id' => container_status['imageID']
+                'name' => container_status[:name],
+                'image' => container_status[:image],
+                'image_id' => container_status[:imageID]
             }
           else
             container_meta[container_id] = {
-                'name' => container_status['name']
+                'name' => container_status[:name]
             }
           end
         end
       rescue
-        log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
+        log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ")
       end
 
       kubernetes_metadata = {
-          'namespace_name' => pod_object['metadata']['namespace'],
-          'pod_id'         => pod_object['metadata']['uid'],
-          'pod_name'       => pod_object['metadata']['name'],
+          'namespace_name' => pod_object[:metadata][:namespace],
+          'pod_id'         => pod_object[:metadata][:uid],
+          'pod_name'       => pod_object[:metadata][:name],
           'containers'     => syms_to_strs(container_meta),
-          'host'           => pod_object['spec']['nodeName']
+          'host'           => pod_object[:spec][:nodeName]
       }
       kubernetes_metadata['annotations'] = annotations unless annotations.empty?
       kubernetes_metadata['labels'] = labels unless labels.empty?
       kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
-      return kubernetes_metadata
+      kubernetes_metadata
     end
 
     def syms_to_strs(hsh)
@@ -111,29 +117,3 @@ module KubernetesMetadata
 
   end
 end
-
-# copied from activesupport
-class Object
-  # An object is blank if it's false, empty, or a whitespace string.
-  # For example, +nil+, '', '   ', [], {}, and +false+ are all blank.
-  #
-  # This simplifies
-  #
-  #   !address || address.empty?
-  #
-  # to
-  #
-  #   address.blank?
-  #
-  # @return [true, false]
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-
-  # An object is present if it's not blank.
-  #
-  # @return [true, false]
-  def present?
-    !blank?
-  end
-end