fluent-plugin-kubernetes-metrics 1.1.4 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca11780c53ec485dd7470ee5b0d644983f75b25d55f7625666fd682868a0e361
-  data.tar.gz: 90a5ffe9e4430c108e863e26fc3dfdeb4c6b667e7683a1de7d1d05de8825e06b
+  metadata.gz: '01494a399012dc40a3a2d9597327839453e95abc642d5510ad3e423b50c9fd25'
+  data.tar.gz: d7d5f2f45267de03f5e519b4ac48ad0b03ae9058a04f399e563c529da1664c7f
 SHA512:
-  metadata.gz: e236979695bbf3cf714d9fc8e43fa9a0214deeb7e033ef039f81c19290c5cffd090567a72dc9bc57be3cf201e5a656cf38ba924c5439c7f33978eeeb21f21af5
-  data.tar.gz: 60d7d6b96b01f92a1754bfdf9e152eff7f07987f6a85ab00cdbfde23001ee19356a8b89d62da9f9be47b95d07fc1a8b7194efd00bdc610e7b4a3ce3328827881
+  metadata.gz: 870c5b311bd3157f485872149ae1d360cbfc0e44a804c92863d17fe40a114854a6722f512d58f505cdbde950f851b780aba2ceb364bc5b01ab8819bb3c37fe1a
+  data.tar.gz: 99b97218aa1a33de365d6ce36f8403f774cba1e2975255253fa5147241b143009082320ffd934752a2b485a42a2d2ed9af2a06d4de0f6ebf99c38ac3ad07f5cd

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-metrics (1.1.4)
-      fluentd (~> 1.9.1)
+    fluent-plugin-kubernetes-metrics (1.1.8)
+      fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
       oj (~> 3.10.2)
@@ -10,65 +10,67 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    docile (1.3.1)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    crack (0.4.5)
+      rexml
+    docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.12.2)
+    ffi (1.15.4)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.9.2)
+    fluentd (1.14.2)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    hashdiff (0.3.8)
-    http (4.3.0)
+    hashdiff (1.0.1)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
     http-accept (1.7.0)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    http-form_data (2.2.0)
-    http-parser (1.2.1)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.6.0)
-    json (2.3.1)
+    http_parser.rb (0.7.0)
+    json (2.6.1)
     kubeclient (4.6.0)
       http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
-    msgpack (1.3.3)
+    mime-types-data (3.2021.0901)
+    msgpack (1.4.2)
     multi_json (1.14.1)
     netrc (0.11.0)
-    oj (3.10.2)
-    power_assert (1.1.3)
-    public_suffix (3.0.3)
-    rake (13.0.1)
-    recursive-open-struct (1.1.0)
+    oj (3.10.18)
+    power_assert (2.0.1)
+    public_suffix (4.0.6)
+    rake (13.0.6)
+    recursive-open-struct (1.1.3)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    safe_yaml (1.0.4)
-    serverengine (2.2.1)
+    rexml (3.2.5)
+    serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.16.1)
@@ -76,20 +78,21 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    strptime (0.2.3)
-    test-unit (3.3.0)
+    strptime (0.2.5)
+    test-unit (3.3.9)
       power_assert
-    tzinfo (2.0.1)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2019.3)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.8)
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
+    webrick (1.7.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
@@ -104,4 +107,4 @@ DEPENDENCIES
   webmock (~> 3.5.1)
 
 BUNDLED WITH
-   2.1.4
+   2.2.33

data/README.md

@@ -1,4 +1,3 @@
-[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-kubernetes-metrics)
 # Fluentd Plugin for Kubernetes Metrics
 
 The [Fluentd](https://fluentd.org/) input plugin collects Kubernetes cluster metrics which are exposed by the [Kubelet API](https://kubernetes.io/docs/admin/kubelet/) and forwards them to fluentd.

data/VERSION

@@ -1 +1 @@
-1.1.4
+1.1.8

data/fluent-plugin-kubernetes-metrics.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov', '~> 0.16.1'
   spec.add_development_dependency 'test-unit', '~> 3.3.0'
   spec.add_development_dependency 'webmock', '~> 3.5.1'
-  spec.add_runtime_dependency 'fluentd', '~> 1.9.1'
+  spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
   spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
   spec.add_runtime_dependency 'multi_json', '~> 1.14.1'
   spec.add_runtime_dependency 'oj', '~> 3.10.2'

data/lib/fluent/plugin/in_kubernetes_metrics.rb

@@ -90,8 +90,13 @@ module Fluent
         super
 
         timer_execute :metric_scraper, @interval, &method(:scrape_metrics)
-        timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
         timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics)
+        # It is done to optionally fetch from 'stats' for k8s version <1.21
+        if is_stats_endpoint_available?
+          timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
+        else
+          log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards"
+        end
       end
 
       def close
@@ -597,33 +602,35 @@ module Fluent
         emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil?
       end
 
+      # Make sure regex has only one capturing group
+      def grep_using_regex(metric, regex)
+        match = metric.match(regex)
+        return nil if match.nil?
+        match[1]
+      end
+
       def emit_cadvisor_metrics(metrics)
         metrics = metrics.split("\n")
         metrics.each do |metric|
-          next unless metric.include? 'container_name='
-
-          next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
-
+          
+          next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/)
+          next if container_name.empty?
+          
           metric_str, metric_val = metric.split(' ')
           metric_val = metric_val.to_f if metric_val.is_a? String
           first_occur = metric_str.index('{')
           metric_name = metric_str[0..first_occur - 1]
-          pod_name = metric.match(/pod_name="\S*"/).to_s
-          pod_name = pod_name.split('"')[1]
-          image_name = metric.match(/image="\S*"/).to_s
-          image_name = image_name.split('"')[1]
-          namespace = metric.match(/namespace="\S*"/).to_s
-          namespace = namespace.split('"')[1]
+          pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s
+          image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s
+          namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s
           metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-          if metric =~ /^((?!container_name="POD").)*$/
+          if container_name=="POD"
             tag = 'pod'
             tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}")
             tag = tag.gsub('container', '')
           else
-            container_name = metric.match(/container_name="\S*"/).to_s
-            container_name = container_name.split('"')[1]
             container_label = { 'container_name' => container_name }
-            metric_labels.merge(container_label)
+            metric_labels.merge!(container_label)
             tag = generate_tag(metric_name.tr('_', '.').to_s)
           end
           router.emit tag, @scraped_at_cadvisor, metric_labels
@@ -642,6 +649,20 @@ module Fluent
         end
       end
 
+      def is_stats_endpoint_available?
+        if @use_rest_client
+          response_stats = RestClient::Request.execute request_options_stats
+        else
+          @node_names.each do |node|
+            @node_name = node
+            response_stats = stats_proxy_api(node).get(@client.headers)
+          end
+        end
+        true
+        rescue RestClient::NotFound
+          false
+      end
+
       def scrape_stats_metrics
         if @use_rest_client
           response_stats = RestClient::Request.execute request_options_stats

data/test/plugin/test_in_kubernetes_metrics.rb

@@ -35,6 +35,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
   ).freeze
 
   setup do
+    stub_k8s_requests
+
+    return unless @@hash_map_test.empty?
     Fluent::Test.setup
 
     @@parsed_unit_string = JSON.parse(get_unit_parsed_string)
@@ -45,8 +48,6 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       get_cadvisor_parsed_string = f.read
     end.close
 
-    stub_k8s_requests
-
     @@ca_driver = create_driver
     @@ca_driver.run timeout: 20, expect_emits: 1, shutdown: true
 
@@ -56,8 +57,12 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
     metrics = get_cadvisor_parsed_string.split("\n")
     metrics.each do |metric|
       next unless metric.include? 'container_name='
+      next unless  metric[0] != '#'
 
-      next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
+      container_name = metric.match(/container_name="\S*"/).to_s
+      container_name = container_name.split('"')[1]
+      
+      next if container_name.empty?
 
       metric_str, metric_val = metric.split(' ')
       metric_val = metric_val.to_f if metric_val.is_a? String
@@ -70,13 +75,11 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       namespace = metric.match(/namespace="\S*"/).to_s
       namespace = namespace.split('"')[1]
       metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-      if metric =~ /^((?!container_name="POD").)*$/
+      if container_name == 'POD'
         tag = 'pod'
         tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}", @@driver.instance.tag)
         tag = tag.gsub('container', '')
       else
-        container_name = metric.match(/container_name="\S*"/).to_s
-        container_name = container_name.split('"')[1]
         container_label = { 'container_name' => container_name }
         metric_labels.merge(container_label)
         tag = generate_tag(metric_name.tr('_', '.').to_s, @@driver.instance.tag)
@@ -252,11 +255,10 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_true @@hash_map_cadvisor.key?('kube.container.fs.read.seconds.total')
       assert_equal @@hash_map_cadvisor['kube.container.fs.read.seconds.total'], @@hash_map_test['kube.container.fs.read.seconds.total'][2]['value']
     end
-
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
+    
     test 'Test - metrics cadvisor: container_fs_reads_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_reads_merged_total' do
@@ -289,10 +291,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_equal @@hash_map_cadvisor['kube.container.fs.write.seconds.total'], @@hash_map_test['kube.container.fs.write.seconds.total'][2]['value']
     end
 
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
     test 'Test - metrics cadvisor: container_fs_writes_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_writes_merged_total' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-metrics
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.8
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,14 +84,14 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.1
 - !ruby/object:Gem::Dependency
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A fluentd input plugin that collects kubernetes cluster metrics.