fluent-plugin-kubernetes-metrics 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7953f3fade3d0c126f70d9b6922d6b36eccb92461903a723975eb5cb29ee958
-  data.tar.gz: ceb8d47bbb69b1de2bfafa6c9c05cf8f5ad2909db2d432c7eed78721f6c985f5
+  metadata.gz: '01494a399012dc40a3a2d9597327839453e95abc642d5510ad3e423b50c9fd25'
+  data.tar.gz: d7d5f2f45267de03f5e519b4ac48ad0b03ae9058a04f399e563c529da1664c7f
 SHA512:
-  metadata.gz: 5a310b0ac5c2f1af6cdb34c9c74413dc6b1e4bef7a42f996dc6741a512a80f1fc1822302c313f57ef6772f0a6416b61b64188fe75ff1a3c2a3cee9a969c13828
-  data.tar.gz: 63dec59922164f39a20eb8ed51bb4f11d464811544eee28b591bf9eba79a7b29f29c3ffb80a89496f7abe433a0fd8b5872f8fe7d841e7bff5f6919a438977b45
+  metadata.gz: 870c5b311bd3157f485872149ae1d360cbfc0e44a804c92863d17fe40a114854a6722f512d58f505cdbde950f851b780aba2ceb364bc5b01ab8819bb3c37fe1a
+  data.tar.gz: 99b97218aa1a33de365d6ce36f8403f774cba1e2975255253fa5147241b143009082320ffd934752a2b485a42a2d2ed9af2a06d4de0f6ebf99c38ac3ad07f5cd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-metrics (1.1.7)
+    fluent-plugin-kubernetes-metrics (1.1.8)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
@@ -107,4 +107,4 @@ DEPENDENCIES
   webmock (~> 3.5.1)
 
 BUNDLED WITH
-   2.2.30
+   2.2.33

data/VERSION

@@ -1 +1 @@
-1.1.7
+1.1.8

data/lib/fluent/plugin/in_kubernetes_metrics.rb

@@ -90,8 +90,13 @@ module Fluent
         super
 
         timer_execute :metric_scraper, @interval, &method(:scrape_metrics)
-        timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
         timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics)
+        # It is done to optionally fetch from 'stats' for k8s version <1.21
+        if is_stats_endpoint_available?
+          timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
+        else
+          log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards"
+        end
       end
 
       def close
@@ -597,33 +602,35 @@ module Fluent
         emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil?
       end
 
+      # Make sure regex has only one capturing group
+      def grep_using_regex(metric, regex)
+        match = metric.match(regex)
+        return nil if match.nil?
+        match[1]
+      end
+
       def emit_cadvisor_metrics(metrics)
         metrics = metrics.split("\n")
         metrics.each do |metric|
-          next unless metric.include? 'container_name='
-
-          next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
-
+          
+          next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/)
+          next if container_name.empty?
+          
           metric_str, metric_val = metric.split(' ')
           metric_val = metric_val.to_f if metric_val.is_a? String
           first_occur = metric_str.index('{')
           metric_name = metric_str[0..first_occur - 1]
-          pod_name = metric.match(/pod_name="\S*"/).to_s
-          pod_name = pod_name.split('"')[1]
-          image_name = metric.match(/image="\S*"/).to_s
-          image_name = image_name.split('"')[1]
-          namespace = metric.match(/namespace="\S*"/).to_s
-          namespace = namespace.split('"')[1]
+          pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s
+          image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s
+          namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s
           metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-          if metric =~ /^((?!container_name="POD").)*$/
+          if container_name=="POD"
             tag = 'pod'
             tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}")
             tag = tag.gsub('container', '')
           else
-            container_name = metric.match(/container_name="\S*"/).to_s
-            container_name = container_name.split('"')[1]
             container_label = { 'container_name' => container_name }
-            metric_labels.merge(container_label)
+            metric_labels.merge!(container_label)
             tag = generate_tag(metric_name.tr('_', '.').to_s)
           end
           router.emit tag, @scraped_at_cadvisor, metric_labels
@@ -642,6 +649,20 @@ module Fluent
         end
       end
 
+      def is_stats_endpoint_available?
+        if @use_rest_client
+          response_stats = RestClient::Request.execute request_options_stats
+        else
+          @node_names.each do |node|
+            @node_name = node
+            response_stats = stats_proxy_api(node).get(@client.headers)
+          end
+        end
+        true
+        rescue RestClient::NotFound
+          false
+      end
+
       def scrape_stats_metrics
         if @use_rest_client
           response_stats = RestClient::Request.execute request_options_stats

data/test/plugin/test_in_kubernetes_metrics.rb

@@ -35,6 +35,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
   ).freeze
 
   setup do
+    stub_k8s_requests
+
+    return unless @@hash_map_test.empty?
     Fluent::Test.setup
 
     @@parsed_unit_string = JSON.parse(get_unit_parsed_string)
@@ -45,8 +48,6 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       get_cadvisor_parsed_string = f.read
     end.close
 
-    stub_k8s_requests
-
     @@ca_driver = create_driver
     @@ca_driver.run timeout: 20, expect_emits: 1, shutdown: true
 
@@ -56,8 +57,12 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
     metrics = get_cadvisor_parsed_string.split("\n")
     metrics.each do |metric|
       next unless metric.include? 'container_name='
+      next unless  metric[0] != '#'
 
-      next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
+      container_name = metric.match(/container_name="\S*"/).to_s
+      container_name = container_name.split('"')[1]
+      
+      next if container_name.empty?
 
       metric_str, metric_val = metric.split(' ')
       metric_val = metric_val.to_f if metric_val.is_a? String
@@ -70,13 +75,11 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       namespace = metric.match(/namespace="\S*"/).to_s
       namespace = namespace.split('"')[1]
       metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-      if metric =~ /^((?!container_name="POD").)*$/
+      if container_name == 'POD'
         tag = 'pod'
         tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}", @@driver.instance.tag)
         tag = tag.gsub('container', '')
       else
-        container_name = metric.match(/container_name="\S*"/).to_s
-        container_name = container_name.split('"')[1]
         container_label = { 'container_name' => container_name }
         metric_labels.merge(container_label)
         tag = generate_tag(metric_name.tr('_', '.').to_s, @@driver.instance.tag)
@@ -252,11 +255,10 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_true @@hash_map_cadvisor.key?('kube.container.fs.read.seconds.total')
       assert_equal @@hash_map_cadvisor['kube.container.fs.read.seconds.total'], @@hash_map_test['kube.container.fs.read.seconds.total'][2]['value']
     end
-
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
+    
     test 'Test - metrics cadvisor: container_fs_reads_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_reads_merged_total' do
@@ -289,10 +291,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_equal @@hash_map_cadvisor['kube.container.fs.write.seconds.total'], @@hash_map_test['kube.container.fs.write.seconds.total'][2]['value']
     end
 
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
     test 'Test - metrics cadvisor: container_fs_writes_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_writes_merged_total' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-metrics
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -180,6 +180,6 @@ signing_key:
 specification_version: 4
 summary: A fluentd input plugin that collects kubernetes cluster metrics.
 test_files:
+- test/helper.rb
 - test/plugin/test_missing_timestamps.rb
 - test/plugin/test_in_kubernetes_metrics.rb
-- test/helper.rb