fluent-plugin-kubernetes-metrics 1.1.7 → 1.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7953f3fade3d0c126f70d9b6922d6b36eccb92461903a723975eb5cb29ee958
-  data.tar.gz: ceb8d47bbb69b1de2bfafa6c9c05cf8f5ad2909db2d432c7eed78721f6c985f5
+  metadata.gz: 7cba1f817de8b2ebcb68a6f96dcb0776105c9330ed7b7e6d46888bd1fe2a8290
+  data.tar.gz: 79b5be5e27f54edad68981e9f48f8827fe698c93584115287ea41b594591e928
 SHA512:
-  metadata.gz: 5a310b0ac5c2f1af6cdb34c9c74413dc6b1e4bef7a42f996dc6741a512a80f1fc1822302c313f57ef6772f0a6416b61b64188fe75ff1a3c2a3cee9a969c13828
-  data.tar.gz: 63dec59922164f39a20eb8ed51bb4f11d464811544eee28b591bf9eba79a7b29f29c3ffb80a89496f7abe433a0fd8b5872f8fe7d841e7bff5f6919a438977b45
+  metadata.gz: 4d4fd43d51357507698126408a618d38e81d7a5a9a956a3aafe4c46521b9d95bf22d680c52fca119d9c3f236b60dcdf028368948cc258981a64e34cd0f1e4ce8
+  data.tar.gz: 03ff9d6abf9511af983f52c1083062bc389fd976e0baf3045609df3a81e833e8ae2c787c5078870921a76246c37f3f2e01c3e20270ba0e0ca502aaa7105d8011

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-metrics (1.1.7)
+    fluent-plugin-kubernetes-metrics (1.1.10)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
@@ -19,14 +19,14 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.15.4)
+    ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.2)
+    fluentd (1.14.4)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
@@ -47,16 +47,16 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.7.0)
+    http_parser.rb (0.8.0)
     json (2.6.1)
     kubeclient (4.6.0)
       http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
-    msgpack (1.4.2)
+    mime-types-data (3.2022.0105)
+    msgpack (1.4.4)
     multi_json (1.14.1)
     netrc (0.11.0)
     oj (3.10.18)
@@ -70,7 +70,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    serverengine (2.2.4)
+    serverengine (2.2.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.16.1)
@@ -107,4 +107,4 @@ DEPENDENCIES
   webmock (~> 3.5.1)
 
 BUNDLED WITH
-   2.2.30
+   2.3.9

data/VERSION

@@ -1 +1 @@
-1.1.7
+1.1.10

data/lib/fluent/plugin/in_kubernetes_metrics.rb

@@ -90,8 +90,13 @@ module Fluent
         super
 
         timer_execute :metric_scraper, @interval, &method(:scrape_metrics)
-        timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
         timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics)
+        # It is done to optionally fetch from 'stats' for k8s version <1.21
+        if is_stats_endpoint_available?
+          timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics)
+        else
+          log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards"
+        end
       end
 
       def close
@@ -310,7 +315,7 @@ module Fluent
         unless metrics['time'].nil?
           time = parse_time metrics['time']
           if usage_rate = metrics['usageNanoCores']
-            router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000)
+            router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000.0)
           end
           if usage = metrics['usageNanoCores']
             router.emit generate_tag("#{tag}.cpu.usage"), time, labels.merge('value' => usage)
@@ -411,13 +416,13 @@ module Fluent
 
       def emit_cpu_metrics_stats(tag:, metrics:, labels:, time:)
         if cpu_usage_total = metrics['usage']['total']
-          router.emit generate_tag("#{tag}.cpu.usage.total"), time, labels.merge('value' => cpu_usage_total / 1_000_000)
+          router.emit generate_tag("#{tag}.cpu.usage.total"), time, labels.merge('value' => cpu_usage_total / 1_000_000.0)
         end
         if cpu_usage_user = metrics['usage']['user']
-          router.emit generate_tag("#{tag}.cpu.usage.user"), time, labels.merge('value' => cpu_usage_user / 1_000_000)
+          router.emit generate_tag("#{tag}.cpu.usage.user"), time, labels.merge('value' => cpu_usage_user / 1_000_000.0)
         end
         if cpu_usage_system = metrics['usage']['system']
-          router.emit generate_tag("#{tag}.cpu.usage.system"), time, labels.merge('value' => cpu_usage_system / 1_000_000)
+          router.emit generate_tag("#{tag}.cpu.usage.system"), time, labels.merge('value' => cpu_usage_system / 1_000_000.0)
         end
 
         if cpu_cfs_periods = metrics['cfs']['periods']
@@ -571,8 +576,42 @@ module Fluent
 
         unless pod['startTime'].nil?
           emit_uptime tag: tag, start_time: pod['startTime'], labels: labels
-          emit_cpu_metrics tag: tag, metrics: pod['cpu'], labels: labels if pod['cpu'] unless pod['cpu'].nil?
-          emit_memory_metrics tag: tag, metrics: pod['memory'], labels: labels if pod['memory'] unless pod['memory'].nil?
+          if pod['cpu'].nil?
+            if pod['containers'].nil? or Array(pod['containers']).empty?
+              log.warn "Summary API response has no pod cpu metrics information"
+            else
+              usageNanoCores = 0
+              usageCoreNanoSeconds = 0
+              time = nil
+              Array(pod['containers']).each do |container|
+                time = container['time'] unless container['time'].nil?
+                usageNanoCores += container['usageNanoCores']
+                usageCoreNanoSeconds += container['usageCoreNanoSeconds']
+              end
+              pod['cpu'] = { 'time' => time, 'usageNanoCores' => usageNanoCores, 'usageCoreNanoSeconds' => usageCoreNanoSeconds }
+            end
+          end
+          emit_cpu_metrics tag: tag, metrics: pod['cpu'], labels: labels unless pod['cpu'].nil?
+          if pod['memory'].nil?
+            if pod['containers'].nil? or Array(pod['containers']).empty?
+              log.warn "Summary API response has no pod memory metrics information"
+            else
+              Array(pod['containers']).each do |container|
+                time = nil
+                memory_metrics = {}
+                %w[availableBytes usageBytes workingSetBytes rssBytes pageFaults majorPageFaults].each do |name|
+                  time = container['time'] unless container['time'].nil?
+                  if value = metrics[name]
+                    memory_metrics[name] = 0 if memory_metrics[name].nil?
+                    memory_metrics[name] += value 
+                  end
+                end
+              end
+              memory_metrics['time'] = time
+              pod['memory'] = memory_metrics
+            end
+          end
+          emit_memory_metrics tag: tag, metrics: pod['memory'], labels: labels unless pod['memory'].nil?
           emit_network_metrics tag: tag, metrics: pod['network'], labels: labels unless pod['network'].nil?
           emit_fs_metrics tag: "#{tag}.ephemeral-storage", metrics: pod['ephemeral-storage'], labels: labels unless pod['ephemeral-storage'].nil?
           unless pod['volume'].nil?
@@ -590,6 +629,7 @@ module Fluent
 
       def emit_metrics(metrics)
         emit_node_metrics(metrics['node']) unless metrics['node'].nil?
+        log.warn "Summary API received empty pods info" if (metrics['pods'].nil? or metrics['pods'].empty?)
         Array(metrics['pods']).each &method(:emit_pod_metrics).curry.call(metrics['node']['nodeName']) unless metrics['pods'].nil?
       end
 
@@ -597,33 +637,35 @@ module Fluent
         emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil?
       end
 
+      # Make sure regex has only one capturing group
+      def grep_using_regex(metric, regex)
+        match = metric.match(regex)
+        return nil if match.nil?
+        match[1]
+      end
+
       def emit_cadvisor_metrics(metrics)
         metrics = metrics.split("\n")
         metrics.each do |metric|
-          next unless metric.include? 'container_name='
-
-          next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
-
+          
+          next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/)
+          next if container_name.empty?
+          
           metric_str, metric_val = metric.split(' ')
           metric_val = metric_val.to_f if metric_val.is_a? String
           first_occur = metric_str.index('{')
           metric_name = metric_str[0..first_occur - 1]
-          pod_name = metric.match(/pod_name="\S*"/).to_s
-          pod_name = pod_name.split('"')[1]
-          image_name = metric.match(/image="\S*"/).to_s
-          image_name = image_name.split('"')[1]
-          namespace = metric.match(/namespace="\S*"/).to_s
-          namespace = namespace.split('"')[1]
+          pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s
+          image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s
+          namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s
           metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-          if metric =~ /^((?!container_name="POD").)*$/
+          if container_name=="POD"
             tag = 'pod'
             tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}")
             tag = tag.gsub('container', '')
           else
-            container_name = metric.match(/container_name="\S*"/).to_s
-            container_name = container_name.split('"')[1]
             container_label = { 'container_name' => container_name }
-            metric_labels.merge(container_label)
+            metric_labels.merge!(container_label)
             tag = generate_tag(metric_name.tr('_', '.').to_s)
           end
           router.emit tag, @scraped_at_cadvisor, metric_labels
@@ -642,6 +684,20 @@ module Fluent
         end
       end
 
+      def is_stats_endpoint_available?
+        if @use_rest_client
+          response_stats = RestClient::Request.execute request_options_stats
+        else
+          @node_names.each do |node|
+            @node_name = node
+            response_stats = stats_proxy_api(node).get(@client.headers)
+          end
+        end
+        true
+        rescue RestClient::NotFound
+          false
+      end
+
       def scrape_stats_metrics
         if @use_rest_client
           response_stats = RestClient::Request.execute request_options_stats

data/test/plugin/test_in_kubernetes_metrics.rb

@@ -35,6 +35,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
   ).freeze
 
   setup do
+    stub_k8s_requests
+
+    return unless @@hash_map_test.empty?
     Fluent::Test.setup
 
     @@parsed_unit_string = JSON.parse(get_unit_parsed_string)
@@ -45,8 +48,6 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       get_cadvisor_parsed_string = f.read
     end.close
 
-    stub_k8s_requests
-
     @@ca_driver = create_driver
     @@ca_driver.run timeout: 20, expect_emits: 1, shutdown: true
 
@@ -56,8 +57,12 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
     metrics = get_cadvisor_parsed_string.split("\n")
     metrics.each do |metric|
       next unless metric.include? 'container_name='
+      next unless  metric[0] != '#'
 
-      next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#'
+      container_name = metric.match(/container_name="\S*"/).to_s
+      container_name = container_name.split('"')[1]
+      
+      next if container_name.empty?
 
       metric_str, metric_val = metric.split(' ')
       metric_val = metric_val.to_f if metric_val.is_a? String
@@ -70,13 +75,11 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       namespace = metric.match(/namespace="\S*"/).to_s
       namespace = namespace.split('"')[1]
       metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name }
-      if metric =~ /^((?!container_name="POD").)*$/
+      if container_name == 'POD'
         tag = 'pod'
         tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}", @@driver.instance.tag)
         tag = tag.gsub('container', '')
       else
-        container_name = metric.match(/container_name="\S*"/).to_s
-        container_name = container_name.split('"')[1]
         container_label = { 'container_name' => container_name }
         metric_labels.merge(container_label)
         tag = generate_tag(metric_name.tr('_', '.').to_s, @@driver.instance.tag)
@@ -112,7 +115,7 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'], @@hash_map_test['kube.node.cpu.usage'][2]['value']
 
       assert_not_nil @@hash_map_test.key?('kube.node.cpu.usage_rate')
-      assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'] / 1_000_000, @@hash_map_test['kube.node.cpu.usage_rate'][2]['value']
+      assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'] / 1_000_000.0, @@hash_map_test['kube.node.cpu.usage_rate'][2]['value']
     end
 
     test 'test_emit_memory_metrics' do
@@ -252,11 +255,10 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_true @@hash_map_cadvisor.key?('kube.container.fs.read.seconds.total')
       assert_equal @@hash_map_cadvisor['kube.container.fs.read.seconds.total'], @@hash_map_test['kube.container.fs.read.seconds.total'][2]['value']
     end
-
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
+    
     test 'Test - metrics cadvisor: container_fs_reads_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_reads_merged_total' do
@@ -289,10 +291,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
       assert_equal @@hash_map_cadvisor['kube.container.fs.write.seconds.total'], @@hash_map_test['kube.container.fs.write.seconds.total'][2]['value']
     end
 
-    # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production
     test 'Test - metrics cadvisor: container_fs_writes_bytes_total' do
-      assert_false @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
-      # assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
+      assert_true @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total')
+      assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"]
     end
 
     test 'Test - metrics cadvisor: container_fs_writes_merged_total' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-metrics
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.10
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2022-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -180,6 +180,6 @@ signing_key:
 specification_version: 4
 summary: A fluentd input plugin that collects kubernetes cluster metrics.
 test_files:
-- test/plugin/test_missing_timestamps.rb
-- test/plugin/test_in_kubernetes_metrics.rb
 - test/helper.rb
+- test/plugin/test_in_kubernetes_metrics.rb
+- test/plugin/test_missing_timestamps.rb