fluent-plugin-k8s-metrics-agg 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11e9b540230a927f8cb2dd469d4b16d16941c8e7cde2d4019394509c8562874e
-  data.tar.gz: 38e231e8cde3aa56f386c3cc835065fec257ecffe502866ae0db8e00a818f970
+  metadata.gz: 98a2dd49ad2df50f48934daa1f41f92d9466a0c87149b74a059755e2a12b55a1
+  data.tar.gz: 4549d0845d04a933c89a6599a18b15ef83e468c0e739c5b9f66458b0c5f8116f
 SHA512:
-  metadata.gz: 298e621affd96c9185a95e019af9bf947d4f1a81bdb850606fc9c9729ba0f7ec3072ad4df4f176e4280ae70bba1a012d59d183ecd858cc80f1c085c4b99ec44b
-  data.tar.gz: 3ea6fe8cd14f6820a8c3126cc8fc226b28d06786e467fdba6cbdc8d8b1445724a11406a4d28850f87685959ac59170be0e45fab8cee35570476167fb96f3f01e
+  metadata.gz: c1acc729827a62b91a8cf8fa33f230d651eaf733e66d395ba4610ff2b55407013e9c3fdb8b4db6d8913a4e6320fbe37f04b910b62de3c894216c347bacb86be5
+  data.tar.gz: 79c4d760559ef7edb0789133aa7d9c7e1a89cfd28b3ac71df146dd755082338b1c480a9b95940fc15f54b1984e22cf81f72e6c48f4b7e8c93354a1aaed9985b6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-k8s-metrics-agg (1.1.7)
+    fluent-plugin-k8s-metrics-agg (1.1.8)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
@@ -107,4 +107,4 @@ DEPENDENCIES
   webmock (~> 3.5.1)
 
 BUNDLED WITH
-   2.2.30
+   2.2.33

data/VERSION

@@ -1 +1 @@
-1.1.7
+1.1.8

data/docker/Dockerfile

@@ -37,7 +37,7 @@ RUN yum update -y \
    && gem install bundler \
    && gem unpack /tmp/*.gem --target gem \
    && bundle install \
-   && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-*  glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-*  git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* 
+   && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-*  glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-*  git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true
 
 RUN groupadd -r $FLUENT_USER && \
   useradd -r -g $FLUENT_USER $FLUENT_USER && \

data/docker/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: gem
   specs:
-    fluent-plugin-k8s-metrics-agg (1.1.7)
+    fluent-plugin-k8s-metrics-agg (1.1.8)
       fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
@@ -10,9 +10,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (6.1.4.3)
+      activesupport (= 6.1.4.3)
+    activesupport (6.1.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -33,7 +33,7 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluent-plugin-kubernetes_metadata_filter (2.9.2)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -42,16 +42,16 @@ GEM
       prometheus-client (>= 2.1.0)
     fluent-plugin-record-modifier (2.1.0)
       fluentd (>= 1.0, < 2)
-    fluent-plugin-splunk-hec (1.2.7)
+    fluent-plugin-splunk-hec (1.2.9)
       fluentd (>= 1.4)
       multi_json (~> 1.13)
       net-http-persistent (~> 3.1)
       openid_connect (~> 1.1.8)
       prometheus-client (>= 2.1.0)
-    fluentd (1.14.2)
+    fluentd (1.14.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
@@ -86,11 +86,11 @@ GEM
     lru_redux (1.1.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
+    mime-types-data (3.2021.1115)
     mini_mime (1.1.2)
-    minitest (5.14.4)
+    minitest (5.15.0)
     msgpack (1.4.2)
     multi_json (1.14.1)
     net-http-persistent (3.1.0)

data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb

@@ -51,11 +51,11 @@ module Fluent
           # m cpu is assumed standard
           @cpu_mult = 1
           @cpu_mult = 1000 if cpu[-1] != 'm'
-          cpu.delete('^0-9').to_i
+          cpu.delete('^0-9.').to_i
         end
 
         def get_cpu_or_memory_value(resource)
-          resource = resource.tr('^0-9', '').to_i
+          resource = resource.tr('^0-9.', '').to_i
           resource
         end
 
@@ -66,29 +66,35 @@ module Fluent
         end
 
         # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory
+        # 1 Ki = 1024 bytes
+        # 1 K = 1000 bytes = 1000/1024 Ki = 1000/1024*1024 Mi
         def get_memory_mult(memory)
-          memory_mult = if memory[-2] == 'Ki'
-                          0.001
-                        elsif memory[-2] == 'K'
-                          1.0 / 1024
-                        elsif memory[-2] == 'Mi'
+          memory_mult = if memory[-2..] == 'Ki'
+                          1.0/1024
+                        elsif memory[-1] == 'K'
+                          1e3 / 1024 ** 2
+                        elsif memory[-2..] == 'Mi'
                           1
-                        elsif memory[-2] == 'M'
-                          1
-                        elsif memory[-2] == 'Gi'
-                          1000
-                        elsif memory[-2] == 'G'
+                        elsif memory[-1] == 'M'
+                          1e6 / 1024 ** 2
+                        elsif memory[-2..] == 'Gi'
                           1024
-                        elsif memory[-2] == 'Ti'
-                          1_000_000
-                        elsif memory[-2] == 'T'
-                          1_048_576
-                        elsif memory[-2] == 'Ei'
-                          1_000_000_000
+                        elsif memory[-1] == 'G'
+                          1e9 / 1024 ** 2
+                        elsif memory[-2..] == 'Ti'
+                          1024 ** 2
+                        elsif memory[-1] == 'T'
+                          1e12 / 1024 ** 2
+                        elsif memory[-2..] == 'Pi'
+                          1024**3
+                        elsif memory[-2] == 'P'
+                          1e15 / 1024 ** 2
+                        elsif memory[-2..] == 'Ei'
+                          1024**4
                         elsif memory[-2] == 'E'
-                          1_073_741_824
+                          1e18 / 1024 ** 2
                         else
-                          0.000001
+                          1 / 1024 ** 2
                         end
           memory_mult
         end
@@ -278,43 +284,47 @@ module Fluent
       end
 
       def get_cpu_value(resource)
-        cpu_val = resource.tr('^0-9', '').to_i
+        cpu_val = resource.tr('^0-9.', '').to_i
         mult = get_cpu_mult(resource)
-        cpu_val += cpu_val * mult
+        cpu_val = cpu_val * mult
         cpu_val
       end
 
       def get_memory_mult(memory)
-        memory_mult = if memory[-2] == 'Ki'
-                        0.001
-                      elsif memory[-2] == 'K'
-                        1.0 / 1024
-                      elsif memory[-2] == 'Mi'
-                        1
-                      elsif memory[-2] == 'M'
+        memory_mult = if memory[-2..] == 'Ki'
+                        1.0/1024
+                      elsif memory[-1] == 'K'
+                        1e3 / 1024 ** 2
+                      elsif memory[-2..] == 'Mi'
                         1
-                      elsif memory[-2] == 'Gi'
-                        1000
-                      elsif memory[-2] == 'G'
+                      elsif memory[-1] == 'M'
+                        1e6 / 1024 ** 2
+                      elsif memory[-2..] == 'Gi'
                         1024
-                      elsif memory[-2] == 'Ti'
-                        1_000_000
-                      elsif memory[-2] == 'T'
-                        1_048_576 # 1024*1024
-                      elsif memory[-2] == 'Ei'
-                        1_000_000_000
+                      elsif memory[-1] == 'G'
+                        1e9 / 1024 ** 2
+                      elsif memory[-2..] == 'Ti'
+                        1024 ** 2
+                      elsif memory[-1] == 'T'
+                        1e12 / 1024 ** 2
+                      elsif memory[-2..] == 'Pi'
+                        1024**3
+                      elsif memory[-2] == 'P'
+                        1e15 / 1024 ** 2
+                      elsif memory[-2..] == 'Ei'
+                        1024**4
                       elsif memory[-2] == 'E'
-                        1_073_741_824 # 1024*1024*1024
+                        1e18 / 1024 ** 2
                       else
-                        0.000001
+                        1 / 1024 ** 2
                       end
         memory_mult
       end
 
       def get_memory_value(resource)
-        mem_val = resource.tr('^0-9', '').to_i
+        mem_val = resource.tr('^0-9.', '').to_i
         mult = get_memory_mult(resource)
-        mem_val += mem_val * mult
+        mem_val = mem_val * mult
         mem_val
       end
 
@@ -513,7 +523,7 @@ module Fluent
           router.emit generate_tag('node') << ('.cpu.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_utilization
           node_cpu_reservation = node_req_lim.instance_variable_get(:@cpu_request).to_f / node_cpu_allocatable
           router.emit generate_tag('node') << ('.cpu.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_reservation
-          node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / 1_000_000 * node_memory_allocatable # converting from bytes to megabytes
+          node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f /  node_memory_allocatable # converting from bytes to megabytes
           router.emit generate_tag('node') << ('.memory.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_utilization
           node_memory_reservation = node_req_lim.instance_variable_get(:@memory_request).to_f / node_memory_allocatable
           router.emit generate_tag('node') << ('.memory.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_reservation
@@ -573,11 +583,17 @@ module Fluent
                 end
               end
 
-            node_response = JSON.parse(node_rest_client.get(@client.headers))
+            begin
+              node_response = JSON.parse(node_rest_client.get(@client.headers))
+            rescue RestClient::ServiceUnavailable
+              log.warn("Couldn't scrap metric for node '#{node_name} as it is unavailable. Ignoring it.'")
+              next
+            end
+            
             Array(node_response['pods']).each do |pod_json|
               unless pod_json['cpu'].nil? || pod_json['memory'].nil?
                 pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
-                pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0)
+                pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0) / 1024 ** 2 # Converting to Mi
                 pod_namespace = pod_json['podRef']['namespace']
                 pod_usage = ResourceUsageMetricsUnit.new
                 pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-k8s-metrics-agg
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-03 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A fluentd input plugin that collects kubernetes cluster metrics.