fluent-plugin-k8s-metrics-agg 1.1.4 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9e5776b755e33b3424b6f69fdb6780e258399a14ea14f0240a4c8d9e228950e
-  data.tar.gz: 8c65a1ee8e6638b1df574dd5e93c567692cf449a084a097bd1c9afb70f2b71c2
+  metadata.gz: 1ed001cdfb56d96374f5fc2d09ee318827286b4fadefd0295d93b682cbe6921c
+  data.tar.gz: 2d863a2438ac373c19252f551c83c9e4d2b18e8042b656747597c093497e8a69
 SHA512:
-  metadata.gz: 5fd7390614b3e3d817d7a6886983b26fc9da16f29d5e890a6d1e99be2c7614cd7ef5ca0779cde3f8f4d54201a0ea752621d95166a90358be97800d13b79c76cc
-  data.tar.gz: 51b3b36d36951a0e82240dba44c5667d1670f6ac1e71bff460d16bef954f3a1e7341a372ab614b13c7f95ee63945e8b3316dd9639f2d12e9abf74fe88d21fcda
+  metadata.gz: 6230fa97e57503d1e120cb84e116be613429a441a4c0c7bd156001a6283a467ee5f430ff7baa269fc8df099564b8595222f62515d71f07fb57c3a58b8919ad13
+  data.tar.gz: 6b3acd0eff4fdb3155eb1cbc5629ec2a254965504ba647200067101d1bbe10a8ae33fddff6929f4264826bca343281a8c1082b30653fe8c8d5fbce0ac5f38f6a

data/.circleci/config.yml

@@ -41,7 +41,7 @@ jobs:
           command: |
             bundle exec rake test -t -v
 
-  func_test_master:
+  func_test_main:
     docker:
       - image: circleci/ruby:2.6.1-node-browsers
     working_directory: ~/repo
@@ -52,7 +52,7 @@ jobs:
       - run:
           name: trigger
           command: |
-            .circleci/trigger_func_test.sh master
+            .circleci/trigger_func_test.sh main
 
   func_test_develop:
     docker:
@@ -76,11 +76,11 @@ workflows:
             branches:
               ignore: 
                 - /^release\/.*/
-                - master
+                - main
       - unit_test:
           requires:
             - build
-      - func_test_master:
+      - func_test_main:
           requires:
             - unit_test
       - func_test_develop:

data/.circleci/trigger_func_test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# trigger SCK 'master' branch to introduce this new image from this commit 
+# trigger SCK 'main' branch to introduce this new image from this commit 
 # to working version of every other component.
 ORGANIZATION=splunk
 PROJECT=splunk-connect-for-kubernetes

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-k8s-metrics-agg (1.1.4)
-      fluentd (~> 1.9.1)
+    fluent-plugin-k8s-metrics-agg (1.1.5)
+      fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
       oj (~> 3.10.2)
@@ -10,31 +10,32 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    concurrent-ruby (1.1.6)
-    cool.io (1.6.0)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    docile (1.3.1)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.1)
+    crack (0.4.5)
+      rexml
+    docile (1.3.5)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.12.2)
+    ffi (1.14.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.9.2)
+    fluentd (1.12.1)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    hashdiff (0.3.8)
-    http (4.3.0)
+    hashdiff (1.0.1)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
@@ -42,33 +43,33 @@ GEM
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.2.0)
-    http-parser (1.2.1)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.6.0)
-    json (2.3.1)
+    json (2.5.1)
     kubeclient (4.6.0)
       http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
-    msgpack (1.3.3)
+    mime-types-data (3.2021.0225)
+    msgpack (1.4.2)
     multi_json (1.14.1)
     netrc (0.11.0)
-    oj (3.10.2)
-    power_assert (1.1.3)
-    public_suffix (3.0.3)
-    rake (13.0.1)
-    recursive-open-struct (1.1.0)
+    oj (3.10.18)
+    power_assert (2.0.0)
+    public_suffix (4.0.6)
+    rake (13.0.3)
+    recursive-open-struct (1.1.3)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    safe_yaml (1.0.4)
-    serverengine (2.2.1)
+    rexml (3.2.4)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.16.1)
@@ -76,16 +77,16 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    strptime (0.2.3)
-    test-unit (3.3.0)
+    strptime (0.2.5)
+    test-unit (3.3.9)
       power_assert
-    tzinfo (2.0.1)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2019.3)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.7.7)
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -104,4 +105,4 @@ DEPENDENCIES
   webmock (~> 3.5.1)
 
 BUNDLED WITH
-   2.1.4
+   2.2.7

data/Makefile

@@ -1,10 +1,11 @@
 VERSION := $(shell sh -c 'cat VERSION')
+NODEJS_VERSION := 14.15.1
 
 clean_pkg: 
 	@rm -rf pkg/* docker/*.gem 
 
 clean_gems:
-	@rm -rf docker/gem/ docker/gems/
+	@rm -rf docker/gem/* docker/gems/*
 
 clean: clean_pkg clean_gems
 	@rm -rf docker/licenses
@@ -16,7 +17,7 @@ docker: build install-deps
 	@cp pkg/fluent-plugin-*.gem docker
 	@mkdir -p docker/licenses
 	@cp -rp LICENSE docker/licenses/
-	@docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics-aggr:$(VERSION) ./docker
+	@docker build --no-cache --pull --build-arg VERSION=$(VERSION) --build-arg NODEJS_VERSION=$(NODEJS_VERSION) -t splunk/k8s-metrics-aggr:$(VERSION) ./docker
 
 unit-test: 
 	@bundle exec rake test
@@ -28,6 +29,7 @@ install-deps:
 
 unpack: build
 	@cp pkg/fluent-plugin-*.gem docker
-	@rm docker/gem/*
+	@mkdir -p docker/gem
+	@rm -rf docker/gem
 	@gem unpack docker/fluent-plugin-*.gem --target docker/gem
 	@cd docker && bundle install

data/VERSION

@@ -1 +1 @@
-1.1.4
+1.1.5

data/docker/Dockerfile

@@ -1,6 +1,7 @@
-FROM registry.access.redhat.com/ubi8/ruby-25
+FROM registry.access.redhat.com/ubi8/ruby-27
 
 ARG VERSION
+ARG NODEJS_VERSION
 
 LABEL name="Splunk Connect for Kubernetes Metrics Aggregator container" \
       maintainer="DataEdge@splunk.com" \
@@ -19,9 +20,13 @@ COPY licenses /licenses
 COPY *.gem /tmp/
 
 COPY Gemfile* ./
-RUN gem install bundler \
+RUN yum update -y \
+   && npm install -g n \
+   && yum remove -y nodejs \
+   && n ${NODEJS_VERSION} \
+   && gem install bundler \
    && gem unpack /tmp/*.gem --target gem \
-   && bundle install
+   && bundle install 
 
 RUN groupadd -r $FLUENT_USER && \
   useradd -r -g $FLUENT_USER $FLUENT_USER && \

data/docker/Gemfile

@@ -3,15 +3,16 @@ source 'https://rubygems.org'
 # This is separate gemfile for building docker image that has all plugins 
 # for kubernetes log collection agent
 # List all required gems here and install via bundler to resolve dependencies
-gem "fluentd", "=1.9.1" 
-gem "fluent-plugin-prometheus", "=1.7.0"
+gem "fluentd", "=1.11.5" 
+gem "fluent-plugin-prometheus", "=1.8.5"
 gem "fluent-plugin-record-modifier", "=2.1.0"
+gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3"
 gem "kubeclient", "=4.6.0"
 gem "oj", "=3.10.2"
 gem 'multi_json', '=1.14.1'
 gem 'http_parser.rb', '=0.5.3'
-gem 'bigdecimal', '=1.4.3'
+gem 'bigdecimal', '=3.0.0'
 
-gem "fluent-plugin-splunk-hec", ">= 1.2.3"
+gem "fluent-plugin-splunk-hec", ">= 1.2.5"
 
 gem 'fluent-plugin-k8s-metrics-agg', path: 'gem/'

data/docker/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: gem
   specs:
-    fluent-plugin-k8s-metrics-agg (1.1.4)
-      fluentd (~> 1.9.1)
+    fluent-plugin-k8s-metrics-agg (1.1.5)
+      fluentd (>= 1.9.1)
       kubeclient (~> 4.6.0)
       multi_json (~> 1.14.1)
       oj (~> 3.10.2)
@@ -10,50 +10,55 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.4.3)
-      activesupport (= 5.2.4.3)
-    activesupport (5.2.4.3)
+    activemodel (6.1.3.1)
+      activesupport (= 6.1.3.1)
+    activesupport (6.1.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    aes_key_wrap (1.0.1)
+    aes_key_wrap (1.1.0)
     attr_required (1.0.1)
-    bigdecimal (1.4.3)
-    bindata (2.4.4)
-    concurrent-ruby (1.1.6)
-    connection_pool (2.2.2)
-    cool.io (1.6.0)
+    bigdecimal (3.0.0)
+    bindata (2.4.8)
+    concurrent-ruby (1.1.8)
+    connection_pool (2.2.3)
+    cool.io (1.7.1)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.12.2)
+    ffi (1.15.0)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluent-plugin-prometheus (1.7.0)
-      fluentd (>= 0.14.20, < 2)
+    fluent-plugin-kubernetes_metadata_filter (2.5.3)
+      fluentd (>= 0.14.0, < 1.12)
+      kubeclient (< 5)
+      lru_redux
+    fluent-plugin-prometheus (1.8.5)
+      fluentd (>= 1.9.1, < 2)
       prometheus-client (< 0.10)
     fluent-plugin-record-modifier (2.1.0)
       fluentd (>= 1.0, < 2)
-    fluent-plugin-splunk-hec (1.2.3)
+    fluent-plugin-splunk-hec (1.2.5)
       fluentd (>= 1.4)
       multi_json (~> 1.13)
       net-http-persistent (~> 3.1)
       openid_connect (~> 1.1.8)
       prometheus-client (< 0.10.0)
-    fluentd (1.9.1)
+    fluentd (1.11.5)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    http (4.3.0)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
@@ -61,14 +66,14 @@ GEM
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.2.0)
-    http-parser (1.2.1)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
     http_parser.rb (0.5.3)
     httpclient (2.8.3)
-    i18n (1.8.2)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
-    json-jwt (1.11.0)
+    json-jwt (1.13.0)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
@@ -76,14 +81,15 @@ GEM
       http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
+    lru_redux (1.1.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
-    mini_mime (1.0.2)
-    minitest (5.14.0)
-    msgpack (1.3.3)
+    mime-types-data (3.2021.0225)
+    mini_mime (1.0.3)
+    minitest (5.14.4)
+    msgpack (1.4.2)
     multi_json (1.14.1)
     net-http-persistent (3.1.0)
       connection_pool (~> 2.2)
@@ -101,7 +107,7 @@ GEM
       webfinger (>= 1.0.1)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
-    public_suffix (4.0.3)
+    public_suffix (4.0.6)
     quantile (0.2.1)
     rack (2.2.3)
     rack-oauth2 (1.16.0)
@@ -110,50 +116,51 @@ GEM
       httpclient
       json-jwt (>= 1.11.0)
       rack (>= 2.1.0)
-    rake (13.0.1)
-    recursive-open-struct (1.1.0)
+    rake (13.0.3)
+    recursive-open-struct (1.1.3)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    serverengine (2.2.1)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.2.3)
-    swd (1.1.2)
+    strptime (0.2.5)
+    swd (1.2.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
       httpclient (>= 2.4)
-    thread_safe (0.3.6)
-    tzinfo (1.2.6)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2019.3)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
+    unf_ext (0.0.7.7)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
-    validate_url (1.0.8)
+    validate_url (1.0.13)
       activemodel (>= 3.0.0)
       public_suffix
     webfinger (1.1.0)
       activesupport
       httpclient (>= 2.4)
     yajl-ruby (1.4.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bigdecimal (= 1.4.3)
+  bigdecimal (= 3.0.0)
   fluent-plugin-k8s-metrics-agg!
-  fluent-plugin-prometheus (= 1.7.0)
+  fluent-plugin-kubernetes_metadata_filter (= 2.5.3)
+  fluent-plugin-prometheus (= 1.8.5)
   fluent-plugin-record-modifier (= 2.1.0)
-  fluent-plugin-splunk-hec (>= 1.2.3)
-  fluentd (= 1.9.1)
+  fluent-plugin-splunk-hec (>= 1.2.5)
+  fluentd (= 1.11.5)
   http_parser.rb (= 0.5.3)
   kubeclient (= 4.6.0)
   multi_json (= 1.14.1)

data/docker/build.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TAG=$1
+NODEJS_VERSION=14.15.1
 
 # Install dependecies
 gem install bundler
@@ -16,4 +17,4 @@ VERSION=`cat VERSION`
 echo "Copying licenses to be included in the docker image..."
 mkdir -p docker/licenses
 cp -rp LICENSE docker/licenses/
-docker build --no-cache --pull --build-arg VERSION=$VERSION --no-cache -t splunk/k8s-metrics-aggr:$TAG ./docker
+docker build --no-cache --pull --build-arg VERSION=$VERSION --build-arg NODEJS_VERSION=$NODEJS_VERSION --no-cache -t splunk/k8s-metrics-aggr:$TAG ./docker

data/fluent-plugin-k8s-metrics-agg.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'test-unit', "~> 3.3.0"
   spec.add_development_dependency "simplecov", '~> 0.16.1'
   spec.add_development_dependency 'webmock', '~> 3.5.1'
-  spec.add_runtime_dependency 'fluentd', '~> 1.9.1'
+  spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
   spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
   spec.add_runtime_dependency 'multi_json', '~> 1.14.1'
   spec.add_runtime_dependency 'oj', '~> 3.10.2'

data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb

@@ -575,23 +575,25 @@ module Fluent
 
             node_response = JSON.parse(node_rest_client.get(@client.headers))
             Array(node_response['pods']).each do |pod_json|
-              pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
-              pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0)
-              pod_namespace = pod_json['podRef']['namespace']
-              pod_usage = ResourceUsageMetricsUnit.new
-              pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
-              if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
-                namespace_usage_metrics = ResourceUsageMetricsUnit.new
-                @@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
-              else
-                @@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
-              end
-              if @@node_resource_usage_metrics_map[node_name].nil?
-                node_name_usage_metrics = ResourceUsageMetricsUnit.new
-                @@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
+              unless pod_json['cpu'].nil? || pod_json['memory'].nil?
+                pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
+                pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0)
+                pod_namespace = pod_json['podRef']['namespace']
+                pod_usage = ResourceUsageMetricsUnit.new
+                pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
+                if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
+                  namespace_usage_metrics = ResourceUsageMetricsUnit.new
+                  @@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
+                else
+                  @@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
+                end
+                if @@node_resource_usage_metrics_map[node_name].nil?
+                  node_name_usage_metrics = ResourceUsageMetricsUnit.new
+                  @@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
+                end
+                @@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
+                pod_usage = nil
               end
-              @@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
-              pod_usage = nil
             end
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-k8s-metrics-agg
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.5
 platform: ruby
 authors:
 - Splunk Inc.
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-06 00:00:00.000000000 Z
+date: 2021-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,14 +84,14 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.1
 - !ruby/object:Gem::Dependency
@@ -185,7 +185,7 @@ homepage: https://github.com/splunk/fluent-plugin-k8s-metrics-agg
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A fluentd input plugin that collects kubernetes cluster metrics.
 test_files: