fluent-plugin-k8s-metrics-agg 1.1.0 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -24,6 +24,6 @@ namespace :docker do
24
24
  raise 'Argument `tag` was not provided.' unless args.tag
25
25
 
26
26
  cp Dir['pkg/fluent-plugin-k8s-metrics-agg-*.gem'], 'docker/'
27
- sh "docker build --no-cache -t splunk/fluent-plugin-k8s-metrics-agg:#{args.tag} ./docker"
27
+ sh "docker build --no-cache --pull --build-arg VERSION=$(cat VERSION) -t splunk/fluent-plugin-k8s-metrics-agg:#{args.tag} ./docker"
28
28
  end
29
29
  end
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.1.0
1
+ 1.1.5
data/docker/Dockerfile CHANGED
@@ -1,52 +1,37 @@
1
- FROM alpine:3.8
2
-
3
- LABEL maintainer="Splunk Inc. <DataEdge@splunk.com>"
4
- LABEL Description="Splunk Connect for Kubernetes docker image" Vendor="Splunk Inc."
1
+ FROM registry.access.redhat.com/ubi8/ruby-27
5
2
 
6
3
  ARG VERSION
4
+ ARG NODEJS_VERSION
5
+
6
+ LABEL name="Splunk Connect for Kubernetes Metrics Aggregator container" \
7
+ maintainer="DataEdge@splunk.com" \
8
+ vendor="Splunk Inc." \
9
+ version=${VERSION} \
10
+ release=${VERSION} \
11
+ summary="Splunk Connect for Kubernetes Metrics Aggregator container" \
12
+ description="Splunk Connect for Kubernetes Metrics Aggregator container"
13
+
7
14
  ENV VERSION=${VERSION}
15
+ ENV FLUENT_USER fluent
8
16
 
9
- ENV DUMB_INIT_VERSION=1.2.1
10
- ENV SU_EXEC_VERSION=0.2
17
+ USER root
18
+ COPY licenses /licenses
11
19
 
12
20
  COPY *.gem /tmp/
13
21
 
14
- ARG DEBIAN_FRONTEND=noninteractive
15
- # Do not split this into multiple RUN!
16
- # Docker creates a layer for every RUN-Statement
17
- # therefore an 'apk delete' has no effect
18
- RUN apk update \
19
- && apk upgrade \
20
- && apk add --no-cache \
21
- ca-certificates \
22
- && update-ca-certificates \
23
- && apk add --no-cache \
24
- ruby ruby-irb ruby-etc ruby-webrick ruby-json \
25
- su-exec==${SU_EXEC_VERSION}-r0 \
26
- dumb-init==${DUMB_INIT_VERSION}-r0 \
27
- && apk add --no-cache --virtual .build-deps \
28
- build-base \
29
- ruby-dev wget gnupg \
30
- && echo 'gem: --no-document' >> /etc/gemrc \
31
- && gem install -N \
32
- fluentd:1.4.0 \
33
- fluent-plugin-record-modifier:2.0.1 \
34
- fluent-plugin-splunk-hec:${VERSION} \
35
- oj:3.7.9 \
36
- multi_json:1.13.1 \
37
- bigdecimal:1.4.3 \
38
- kubeclient:4.3.0 \
39
- && gem install -N /tmp/*.gem \
40
- && apk del .build-deps \
41
- && rm -rf /var/cache/apk/* \
42
- && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
43
-
44
- ENV LD_PRELOAD="" \
45
- FLUENTD_CONF="fluent.conf" \
46
- FLUENTD_OPT="" \
47
- DUMB_INIT_SETSID=0
48
-
49
- COPY entrypoint.sh /bin/entrypoint.sh
50
- ENTRYPOINT ["/bin/entrypoint.sh"]
51
-
52
- CMD exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins $FLUENTD_OPT
22
+ COPY Gemfile* ./
23
+ RUN yum update -y \
24
+ && npm install -g n \
25
+ && yum remove -y nodejs \
26
+ && n ${NODEJS_VERSION} \
27
+ && gem install bundler \
28
+ && gem unpack /tmp/*.gem --target gem \
29
+ && bundle install
30
+
31
+ RUN groupadd -r $FLUENT_USER && \
32
+ useradd -r -g $FLUENT_USER $FLUENT_USER && \
33
+ mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins &&\
34
+ chown -R $FLUENT_USER /fluentd && chgrp -R $FLUENT_USER /fluentd
35
+
36
+ USER $FLUENT_USER
37
+ CMD bundle exec fluentd -c /fluentd/etc/fluent.conf
data/docker/Gemfile ADDED
@@ -0,0 +1,18 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # This is separate gemfile for building docker image that has all plugins
4
+ # for kubernetes log collection agent
5
+ # List all required gems here and install via bundler to resolve dependencies
6
+ gem "fluentd", "=1.11.5"
7
+ gem "fluent-plugin-prometheus", "=1.8.5"
8
+ gem "fluent-plugin-record-modifier", "=2.1.0"
9
+ gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3"
10
+ gem "kubeclient", "=4.6.0"
11
+ gem "oj", "=3.10.2"
12
+ gem 'multi_json', '=1.14.1'
13
+ gem 'http_parser.rb', '=0.5.3'
14
+ gem 'bigdecimal', '=3.0.0'
15
+
16
+ gem "fluent-plugin-splunk-hec", ">= 1.2.5"
17
+
18
+ gem 'fluent-plugin-k8s-metrics-agg', path: 'gem/'
@@ -0,0 +1,170 @@
1
+ PATH
2
+ remote: gem
3
+ specs:
4
+ fluent-plugin-k8s-metrics-agg (1.1.5)
5
+ fluentd (>= 1.9.1)
6
+ kubeclient (~> 4.6.0)
7
+ multi_json (~> 1.14.1)
8
+ oj (~> 3.10.2)
9
+
10
+ GEM
11
+ remote: https://rubygems.org/
12
+ specs:
13
+ activemodel (6.1.3.1)
14
+ activesupport (= 6.1.3.1)
15
+ activesupport (6.1.3.1)
16
+ concurrent-ruby (~> 1.0, >= 1.0.2)
17
+ i18n (>= 1.6, < 2)
18
+ minitest (>= 5.1)
19
+ tzinfo (~> 2.0)
20
+ zeitwerk (~> 2.3)
21
+ addressable (2.7.0)
22
+ public_suffix (>= 2.0.2, < 5.0)
23
+ aes_key_wrap (1.1.0)
24
+ attr_required (1.0.1)
25
+ bigdecimal (3.0.0)
26
+ bindata (2.4.8)
27
+ concurrent-ruby (1.1.8)
28
+ connection_pool (2.2.3)
29
+ cool.io (1.7.1)
30
+ domain_name (0.5.20190701)
31
+ unf (>= 0.0.5, < 1.0.0)
32
+ ffi (1.15.0)
33
+ ffi-compiler (1.0.1)
34
+ ffi (>= 1.0.0)
35
+ rake
36
+ fluent-plugin-kubernetes_metadata_filter (2.5.3)
37
+ fluentd (>= 0.14.0, < 1.12)
38
+ kubeclient (< 5)
39
+ lru_redux
40
+ fluent-plugin-prometheus (1.8.5)
41
+ fluentd (>= 1.9.1, < 2)
42
+ prometheus-client (< 0.10)
43
+ fluent-plugin-record-modifier (2.1.0)
44
+ fluentd (>= 1.0, < 2)
45
+ fluent-plugin-splunk-hec (1.2.5)
46
+ fluentd (>= 1.4)
47
+ multi_json (~> 1.13)
48
+ net-http-persistent (~> 3.1)
49
+ openid_connect (~> 1.1.8)
50
+ prometheus-client (< 0.10.0)
51
+ fluentd (1.11.5)
52
+ cool.io (>= 1.4.5, < 2.0.0)
53
+ http_parser.rb (>= 0.5.1, < 0.7.0)
54
+ msgpack (>= 1.3.1, < 2.0.0)
55
+ serverengine (>= 2.2.2, < 3.0.0)
56
+ sigdump (~> 0.2.2)
57
+ strptime (>= 0.2.2, < 1.0.0)
58
+ tzinfo (>= 1.0, < 3.0)
59
+ tzinfo-data (~> 1.0)
60
+ yajl-ruby (~> 1.0)
61
+ http (4.4.1)
62
+ addressable (~> 2.3)
63
+ http-cookie (~> 1.0)
64
+ http-form_data (~> 2.2)
65
+ http-parser (~> 1.2.0)
66
+ http-accept (1.7.0)
67
+ http-cookie (1.0.3)
68
+ domain_name (~> 0.5)
69
+ http-form_data (2.3.0)
70
+ http-parser (1.2.3)
71
+ ffi-compiler (>= 1.0, < 2.0)
72
+ http_parser.rb (0.5.3)
73
+ httpclient (2.8.3)
74
+ i18n (1.8.9)
75
+ concurrent-ruby (~> 1.0)
76
+ json-jwt (1.13.0)
77
+ activesupport (>= 4.2)
78
+ aes_key_wrap
79
+ bindata
80
+ kubeclient (4.6.0)
81
+ http (>= 3.0, < 5.0)
82
+ recursive-open-struct (~> 1.0, >= 1.0.4)
83
+ rest-client (~> 2.0)
84
+ lru_redux (1.1.0)
85
+ mail (2.7.1)
86
+ mini_mime (>= 0.1.1)
87
+ mime-types (3.3.1)
88
+ mime-types-data (~> 3.2015)
89
+ mime-types-data (3.2021.0225)
90
+ mini_mime (1.0.3)
91
+ minitest (5.14.4)
92
+ msgpack (1.4.2)
93
+ multi_json (1.14.1)
94
+ net-http-persistent (3.1.0)
95
+ connection_pool (~> 2.2)
96
+ netrc (0.11.0)
97
+ oj (3.10.2)
98
+ openid_connect (1.1.8)
99
+ activemodel
100
+ attr_required (>= 1.0.0)
101
+ json-jwt (>= 1.5.0)
102
+ rack-oauth2 (>= 1.6.1)
103
+ swd (>= 1.0.0)
104
+ tzinfo
105
+ validate_email
106
+ validate_url
107
+ webfinger (>= 1.0.1)
108
+ prometheus-client (0.9.0)
109
+ quantile (~> 0.2.1)
110
+ public_suffix (4.0.6)
111
+ quantile (0.2.1)
112
+ rack (2.2.3)
113
+ rack-oauth2 (1.16.0)
114
+ activesupport
115
+ attr_required
116
+ httpclient
117
+ json-jwt (>= 1.11.0)
118
+ rack (>= 2.1.0)
119
+ rake (13.0.3)
120
+ recursive-open-struct (1.1.3)
121
+ rest-client (2.1.0)
122
+ http-accept (>= 1.7.0, < 2.0)
123
+ http-cookie (>= 1.0.2, < 2.0)
124
+ mime-types (>= 1.16, < 4.0)
125
+ netrc (~> 0.8)
126
+ serverengine (2.2.3)
127
+ sigdump (~> 0.2.2)
128
+ sigdump (0.2.4)
129
+ strptime (0.2.5)
130
+ swd (1.2.0)
131
+ activesupport (>= 3)
132
+ attr_required (>= 0.0.5)
133
+ httpclient (>= 2.4)
134
+ tzinfo (2.0.4)
135
+ concurrent-ruby (~> 1.0)
136
+ tzinfo-data (1.2021.1)
137
+ tzinfo (>= 1.0.0)
138
+ unf (0.1.4)
139
+ unf_ext
140
+ unf_ext (0.0.7.7)
141
+ validate_email (0.1.6)
142
+ activemodel (>= 3.0)
143
+ mail (>= 2.2.5)
144
+ validate_url (1.0.13)
145
+ activemodel (>= 3.0.0)
146
+ public_suffix
147
+ webfinger (1.1.0)
148
+ activesupport
149
+ httpclient (>= 2.4)
150
+ yajl-ruby (1.4.1)
151
+ zeitwerk (2.4.2)
152
+
153
+ PLATFORMS
154
+ ruby
155
+
156
+ DEPENDENCIES
157
+ bigdecimal (= 3.0.0)
158
+ fluent-plugin-k8s-metrics-agg!
159
+ fluent-plugin-kubernetes_metadata_filter (= 2.5.3)
160
+ fluent-plugin-prometheus (= 1.8.5)
161
+ fluent-plugin-record-modifier (= 2.1.0)
162
+ fluent-plugin-splunk-hec (>= 1.2.5)
163
+ fluentd (= 1.11.5)
164
+ http_parser.rb (= 0.5.3)
165
+ kubeclient (= 4.6.0)
166
+ multi_json (= 1.14.1)
167
+ oj (= 3.10.2)
168
+
169
+ BUNDLED WITH
170
+ 2.1.4
data/docker/build.sh ADDED
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env bash
2
+ set -e
3
+ TAG=$1
4
+ NODEJS_VERSION=14.15.1
5
+
6
+ # Install dependecies
7
+ gem install bundler
8
+ bundle update --bundler
9
+ bundle install
10
+
11
+ # Build Gem
12
+ bundle exec rake build -t -v
13
+ cp pkg/fluent-plugin-*.gem docker
14
+
15
+ # Build Docker Image
16
+ VERSION=`cat VERSION`
17
+ echo "Copying licenses to be included in the docker image..."
18
+ mkdir -p docker/licenses
19
+ cp -rp LICENSE docker/licenses/
20
+ docker build --no-cache --pull --build-arg VERSION=$VERSION --build-arg NODEJS_VERSION=$NODEJS_VERSION --no-cache -t splunk/k8s-metrics-aggr:$TAG ./docker
@@ -1,4 +1,4 @@
1
- lib = File.expand_path("../lib", __FILE__)
1
+ lib = File.expand_path('lib', __dir__)
2
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
@@ -20,14 +20,14 @@ Gem::Specification.new do |spec|
20
20
  spec.test_files = test_files
21
21
  spec.require_paths = ["lib"]
22
22
 
23
- spec.add_development_dependency 'bundler', '~> 2.0.1'
24
- spec.add_development_dependency 'rake', '~> 12.3.2'
23
+ spec.add_development_dependency 'bundler', '~> 2.0'
24
+ spec.add_development_dependency 'rake', '~> 13.0'
25
25
  spec.add_development_dependency 'test-unit', "~> 3.3.0"
26
26
  spec.add_development_dependency "simplecov", '~> 0.16.1'
27
27
  spec.add_development_dependency 'webmock', '~> 3.5.1'
28
- spec.add_runtime_dependency 'fluentd', '~> 1.3.3'
29
- spec.add_runtime_dependency 'kubeclient', '~> 4.2.2'
30
- spec.add_runtime_dependency 'multi_json', '~> 1.13.1'
31
- spec.add_runtime_dependency 'oj', '~> 3.7.8'
28
+ spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
29
+ spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
30
+ spec.add_runtime_dependency 'multi_json', '~> 1.14.1'
31
+ spec.add_runtime_dependency 'oj', '~> 3.10.2'
32
32
 
33
33
  end
@@ -355,7 +355,7 @@ module Fluent
355
355
  begin
356
356
  @client.discover unless @client.discovered
357
357
  @client.rest_client['/pods'].tap do |endpoint|
358
- log.info("Use URL #{endpoint.url} for scraping limits requests metrics")
358
+ log.debug("Use URL #{endpoint.url} for scraping limits requests metrics")
359
359
  end
360
360
  end
361
361
  end
@@ -363,6 +363,9 @@ module Fluent
363
363
  def scrape_limits_requests_metrics
364
364
  response = limits_requests_api.get(@client.headers)
365
365
  handle_limits_requests_res(response)
366
+ rescue StandardError => e
367
+ log.error "Failed to get limit metrics, error=#{$ERROR_INFO}, #{e.inspect}"
368
+ log.error_backtrace
366
369
  end
367
370
 
368
371
  # This method is used to handle responses from the kube apiserver api
@@ -375,7 +378,7 @@ module Fluent
375
378
  log.error "ExMultiJson.load(response.body) expected 2xx from summary API, but got #{response.code}. Response body = #{response.body}"
376
379
  end
377
380
  rescue StandardError => e
378
- log.error "Failed to scrape metrics, error=#{$ERROR_INFO}, #{e.inspect}"
381
+ log.error "Failed to scrape limit metrics, error=#{$ERROR_INFO}, #{e.inspect}"
379
382
  log.error_backtrace
380
383
  end
381
384
 
@@ -420,7 +423,7 @@ module Fluent
420
423
  pod_usage_metrics.add_usage_metrics(cpu_limit, cpu_request, memory_limit, memory_request)
421
424
  end
422
425
 
423
- pod_labels = { 'name' => pod_json['metadata']['name'], 'namespace' => pod_json['metadata']['name'], 'node' => pod_json['spec']['nodeName'] }
426
+ pod_labels = { 'name' => pod_json['metadata']['name'], 'namespace' => pod_json['metadata']['namespace'], 'node' => pod_json['spec']['nodeName'] }
424
427
  emit_limits_requests_metrics(generate_tag('pod'), @scraped_at, pod_labels, pod_usage_metrics)
425
428
  @@namespace_usage_metrics_map[pod_namespace].add_usage_metrics(pod_usage_metrics.instance_variable_get(:@cpu_limit).to_s + ('m'), pod_usage_metrics.instance_variable_get(:@cpu_request).to_s + ('m'),
426
429
  pod_usage_metrics.instance_variable_get(:@memory_limit).to_s + ('Mi'), pod_usage_metrics.instance_variable_get(:@memory_request).to_s + ('Mi'))
@@ -454,7 +457,7 @@ module Fluent
454
457
  begin
455
458
  @client.discover unless @client.discovered
456
459
  @client.rest_client['/nodes'].tap do |endpoint|
457
- log.info("Use URL #{endpoint.url} for scraping node metrics")
460
+ log.debug("Use URL #{endpoint.url} for scraping node metrics")
458
461
  end
459
462
  end
460
463
  end
@@ -462,6 +465,9 @@ module Fluent
462
465
  def scrape_node_metrics
463
466
  response = node_api.get(@client.headers)
464
467
  handle_node_response(response)
468
+ rescue StandardError => e
469
+ log.error "Failed to get node metrics, error=#{$ERROR_INFO}, #{e.inspect}"
470
+ log.error_backtrace
465
471
  end
466
472
 
467
473
  # This method is used to handle responses from the kubeapiserver api
@@ -474,7 +480,7 @@ module Fluent
474
480
  log.error "ExMultiJson.load(response.body) expected 2xx from summary API, but got #{response.code}. Response body = #{response.body}"
475
481
  end
476
482
  rescue StandardError => e
477
- log.error "Failed to scrape metrics, error=#{$ERROR_INFO}, #{e.inspect}"
483
+ log.error "Failed to scrape node metrics, error=#{$ERROR_INFO}, #{e.inspect}"
478
484
  log.error_backtrace
479
485
  end
480
486
 
@@ -527,7 +533,7 @@ module Fluent
527
533
  begin
528
534
  @client.discover unless @client.discovered
529
535
  @client.rest_client['/nodes'].tap do |endpoint|
530
- log.info("Use URL #{endpoint.url} for scraping node metrics")
536
+ log.debug("Use URL #{endpoint.url} for scraping node metrics")
531
537
  end
532
538
  end
533
539
  end
@@ -535,6 +541,9 @@ module Fluent
535
541
  def scrape_resource_usage_metrics
536
542
  response = resource_usage_api.get(@client.headers)
537
543
  handle_resource_usage_response(response)
544
+ rescue StandardError => e
545
+ log.error "Failed to get resource usage metrics, error=#{$ERROR_INFO}, #{e.inspect}"
546
+ log.error_backtrace
538
547
  end
539
548
 
540
549
  # This method is used to handle responses from the kubelet summary api
@@ -547,7 +556,7 @@ module Fluent
547
556
  log.error "ExMultiJson.load(response.body) expected 2xx from summary API, but got #{response.code}. Response body = #{response.body}"
548
557
  end
549
558
  rescue StandardError => e
550
- log.error "Failed to scrape metrics, error=#{$ERROR_INFO}, #{e.inspect}"
559
+ log.error "Failed to scrape resource usage metrics, error=#{$ERROR_INFO}, #{e.inspect}"
551
560
  log.error_backtrace
552
561
  end
553
562
 
@@ -560,29 +569,31 @@ module Fluent
560
569
  begin
561
570
  @client.discover unless @client.discovered
562
571
  @client.rest_client["/nodes/#{node_name}:#{@kubelet_port}/proxy/stats/summary"].tap do |endpoint|
563
- log.info("Use URL #{endpoint.url} for scraping resource usage metrics")
572
+ log.debug("Use URL #{endpoint.url} for scraping resource usage metrics")
564
573
  end
565
574
  end
566
575
 
567
576
  node_response = JSON.parse(node_rest_client.get(@client.headers))
568
577
  Array(node_response['pods']).each do |pod_json|
569
- pod_cpu_usage = pod_json['cpu']['usageNanoCores']
570
- pod_memory_usage = pod_json['memory']['usageBytes']
571
- pod_namespace = pod_json['podRef']['namespace']
572
- pod_usage = ResourceUsageMetricsUnit.new
573
- pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
574
- if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
575
- namespace_usage_metrics = ResourceUsageMetricsUnit.new
576
- @@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
577
- else
578
- @@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
579
- end
580
- if @@node_resource_usage_metrics_map[node_name].nil?
581
- node_name_usage_metrics = ResourceUsageMetricsUnit.new
582
- @@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
578
+ unless pod_json['cpu'].nil? || pod_json['memory'].nil?
579
+ pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
580
+ pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0)
581
+ pod_namespace = pod_json['podRef']['namespace']
582
+ pod_usage = ResourceUsageMetricsUnit.new
583
+ pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
584
+ if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
585
+ namespace_usage_metrics = ResourceUsageMetricsUnit.new
586
+ @@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
587
+ else
588
+ @@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
589
+ end
590
+ if @@node_resource_usage_metrics_map[node_name].nil?
591
+ node_name_usage_metrics = ResourceUsageMetricsUnit.new
592
+ @@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
593
+ end
594
+ @@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
595
+ pod_usage = nil
583
596
  end
584
- @@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
585
- pod_usage = nil
586
597
  end
587
598
  end
588
599
  end