fluent-plugin-k8s-metrics-agg 1.1.7 → 1.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +2 -2
  3. data/VERSION +1 -1
  4. data/docker/Dockerfile +1 -1
  5. data/docker/Gemfile.lock +11 -11
  6. data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb +61 -45
  7. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 11e9b540230a927f8cb2dd469d4b16d16941c8e7cde2d4019394509c8562874e
4
- data.tar.gz: 38e231e8cde3aa56f386c3cc835065fec257ecffe502866ae0db8e00a818f970
3
+ metadata.gz: 98a2dd49ad2df50f48934daa1f41f92d9466a0c87149b74a059755e2a12b55a1
4
+ data.tar.gz: 4549d0845d04a933c89a6599a18b15ef83e468c0e739c5b9f66458b0c5f8116f
5
5
  SHA512:
6
- metadata.gz: 298e621affd96c9185a95e019af9bf947d4f1a81bdb850606fc9c9729ba0f7ec3072ad4df4f176e4280ae70bba1a012d59d183ecd858cc80f1c085c4b99ec44b
7
- data.tar.gz: 3ea6fe8cd14f6820a8c3126cc8fc226b28d06786e467fdba6cbdc8d8b1445724a11406a4d28850f87685959ac59170be0e45fab8cee35570476167fb96f3f01e
6
+ metadata.gz: c1acc729827a62b91a8cf8fa33f230d651eaf733e66d395ba4610ff2b55407013e9c3fdb8b4db6d8913a4e6320fbe37f04b910b62de3c894216c347bacb86be5
7
+ data.tar.gz: 79c4d760559ef7edb0789133aa7d9c7e1a89cfd28b3ac71df146dd755082338b1c480a9b95940fc15f54b1984e22cf81f72e6c48f4b7e8c93354a1aaed9985b6
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-k8s-metrics-agg (1.1.7)
4
+ fluent-plugin-k8s-metrics-agg (1.1.8)
5
5
  fluentd (>= 1.9.1)
6
6
  kubeclient (~> 4.6.0)
7
7
  multi_json (~> 1.14.1)
@@ -107,4 +107,4 @@ DEPENDENCIES
107
107
  webmock (~> 3.5.1)
108
108
 
109
109
  BUNDLED WITH
110
- 2.2.30
110
+ 2.2.33
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.1.7
1
+ 1.1.8
data/docker/Dockerfile CHANGED
@@ -37,7 +37,7 @@ RUN yum update -y \
37
37
  && gem install bundler \
38
38
  && gem unpack /tmp/*.gem --target gem \
39
39
  && bundle install \
40
- && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-*
40
+ && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true
41
41
 
42
42
  RUN groupadd -r $FLUENT_USER && \
43
43
  useradd -r -g $FLUENT_USER $FLUENT_USER && \
data/docker/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: gem
3
3
  specs:
4
- fluent-plugin-k8s-metrics-agg (1.1.7)
4
+ fluent-plugin-k8s-metrics-agg (1.1.8)
5
5
  fluentd (>= 1.9.1)
6
6
  kubeclient (~> 4.6.0)
7
7
  multi_json (~> 1.14.1)
@@ -10,9 +10,9 @@ PATH
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- activemodel (6.1.4.1)
14
- activesupport (= 6.1.4.1)
15
- activesupport (6.1.4.1)
13
+ activemodel (6.1.4.3)
14
+ activesupport (= 6.1.4.3)
15
+ activesupport (6.1.4.3)
16
16
  concurrent-ruby (~> 1.0, >= 1.0.2)
17
17
  i18n (>= 1.6, < 2)
18
18
  minitest (>= 5.1)
@@ -33,7 +33,7 @@ GEM
33
33
  ffi-compiler (1.0.1)
34
34
  ffi (>= 1.0.0)
35
35
  rake
36
- fluent-plugin-kubernetes_metadata_filter (2.9.2)
36
+ fluent-plugin-kubernetes_metadata_filter (2.9.3)
37
37
  fluentd (>= 0.14.0, < 1.15)
38
38
  kubeclient (>= 4.0.0, < 5.0.0)
39
39
  lru_redux
@@ -42,16 +42,16 @@ GEM
42
42
  prometheus-client (>= 2.1.0)
43
43
  fluent-plugin-record-modifier (2.1.0)
44
44
  fluentd (>= 1.0, < 2)
45
- fluent-plugin-splunk-hec (1.2.7)
45
+ fluent-plugin-splunk-hec (1.2.9)
46
46
  fluentd (>= 1.4)
47
47
  multi_json (~> 1.13)
48
48
  net-http-persistent (~> 3.1)
49
49
  openid_connect (~> 1.1.8)
50
50
  prometheus-client (>= 2.1.0)
51
- fluentd (1.14.2)
51
+ fluentd (1.14.3)
52
52
  bundler
53
53
  cool.io (>= 1.4.5, < 2.0.0)
54
- http_parser.rb (>= 0.5.1, < 0.8.0)
54
+ http_parser.rb (>= 0.5.1, < 0.9.0)
55
55
  msgpack (>= 1.3.1, < 2.0.0)
56
56
  serverengine (>= 2.2.2, < 3.0.0)
57
57
  sigdump (~> 0.2.2)
@@ -86,11 +86,11 @@ GEM
86
86
  lru_redux (1.1.0)
87
87
  mail (2.7.1)
88
88
  mini_mime (>= 0.1.1)
89
- mime-types (3.3.1)
89
+ mime-types (3.4.1)
90
90
  mime-types-data (~> 3.2015)
91
- mime-types-data (3.2021.0901)
91
+ mime-types-data (3.2021.1115)
92
92
  mini_mime (1.1.2)
93
- minitest (5.14.4)
93
+ minitest (5.15.0)
94
94
  msgpack (1.4.2)
95
95
  multi_json (1.14.1)
96
96
  net-http-persistent (3.1.0)
data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb CHANGED
@@ -51,11 +51,11 @@ module Fluent
51
51
  # m cpu is assumed standard
52
52
  @cpu_mult = 1
53
53
  @cpu_mult = 1000 if cpu[-1] != 'm'
54
- cpu.delete('^0-9').to_i
54
+ cpu.delete('^0-9.').to_i
55
55
  end
56
56
 
57
57
  def get_cpu_or_memory_value(resource)
58
- resource = resource.tr('^0-9', '').to_i
58
+ resource = resource.tr('^0-9.', '').to_i
59
59
  resource
60
60
  end
61
61
 
@@ -66,29 +66,35 @@ module Fluent
66
66
  end
67
67
 
68
68
  # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory
69
+ # 1 Ki = 1024 bytes
70
+ # 1 K = 1000 bytes = 1000/1024 Ki = 1000/1024*1024 Mi
69
71
  def get_memory_mult(memory)
70
- memory_mult = if memory[-2] == 'Ki'
71
- 0.001
72
- elsif memory[-2] == 'K'
73
- 1.0 / 1024
74
- elsif memory[-2] == 'Mi'
72
+ memory_mult = if memory[-2..] == 'Ki'
73
+ 1.0/1024
74
+ elsif memory[-1] == 'K'
75
+ 1e3 / 1024 ** 2
76
+ elsif memory[-2..] == 'Mi'
75
77
  1
76
- elsif memory[-2] == 'M'
77
- 1
78
- elsif memory[-2] == 'Gi'
79
- 1000
80
- elsif memory[-2] == 'G'
78
+ elsif memory[-1] == 'M'
79
+ 1e6 / 1024 ** 2
80
+ elsif memory[-2..] == 'Gi'
81
81
  1024
82
- elsif memory[-2] == 'Ti'
83
- 1_000_000
84
- elsif memory[-2] == 'T'
85
- 1_048_576
86
- elsif memory[-2] == 'Ei'
87
- 1_000_000_000
82
+ elsif memory[-1] == 'G'
83
+ 1e9 / 1024 ** 2
84
+ elsif memory[-2..] == 'Ti'
85
+ 1024 ** 2
86
+ elsif memory[-1] == 'T'
87
+ 1e12 / 1024 ** 2
88
+ elsif memory[-2..] == 'Pi'
89
+ 1024**3
90
+ elsif memory[-2] == 'P'
91
+ 1e15 / 1024 ** 2
92
+ elsif memory[-2..] == 'Ei'
93
+ 1024**4
88
94
  elsif memory[-2] == 'E'
89
- 1_073_741_824
95
+ 1e18 / 1024 ** 2
90
96
  else
91
- 0.000001
97
+ 1 / 1024 ** 2
92
98
  end
93
99
  memory_mult
94
100
  end
@@ -278,43 +284,47 @@ module Fluent
278
284
  end
279
285
 
280
286
  def get_cpu_value(resource)
281
- cpu_val = resource.tr('^0-9', '').to_i
287
+ cpu_val = resource.tr('^0-9.', '').to_i
282
288
  mult = get_cpu_mult(resource)
283
- cpu_val += cpu_val * mult
289
+ cpu_val = cpu_val * mult
284
290
  cpu_val
285
291
  end
286
292
 
287
293
  def get_memory_mult(memory)
288
- memory_mult = if memory[-2] == 'Ki'
289
- 0.001
290
- elsif memory[-2] == 'K'
291
- 1.0 / 1024
292
- elsif memory[-2] == 'Mi'
293
- 1
294
- elsif memory[-2] == 'M'
294
+ memory_mult = if memory[-2..] == 'Ki'
295
+ 1.0/1024
296
+ elsif memory[-1] == 'K'
297
+ 1e3 / 1024 ** 2
298
+ elsif memory[-2..] == 'Mi'
295
299
  1
296
- elsif memory[-2] == 'Gi'
297
- 1000
298
- elsif memory[-2] == 'G'
300
+ elsif memory[-1] == 'M'
301
+ 1e6 / 1024 ** 2
302
+ elsif memory[-2..] == 'Gi'
299
303
  1024
300
- elsif memory[-2] == 'Ti'
301
- 1_000_000
302
- elsif memory[-2] == 'T'
303
- 1_048_576 # 1024*1024
304
- elsif memory[-2] == 'Ei'
305
- 1_000_000_000
304
+ elsif memory[-1] == 'G'
305
+ 1e9 / 1024 ** 2
306
+ elsif memory[-2..] == 'Ti'
307
+ 1024 ** 2
308
+ elsif memory[-1] == 'T'
309
+ 1e12 / 1024 ** 2
310
+ elsif memory[-2..] == 'Pi'
311
+ 1024**3
312
+ elsif memory[-2] == 'P'
313
+ 1e15 / 1024 ** 2
314
+ elsif memory[-2..] == 'Ei'
315
+ 1024**4
306
316
  elsif memory[-2] == 'E'
307
- 1_073_741_824 # 1024*1024*1024
317
+ 1e18 / 1024 ** 2
308
318
  else
309
- 0.000001
319
+ 1 / 1024 ** 2
310
320
  end
311
321
  memory_mult
312
322
  end
313
323
 
314
324
  def get_memory_value(resource)
315
- mem_val = resource.tr('^0-9', '').to_i
325
+ mem_val = resource.tr('^0-9.', '').to_i
316
326
  mult = get_memory_mult(resource)
317
- mem_val += mem_val * mult
327
+ mem_val = mem_val * mult
318
328
  mem_val
319
329
  end
320
330
 
@@ -513,7 +523,7 @@ module Fluent
513
523
  router.emit generate_tag('node') << ('.cpu.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_utilization
514
524
  node_cpu_reservation = node_req_lim.instance_variable_get(:@cpu_request).to_f / node_cpu_allocatable
515
525
  router.emit generate_tag('node') << ('.cpu.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_reservation
516
- node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / 1_000_000 * node_memory_allocatable # converting from bytes to megabytes
526
+ node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / node_memory_allocatable # converting from bytes to megabytes
517
527
  router.emit generate_tag('node') << ('.memory.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_utilization
518
528
  node_memory_reservation = node_req_lim.instance_variable_get(:@memory_request).to_f / node_memory_allocatable
519
529
  router.emit generate_tag('node') << ('.memory.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_reservation
@@ -573,11 +583,17 @@ module Fluent
573
583
  end
574
584
  end
575
585
 
576
- node_response = JSON.parse(node_rest_client.get(@client.headers))
586
+ begin
587
+ node_response = JSON.parse(node_rest_client.get(@client.headers))
588
+ rescue RestClient::ServiceUnavailable
589
+ log.warn("Couldn't scrap metric for node '#{node_name} as it is unavailable. Ignoring it.'")
590
+ next
591
+ end
592
+
577
593
  Array(node_response['pods']).each do |pod_json|
578
594
  unless pod_json['cpu'].nil? || pod_json['memory'].nil?
579
595
  pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
580
- pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0)
596
+ pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0) / 1024 ** 2 # Converting to Mi
581
597
  pod_namespace = pod_json['podRef']['namespace']
582
598
  pod_usage = ResourceUsageMetricsUnit.new
583
599
  pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-k8s-metrics-agg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.7
4
+ version: 1.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Splunk Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-11-03 00:00:00.000000000 Z
11
+ date: 2021-12-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
201
201
  - !ruby/object:Gem::Version
202
202
  version: '0'
203
203
  requirements: []
204
- rubygems_version: 3.0.1
204
+ rubygems_version: 3.1.4
205
205
  signing_key:
206
206
  specification_version: 4
207
207
  summary: A fluentd input plugin that collects kubernetes cluster metrics.