fluent-plugin-k8s-metrics-agg 1.1.8 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +7 -7
- data/VERSION +1 -1
- data/docker/Gemfile.lock +1 -1
- data/lib/fluent/plugin/in_kubernetes_metrics_aggregator.rb +78 -45
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 276b6cf483191eac629b4d605d22296689f26e03d7fd02bf0ba3366b91804242
|
|
4
|
+
data.tar.gz: e36406b6036c87edac3fa5cf12884a7837e68bf1e04043d42689e18c8e347880
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2144bb3bcc031291595986318217b8c52567106290893f7b47bef16df9e10a8a9bd09242791092cfbc3313f610f406de3c65af38b0e7ac295fc3db52d6c077bd
|
|
7
|
+
data.tar.gz: bde4c5f3b7a86257460773d6dc61b7013c8cdadf7626f9e8269c109d62dbb3f4f52050dec3af61bb04f54923938847bf402cfa93fb8844e80016755bd8bc2ef8
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-k8s-metrics-agg (1.1.
|
|
4
|
+
fluent-plugin-k8s-metrics-agg (1.1.9)
|
|
5
5
|
fluentd (>= 1.9.1)
|
|
6
6
|
kubeclient (~> 4.6.0)
|
|
7
7
|
multi_json (~> 1.14.1)
|
|
@@ -23,10 +23,10 @@ GEM
|
|
|
23
23
|
ffi-compiler (1.0.1)
|
|
24
24
|
ffi (>= 1.0.0)
|
|
25
25
|
rake
|
|
26
|
-
fluentd (1.14.
|
|
26
|
+
fluentd (1.14.4)
|
|
27
27
|
bundler
|
|
28
28
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
29
|
-
http_parser.rb (>= 0.5.1, < 0.
|
|
29
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
30
30
|
msgpack (>= 1.3.1, < 2.0.0)
|
|
31
31
|
serverengine (>= 2.2.2, < 3.0.0)
|
|
32
32
|
sigdump (~> 0.2.2)
|
|
@@ -47,15 +47,15 @@ GEM
|
|
|
47
47
|
http-form_data (2.3.0)
|
|
48
48
|
http-parser (1.2.3)
|
|
49
49
|
ffi-compiler (>= 1.0, < 2.0)
|
|
50
|
-
http_parser.rb (0.
|
|
50
|
+
http_parser.rb (0.8.0)
|
|
51
51
|
json (2.6.1)
|
|
52
52
|
kubeclient (4.6.0)
|
|
53
53
|
http (>= 3.0, < 5.0)
|
|
54
54
|
recursive-open-struct (~> 1.0, >= 1.0.4)
|
|
55
55
|
rest-client (~> 2.0)
|
|
56
|
-
mime-types (3.
|
|
56
|
+
mime-types (3.4.1)
|
|
57
57
|
mime-types-data (~> 3.2015)
|
|
58
|
-
mime-types-data (3.
|
|
58
|
+
mime-types-data (3.2022.0105)
|
|
59
59
|
msgpack (1.4.2)
|
|
60
60
|
multi_json (1.14.1)
|
|
61
61
|
netrc (0.11.0)
|
|
@@ -107,4 +107,4 @@ DEPENDENCIES
|
|
|
107
107
|
webmock (~> 3.5.1)
|
|
108
108
|
|
|
109
109
|
BUNDLED WITH
|
|
110
|
-
2.
|
|
110
|
+
2.3.5
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.1.
|
|
1
|
+
1.1.9
|
data/docker/Gemfile.lock
CHANGED
|
@@ -70,7 +70,7 @@ module Fluent
|
|
|
70
70
|
# 1 K = 1000 bytes = 1000/1024 Ki = 1000/1024*1024 Mi
|
|
71
71
|
def get_memory_mult(memory)
|
|
72
72
|
memory_mult = if memory[-2..] == 'Ki'
|
|
73
|
-
1.0/1024
|
|
73
|
+
1.0 / 1024
|
|
74
74
|
elsif memory[-1] == 'K'
|
|
75
75
|
1e3 / 1024 ** 2
|
|
76
76
|
elsif memory[-2..] == 'Mi'
|
|
@@ -94,7 +94,7 @@ module Fluent
|
|
|
94
94
|
elsif memory[-2] == 'E'
|
|
95
95
|
1e18 / 1024 ** 2
|
|
96
96
|
else
|
|
97
|
-
1 / 1024 ** 2
|
|
97
|
+
1.0 / 1024 ** 2
|
|
98
98
|
end
|
|
99
99
|
memory_mult
|
|
100
100
|
end
|
|
@@ -292,7 +292,7 @@ module Fluent
|
|
|
292
292
|
|
|
293
293
|
def get_memory_mult(memory)
|
|
294
294
|
memory_mult = if memory[-2..] == 'Ki'
|
|
295
|
-
1.0/1024
|
|
295
|
+
1.0 / 1024
|
|
296
296
|
elsif memory[-1] == 'K'
|
|
297
297
|
1e3 / 1024 ** 2
|
|
298
298
|
elsif memory[-2..] == 'Mi'
|
|
@@ -316,7 +316,7 @@ module Fluent
|
|
|
316
316
|
elsif memory[-2] == 'E'
|
|
317
317
|
1e18 / 1024 ** 2
|
|
318
318
|
else
|
|
319
|
-
1 / 1024 ** 2
|
|
319
|
+
1.0 / 1024 ** 2
|
|
320
320
|
end
|
|
321
321
|
memory_mult
|
|
322
322
|
end
|
|
@@ -428,7 +428,7 @@ module Fluent
|
|
|
428
428
|
end
|
|
429
429
|
end
|
|
430
430
|
container_usage_metrics.add_usage_metrics(cpu_limit, cpu_request, memory_limit, memory_request)
|
|
431
|
-
container_labels = { 'name' => container_json['name'], 'image' => container_json['image'], 'node' => pod_json['spec']['nodeName'] }
|
|
431
|
+
container_labels = { 'pod-name' => pod_json['metadata']['name'], 'namespace' => pod_json['metadata']['namespace'], 'name' => container_json['name'], 'image' => container_json['image'], 'node' => pod_json['spec']['nodeName'] }
|
|
432
432
|
emit_limits_requests_metrics(generate_tag('container'), @scraped_at, container_labels, container_usage_metrics)
|
|
433
433
|
pod_usage_metrics.add_usage_metrics(cpu_limit, cpu_request, memory_limit, memory_request)
|
|
434
434
|
end
|
|
@@ -495,46 +495,53 @@ module Fluent
|
|
|
495
495
|
end
|
|
496
496
|
|
|
497
497
|
def process_node_response(response)
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
498
|
+
@mutex_node_req_lim.synchronize do
|
|
499
|
+
Array(response['items']).each do |node_json|
|
|
500
|
+
node_name = node_json['metadata']['name']
|
|
501
|
+
node_cpu_capacity = get_cpu_value(node_json['status']['capacity']['cpu'])
|
|
502
|
+
router.emit generate_tag('node') << ('.cpu.capacity'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_capacity
|
|
503
|
+
node_cpu_allocatable = get_cpu_value(node_json['status']['allocatable']['cpu'])
|
|
504
|
+
router.emit generate_tag('node') << ('.cpu.allocatable'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_allocatable
|
|
505
|
+
node_memory_capacity = get_memory_value(node_json['status']['capacity']['memory'])
|
|
506
|
+
router.emit generate_tag('node') << ('.memory.capacity'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_capacity
|
|
507
|
+
node_memory_allocatable = get_memory_value(node_json['status']['allocatable']['memory'])
|
|
508
|
+
router.emit generate_tag('node') << ('.memory.allocatable'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_allocatable
|
|
509
|
+
|
|
510
|
+
node_req_lim = UsageMetricsUnit.new
|
|
511
|
+
if @@node_requests_limits_metrics_map[node_name].nil?
|
|
512
|
+
next
|
|
513
|
+
end
|
|
514
514
|
node_req_lim = @@node_requests_limits_metrics_map[node_name]
|
|
515
|
+
|
|
516
|
+
node_cpu_reservation = node_req_lim.instance_variable_get(:@cpu_request).to_f / node_cpu_allocatable
|
|
517
|
+
router.emit generate_tag('node') << ('.cpu.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_reservation
|
|
518
|
+
node_memory_reservation = node_req_lim.instance_variable_get(:@memory_request).to_f / node_memory_allocatable
|
|
519
|
+
router.emit generate_tag('node') << ('.memory.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_reservation
|
|
515
520
|
end
|
|
516
|
-
|
|
517
|
-
|
|
521
|
+
@@node_requests_limits_metrics_map = nil
|
|
522
|
+
@@node_requests_limits_metrics_map = {}
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
@mutex_node_res_usage.synchronize do
|
|
526
|
+
Array(response['items']).each do |node_json|
|
|
527
|
+
node_name = node_json['metadata']['name']
|
|
528
|
+
node_cpu_allocatable = get_cpu_value(node_json['status']['allocatable']['cpu'])
|
|
529
|
+
node_memory_allocatable = get_memory_value(node_json['status']['allocatable']['memory'])
|
|
518
530
|
|
|
531
|
+
node_res_usage = ResourceUsageMetricsUnit.new
|
|
532
|
+
if @@node_resource_usage_metrics_map[node_name].nil?
|
|
533
|
+
next
|
|
534
|
+
end
|
|
519
535
|
node_res_usage = @@node_resource_usage_metrics_map[node_name]
|
|
536
|
+
|
|
537
|
+
# https://github.com/kubernetes/heapster/blob/c78cc312ab3901acfe5c2f95f7a621909c8455ad/metrics/processors/node_autoscaling_enricher.go#L62
|
|
538
|
+
node_cpu_utilization = node_res_usage.instance_variable_get(:@cpu_usage).to_f / 1_000_000 * node_cpu_allocatable # converting from nano cores to milli core
|
|
539
|
+
router.emit generate_tag('node') << ('.cpu.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_utilization
|
|
540
|
+
node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / node_memory_allocatable # converting from bytes to megabytes
|
|
541
|
+
router.emit generate_tag('node') << ('.memory.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_utilization
|
|
520
542
|
end
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
router.emit generate_tag('node') << ('.cpu.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_utilization
|
|
524
|
-
node_cpu_reservation = node_req_lim.instance_variable_get(:@cpu_request).to_f / node_cpu_allocatable
|
|
525
|
-
router.emit generate_tag('node') << ('.cpu.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_cpu_reservation
|
|
526
|
-
node_memory_utilization = node_res_usage.instance_variable_get(:@memory_usage).to_f / node_memory_allocatable # converting from bytes to megabytes
|
|
527
|
-
router.emit generate_tag('node') << ('.memory.utilization'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_utilization
|
|
528
|
-
node_memory_reservation = node_req_lim.instance_variable_get(:@memory_request).to_f / node_memory_allocatable
|
|
529
|
-
router.emit generate_tag('node') << ('.memory.reservation'), Fluent::EventTime.from_time(@scraped_node_at), 'node' => node_name, 'value' => node_memory_reservation
|
|
530
|
-
@mutex_node_req_lim.synchronize do
|
|
531
|
-
@@node_requests_limits_metrics_map = nil
|
|
532
|
-
@@node_requests_limits_metrics_map = {}
|
|
533
|
-
end
|
|
534
|
-
@mutex_node_res_usage.synchronize do
|
|
535
|
-
@@node_resource_usage_metrics_map = nil
|
|
536
|
-
@@node_resource_usage_metrics_map = {}
|
|
537
|
-
end
|
|
543
|
+
@@node_resource_usage_metrics_map = nil
|
|
544
|
+
@@node_resource_usage_metrics_map = {}
|
|
538
545
|
end
|
|
539
546
|
end
|
|
540
547
|
|
|
@@ -551,7 +558,7 @@ module Fluent
|
|
|
551
558
|
def scrape_resource_usage_metrics
|
|
552
559
|
response = resource_usage_api.get(@client.headers)
|
|
553
560
|
handle_resource_usage_response(response)
|
|
554
|
-
|
|
561
|
+
rescue StandardError => e
|
|
555
562
|
log.error "Failed to get resource usage metrics, error=#{$ERROR_INFO}, #{e.inspect}"
|
|
556
563
|
log.error_backtrace
|
|
557
564
|
end
|
|
@@ -589,16 +596,15 @@ module Fluent
|
|
|
589
596
|
log.warn("Couldn't scrap metric for node '#{node_name} as it is unavailable. Ignoring it.'")
|
|
590
597
|
next
|
|
591
598
|
end
|
|
592
|
-
|
|
599
|
+
|
|
593
600
|
Array(node_response['pods']).each do |pod_json|
|
|
594
601
|
unless pod_json['cpu'].nil? || pod_json['memory'].nil?
|
|
595
|
-
pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000
|
|
596
|
-
pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0) / 1024 ** 2 # Converting to Mi
|
|
602
|
+
pod_cpu_usage = pod_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000.to_f
|
|
603
|
+
pod_memory_usage = pod_json['memory'].fetch('usageBytes', 0) / 1024 ** 2.to_f # Converting to Mi
|
|
597
604
|
pod_namespace = pod_json['podRef']['namespace']
|
|
598
605
|
pod_usage = ResourceUsageMetricsUnit.new
|
|
599
606
|
pod_usage.add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
|
600
607
|
if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
|
|
601
|
-
namespace_usage_metrics = ResourceUsageMetricsUnit.new
|
|
602
608
|
@@namespace_resource_usage_metrics_map[pod_namespace] = pod_usage
|
|
603
609
|
else
|
|
604
610
|
@@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
|
@@ -610,6 +616,33 @@ module Fluent
|
|
|
610
616
|
@@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(pod_cpu_usage, pod_memory_usage)
|
|
611
617
|
pod_usage = nil
|
|
612
618
|
end
|
|
619
|
+
|
|
620
|
+
if pod_json['cpu'].nil? || pod_json['memory'].nil?
|
|
621
|
+
unless pod_json['containers'].nil?
|
|
622
|
+
pod_namespace = pod_json['podRef']['namespace']
|
|
623
|
+
Array(pod_json['containers']).each do |container_json|
|
|
624
|
+
unless container_json['cpu'].nil? || container_json['memory'].nil?
|
|
625
|
+
container_cpu_usage = container_json['cpu'].fetch('usageNanoCores', 0)/ 1_000_000.to_f
|
|
626
|
+
container_memory_usage = container_json['memory'].fetch('usageBytes', 0) / 1024 ** 2.to_f # Converting to Mi
|
|
627
|
+
container_usage = ResourceUsageMetricsUnit.new
|
|
628
|
+
container_usage.add_resource_usage_metrics(container_cpu_usage, container_memory_usage)
|
|
629
|
+
if @@namespace_resource_usage_metrics_map[pod_namespace].nil?
|
|
630
|
+
namespace_usage_metrics = ResourceUsageMetricsUnit.new
|
|
631
|
+
@@namespace_resource_usage_metrics_map[pod_namespace] = container_usage
|
|
632
|
+
else
|
|
633
|
+
@@namespace_resource_usage_metrics_map[pod_namespace].add_resource_usage_metrics(container_cpu_usage, container_memory_usage)
|
|
634
|
+
end
|
|
635
|
+
if @@node_resource_usage_metrics_map[node_name].nil?
|
|
636
|
+
node_name_usage_metrics = ResourceUsageMetricsUnit.new
|
|
637
|
+
@@node_resource_usage_metrics_map[node_name] = node_name_usage_metrics
|
|
638
|
+
else
|
|
639
|
+
@@node_resource_usage_metrics_map[node_name].add_resource_usage_metrics(container_cpu_usage, container_memory_usage)
|
|
640
|
+
end
|
|
641
|
+
container_usage = nil
|
|
642
|
+
end
|
|
643
|
+
end
|
|
644
|
+
end
|
|
645
|
+
end
|
|
613
646
|
end
|
|
614
647
|
end
|
|
615
648
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-k8s-metrics-agg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Splunk Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-01-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|