fluent-plugin-oci-logging-analytics 2.0.2 → 2.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fluent/metrics/metricsLabels.rb +13 -0
- data/lib/fluent/metrics/prometheusMetrics.rb +41 -0
- data/lib/fluent/plugin/out_oci-logging-analytics.rb +265 -119
- metadata +20 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6a06f05f13c8058d4e2a6f3a6d4b8cf065f3c558d27f9fecf43ecbabdbb2f1f1
|
4
|
+
data.tar.gz: 076c4ea4b27fbf7230c237596162ca75d7e17450112f67aee848ffea52828123
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 17a4f913bdb5c7ec6bf9bf8f812efb8f1da9f2e55180c25806a49d5782ffcfac21080eda04a036a89189aa8d33d95bf8947ec34ac9a7d508fd190077efa6bd09
|
7
|
+
data.tar.gz: 6c7dbb1934d8850527fefd7163e993034cf681b239d08041cc9b0085439b6e8b92ba9b4142ec03e5653a3e0d63fc2711b75b0fac9ef78d60d0694f5a0719abd6
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class MetricsLabels
|
2
|
+
attr_accessor :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
|
3
|
+
def initialize
|
4
|
+
@tag = nil
|
5
|
+
@logGroupId = nil
|
6
|
+
@logSourceName = nil
|
7
|
+
@logSet = nil
|
8
|
+
@invalid_reason = nil
|
9
|
+
@records_valid = 0
|
10
|
+
@records_per_tag = 0
|
11
|
+
@latency = 0
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'prometheus/client'
|
2
|
+
require 'prometheus/client/registry'
|
3
|
+
require 'prometheus/client/gauge'
|
4
|
+
require 'prometheus/client/histogram'
|
5
|
+
require 'singleton'
|
6
|
+
|
7
|
+
class PrometheusMetrics
|
8
|
+
include Singleton
|
9
|
+
attr_accessor :records_received, :records_valid, :records_invalid, :records_error, :records_posted,
|
10
|
+
:bytes_received, :bytes_posted, :chunk_time_to_receive, :chunk_time_to_upload
|
11
|
+
def initialize
|
12
|
+
createMetrics
|
13
|
+
registerMetrics
|
14
|
+
end
|
15
|
+
def createMetrics
|
16
|
+
gauge = Prometheus::Client::Gauge
|
17
|
+
@records_received = gauge.new(:oci_la_fluentd_output_plugin_records_received, docstring: 'Number of records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
|
18
|
+
@records_valid = gauge.new(:oci_la_fluentd_output_plugin_records_valid, docstring: 'Number of valid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
|
19
|
+
@records_invalid = gauge.new(:oci_la_fluentd_output_plugin_records_invalid, docstring: 'Number of invalid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:reason])
|
20
|
+
@records_error = gauge.new(:oci_la_fluentd_output_plugin_records_post_error, docstring: 'Number of records failed posting to OCI Logging Analytics by the Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:error_code, :reason])
|
21
|
+
@records_posted = gauge.new(:oci_la_fluentd_output_plugin_records_post_success, docstring: 'Number of records posted by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
|
22
|
+
#@bytes_received = gauge.new(:oci_la_bytes_received, docstring: '...', labels: [:tag])
|
23
|
+
#@bytes_posted = gauge.new(:oci_la_bytes_posted, docstring: '...', labels: [:oci_la_log_group_id])
|
24
|
+
histogram = Prometheus::Client::Histogram
|
25
|
+
@chunk_time_to_receive = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_receive, docstring: 'Average time taken by Fluentd to deliver the collected records from Input plugin to OCI Logging Analytics output plugin.', labels: [:tag])
|
26
|
+
@chunk_time_to_upload = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_post, docstring: 'Average time taken for posting the received records to OCI Logging Analytics by the Fluentd output plugin.', labels: [:oci_la_log_group_id])
|
27
|
+
end
|
28
|
+
|
29
|
+
def registerMetrics
|
30
|
+
registry = Prometheus::Client.registry
|
31
|
+
registry.register(@records_received) unless registry.exist?('oci_la_fluentd_output_plugin_records_received')
|
32
|
+
registry.register(@records_valid) unless registry.exist?('oci_la_fluentd_output_plugin_records_valid')
|
33
|
+
registry.register(@records_invalid) unless registry.exist?('oci_la_fluentd_output_plugin_records_invalid')
|
34
|
+
registry.register(@records_error) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_error')
|
35
|
+
registry.register(@records_posted) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_success')
|
36
|
+
#registry.register(@bytes_received) unless registry.exist?('oci_la_bytes_received')
|
37
|
+
#registry.register(@bytes_posted) unless registry.exist?('oci_la_bytes_valid')
|
38
|
+
registry.register(@chunk_time_to_receive) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_receive')
|
39
|
+
registry.register(@chunk_time_to_upload) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_post')
|
40
|
+
end
|
41
|
+
end
|
@@ -8,6 +8,8 @@ require 'zip'
|
|
8
8
|
require 'logger'
|
9
9
|
require_relative '../dto/logEventsJson'
|
10
10
|
require_relative '../dto/logEvents'
|
11
|
+
require_relative '../metrics/prometheusMetrics'
|
12
|
+
require_relative '../metrics/metricsLabels'
|
11
13
|
|
12
14
|
# Import only specific OCI modules to improve load times and reduce the memory requirements.
|
13
15
|
require 'oci/auth/auth'
|
@@ -53,8 +55,25 @@ module Fluent::Plugin
|
|
53
55
|
helpers :thread, :event_emitter
|
54
56
|
|
55
57
|
MAX_FILES_PER_ZIP = 100
|
58
|
+
METRICS_INVALID_REASON_MESSAGE = "MISSING_FIELD_MESSAGE"
|
59
|
+
METRICS_INVALID_REASON_LOG_GROUP_ID = "MISSING_OCI_LA_LOG_GROUP_ID_FIELD"
|
60
|
+
METRICS_INVALID_REASON_LOG_SOURCE_NAME = "MISSING_OCI_LA_LOG_SOURCE_NAME_FIELD"
|
61
|
+
|
62
|
+
METRICS_SERVICE_ERROR_REASON_400 = "INVALID_PARAMETER"
|
63
|
+
METRICS_SERVICE_ERROR_REASON_401 = "AUTHENTICATION_FAILED"
|
64
|
+
METRICS_SERVICE_ERROR_REASON_404 = "AUTHORIZATION_FAILED"
|
65
|
+
METRICS_SERVICE_ERROR_REASON_429 = "TOO_MANY_REQUESTES"
|
66
|
+
METRICS_SERVICE_ERROR_REASON_500 = "INTERNAL_SERVER_ERROR"
|
67
|
+
METRICS_SERVICE_ERROR_REASON_502 = "BAD_GATEWAY"
|
68
|
+
METRICS_SERVICE_ERROR_REASON_503 = "SERVICE_UNAVAILABLE"
|
69
|
+
METRICS_SERVICE_ERROR_REASON_504 = "GATEWAY_TIMEOUT"
|
70
|
+
METRICS_SERVICE_ERROR_REASON_505 = "HTTP_VERSION_NOT_SUPPORTED"
|
71
|
+
METRICS_SERVICE_ERROR_REASON_UNKNOWN = "UNKNOWN_ERROR"
|
72
|
+
|
73
|
+
|
56
74
|
@@logger = nil
|
57
75
|
@@loganalytics_client = nil
|
76
|
+
@@prometheusMetrics = nil
|
58
77
|
@@logger_config_errors = []
|
59
78
|
|
60
79
|
|
@@ -252,6 +271,7 @@ module Fluent::Plugin
|
|
252
271
|
|
253
272
|
def configure(conf)
|
254
273
|
super
|
274
|
+
@@prometheusMetrics = PrometheusMetrics.instance
|
255
275
|
initialize_logger
|
256
276
|
|
257
277
|
initialize_loganalytics_client
|
@@ -504,32 +524,36 @@ module Fluent::Plugin
|
|
504
524
|
|
505
525
|
def is_valid_record(record_hash,record)
|
506
526
|
begin
|
527
|
+
invalid_reason = nil
|
507
528
|
if !record_hash.has_key?("message")
|
529
|
+
invalid_reason = OutOracleOCILogAnalytics::METRICS_INVALID_REASON_MESSAGE
|
508
530
|
if record_hash.has_key?("tag")
|
509
531
|
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}. 'message' field is not present in the record."}
|
510
532
|
else
|
511
533
|
@@logger.info {"InvalidRecord: #{record}"}
|
512
534
|
@@logger.warn {"Invalid record. 'message' field is not present in the record."}
|
513
535
|
end
|
514
|
-
return false
|
536
|
+
return false,invalid_reason
|
515
537
|
elsif !record_hash.has_key?("oci_la_log_group_id") || !is_valid(record["oci_la_log_group_id"])
|
538
|
+
invalid_reason = OutOracleOCILogAnalytics::METRICS_INVALID_REASON_LOG_GROUP_ID
|
516
539
|
if record_hash.has_key?("tag")
|
517
540
|
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}.'oci_la_log_group_id' must not be empty.
|
518
541
|
Skipping all the records associated with the tag"}
|
519
542
|
else
|
520
543
|
@@logger.warn {"Invalid record.'oci_la_log_group_id' must not be empty"}
|
521
544
|
end
|
522
|
-
return false
|
545
|
+
return false,invalid_reason
|
523
546
|
elsif !record_hash.has_key?("oci_la_log_source_name") || !is_valid(record["oci_la_log_source_name"])
|
547
|
+
invalid_reason = OutOracleOCILogAnalytics::METRICS_INVALID_REASON_LOG_SOURCE_NAME
|
524
548
|
if record_hash.has_key?("tag")
|
525
549
|
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}.'oci_la_log_source_name' must not be empty.
|
526
550
|
Skipping all the records associated with the tag"}
|
527
551
|
else
|
528
552
|
@@logger.warn {"Invalid record.'oci_la_log_source_name' must not be empty"}
|
529
553
|
end
|
530
|
-
return false
|
554
|
+
return false,invalid_reason
|
531
555
|
else
|
532
|
-
return true
|
556
|
+
return true,invalid_reason
|
533
557
|
end
|
534
558
|
end
|
535
559
|
end
|
@@ -580,6 +604,11 @@ module Fluent::Plugin
|
|
580
604
|
current_f, current_s = current.to_f, current.strftime("%Y%m%dT%H%M%S%9NZ")
|
581
605
|
records = []
|
582
606
|
count = 0
|
607
|
+
latency = 0
|
608
|
+
records_per_tag = 0
|
609
|
+
|
610
|
+
tag_metrics_set = Hash.new
|
611
|
+
logGroup_labels_set = Hash.new
|
583
612
|
|
584
613
|
invalid_tag_set = Set.new
|
585
614
|
incoming_records_per_tag = Hash.new
|
@@ -587,130 +616,166 @@ module Fluent::Plugin
|
|
587
616
|
tags_per_logGroupId = Hash.new
|
588
617
|
tag_logSet_map = Hash.new
|
589
618
|
tag_metadata_map = Hash.new
|
619
|
+
incoming_records = 0
|
590
620
|
|
591
621
|
chunk.each do |time, record|
|
592
|
-
|
622
|
+
incoming_records += 1
|
623
|
+
metricsLabels = MetricsLabels.new
|
593
624
|
if !record.nil?
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
625
|
+
begin
|
626
|
+
record_hash = record.keys.map {|x| [x,true]}.to_h
|
627
|
+
is_tag_exists = false
|
628
|
+
if record_hash.has_key?("tag") && is_valid(record["tag"])
|
629
|
+
is_tag_exists = true
|
630
|
+
metricsLabels.tag = record["tag"]
|
631
|
+
end
|
599
632
|
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
end
|
605
|
-
#For any given tag, if one record fails (mandatory fields validation) then all the records from that source will be ignored
|
606
|
-
if is_tag_exists && invalid_tag_set.include?(record["tag"])
|
607
|
-
invalid_records_per_tag[record["tag"]] += 1
|
608
|
-
next #This tag is already present in the invalid_tag_set, so ignoring the message.
|
609
|
-
end
|
610
|
-
#Setting tag/default value for oci_la_log_path, when not provided in config file.
|
611
|
-
if !record_hash.has_key?("oci_la_log_path") || !is_valid(record["oci_la_log_path"])
|
612
|
-
if is_tag_exists
|
613
|
-
record["oci_la_log_path"] = record["tag"]
|
614
|
-
else
|
615
|
-
record["oci_la_log_path"] = 'UNDEFINED'
|
633
|
+
if is_tag_exists && incoming_records_per_tag.has_key?(record["tag"])
|
634
|
+
incoming_records_per_tag[record["tag"]] += 1
|
635
|
+
elsif is_tag_exists
|
636
|
+
incoming_records_per_tag[record["tag"]] = 1
|
616
637
|
end
|
617
|
-
|
638
|
+
#For any given tag, if one record fails (mandatory fields validation) then all the records from that source will be ignored
|
639
|
+
if is_tag_exists && invalid_tag_set.include?(record["tag"])
|
640
|
+
invalid_records_per_tag[record["tag"]] += 1
|
641
|
+
next #This tag is already present in the invalid_tag_set, so ignoring the message.
|
642
|
+
end
|
643
|
+
#Setting tag/default value for oci_la_log_path, when not provided in config file.
|
644
|
+
if !record_hash.has_key?("oci_la_log_path") || !is_valid(record["oci_la_log_path"])
|
645
|
+
if is_tag_exists
|
646
|
+
record["oci_la_log_path"] = record["tag"]
|
647
|
+
else
|
648
|
+
record["oci_la_log_path"] = 'UNDEFINED'
|
649
|
+
end
|
650
|
+
end
|
618
651
|
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
652
|
+
#Extracting oci_la_log_set when oci_la_log_set_key and oci_la_log_set_ext_regex is provided.
|
653
|
+
#1) oci_la_log_set param is not provided in config file and above logic not executed.
|
654
|
+
#2) Valid oci_la_log_set_key + No oci_la_log_set_ext_regex
|
655
|
+
#a) Valid key available in record with oci_la_log_set_key corresponding value (oci_la_log_set_key is a key in config file) --> oci_la_log_set
|
656
|
+
#b) No Valid key available in record with oci_la_log_set_key corresponding value --> nil
|
657
|
+
#3) Valid key available in record with oci_la_log_set_key corresponding value + Valid oci_la_log_set_ext_regex
|
658
|
+
#a) Parse success --> parsed oci_la_log_set
|
659
|
+
#b) Parse failure --> nil (as oci_la_log_set value)
|
660
|
+
#4) No oci_la_log_set_key --> do nothing --> nil
|
661
|
+
|
662
|
+
#Extracting oci_la_log_set when oci_la_log_set and oci_la_log_set_ext_regex is provided.
|
663
|
+
#1) Valid oci_la_log_set + No oci_la_log_set_ext_regex --> oci_la_log_set
|
664
|
+
#2) Valid oci_la_log_set + Valid oci_la_log_set_ext_regex
|
665
|
+
#a) Parse success --> parsed oci_la_log_set
|
666
|
+
#b) Parse failure --> nil (as oci_la_log_set value)
|
667
|
+
#3) No oci_la_log_set --> do nothing --> nil
|
668
|
+
|
669
|
+
unparsed_logSet = nil
|
670
|
+
processed_logSet = nil
|
671
|
+
if is_tag_exists && tag_logSet_map.has_key?(record["tag"])
|
672
|
+
record["oci_la_log_set"] = tag_logSet_map[record["tag"]]
|
673
|
+
else
|
674
|
+
if record_hash.has_key?("oci_la_log_set_key")
|
675
|
+
if is_valid(record["oci_la_log_set_key"]) && record_hash.has_key?(record["oci_la_log_set_key"])
|
676
|
+
if is_valid(record[record["oci_la_log_set_key"]])
|
677
|
+
unparsed_logSet = record[record["oci_la_log_set_key"]]
|
678
|
+
processed_logSet = get_or_parse_logSet(unparsed_logSet,record, record_hash,is_tag_exists)
|
679
|
+
end
|
646
680
|
end
|
647
681
|
end
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
682
|
+
if !is_valid(processed_logSet) && record_hash.has_key?("oci_la_log_set")
|
683
|
+
if is_valid(record["oci_la_log_set"])
|
684
|
+
unparsed_logSet = record["oci_la_log_set"]
|
685
|
+
processed_logSet = get_or_parse_logSet(unparsed_logSet,record, record_hash,is_tag_exists)
|
686
|
+
end
|
653
687
|
end
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
688
|
+
record["oci_la_log_set"] = processed_logSet
|
689
|
+
tag_logSet_map[record["tag"]] = processed_logSet
|
690
|
+
end
|
691
|
+
is_valid, metricsLabels.invalid_reason = is_valid_record(record_hash,record)
|
658
692
|
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
invalid_records_per_tag[record["tag"]] = 1
|
663
|
-
end
|
664
|
-
next
|
665
|
-
end
|
666
|
-
#This will check for null or empty messages and only that record will be ignored.
|
667
|
-
if !is_valid(record["message"])
|
668
|
-
if is_tag_exists
|
669
|
-
@@logger.warn {"'message' field has empty value, Skipping records associated with tag : #{record["tag"]}."}
|
670
|
-
if invalid_records_per_tag.has_key?(record["tag"])
|
671
|
-
invalid_records_per_tag[record["tag"]] += 1
|
672
|
-
else
|
693
|
+
unless is_valid
|
694
|
+
if is_tag_exists
|
695
|
+
invalid_tag_set.add(record["tag"])
|
673
696
|
invalid_records_per_tag[record["tag"]] = 1
|
674
697
|
end
|
698
|
+
next
|
699
|
+
end
|
700
|
+
metricsLabels.logGroupId = record["oci_la_log_group_id"]
|
701
|
+
metricsLabels.logSourceName = record["oci_la_log_source_name"]
|
702
|
+
if record["oci_la_log_set"] != nil
|
703
|
+
metricsLabels.logSet = record["oci_la_log_set"]
|
704
|
+
end
|
705
|
+
#This will check for null or empty messages and only that record will be ignored.
|
706
|
+
if !is_valid(record["message"])
|
707
|
+
metricsLabels.invalid_reason = OutOracleOCILogAnalytics::METRICS_INVALID_REASON_MESSAGE
|
708
|
+
if is_tag_exists
|
709
|
+
@@logger.warn {"'message' field has empty value, Skipping records associated with tag : #{record["tag"]}."}
|
710
|
+
if invalid_records_per_tag.has_key?(record["tag"])
|
711
|
+
invalid_records_per_tag[record["tag"]] += 1
|
712
|
+
else
|
713
|
+
invalid_records_per_tag[record["tag"]] = 1
|
714
|
+
end
|
715
|
+
else
|
716
|
+
@@logger.warn {"'message' field has empty value, Skipping record."}
|
717
|
+
end
|
718
|
+
next
|
675
719
|
else
|
676
|
-
|
720
|
+
record["message"] = json_message_handler(record["message"])
|
677
721
|
end
|
678
|
-
next
|
679
|
-
else
|
680
|
-
record["message"] = json_message_handler(record["message"])
|
681
|
-
end
|
682
722
|
|
683
|
-
|
684
|
-
|
685
|
-
|
723
|
+
if record_hash.has_key?("kubernetes")
|
724
|
+
record["oci_la_metadata"] = get_kubernetes_metadata(record["oci_la_metadata"],record)
|
725
|
+
end
|
686
726
|
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
727
|
+
if tag_metadata_map.has_key?(record["tag"])
|
728
|
+
record["oci_la_metadata"] = tag_metadata_map[record["tag"]]
|
729
|
+
else
|
730
|
+
if record_hash.has_key?("oci_la_metadata")
|
731
|
+
record["oci_la_metadata"] = get_valid_metadata(record["oci_la_metadata"])
|
732
|
+
tags_per_logGroupId[record["tag"]] = record["oci_la_metadata"]
|
733
|
+
else
|
734
|
+
tags_per_logGroupId[record["tag"]] = nil
|
735
|
+
end
|
736
|
+
end
|
697
737
|
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
738
|
+
if is_tag_exists
|
739
|
+
if tags_per_logGroupId.has_key?(record["oci_la_log_group_id"])
|
740
|
+
if !tags_per_logGroupId[record["oci_la_log_group_id"]].include?(record["tag"])
|
741
|
+
tags_per_logGroupId[record["oci_la_log_group_id"]] += ", "+record["tag"]
|
742
|
+
end
|
743
|
+
else
|
744
|
+
tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"]
|
745
|
+
end
|
702
746
|
end
|
703
|
-
else
|
704
|
-
tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"]
|
705
|
-
end
|
706
|
-
end
|
707
747
|
|
708
|
-
|
748
|
+
records << record
|
749
|
+
ensure
|
750
|
+
# To get chunk_time_to_receive metrics per tag, corresponding latency and total records are calculated
|
751
|
+
if tag_metrics_set.has_key?(record["tag"])
|
752
|
+
metricsLabels = tag_metrics_set[record["tag"]]
|
753
|
+
latency = metricsLabels.latency
|
754
|
+
records_per_tag = metricsLabels.records_per_tag
|
755
|
+
else
|
756
|
+
latency = 0
|
757
|
+
records_per_tag = 0
|
758
|
+
end
|
759
|
+
latency += (current_f - time)
|
760
|
+
records_per_tag += 1
|
761
|
+
metricsLabels.latency = latency
|
762
|
+
metricsLabels.records_per_tag = records_per_tag
|
763
|
+
tag_metrics_set[record["tag"]] = metricsLabels
|
764
|
+
if record["oci_la_log_group_id"] != nil && !logGroup_labels_set.has_key?(record["oci_la_log_group_id"])
|
765
|
+
logGroup_labels_set[record["oci_la_log_group_id"]] = metricsLabels
|
766
|
+
end
|
767
|
+
end
|
709
768
|
else
|
710
769
|
@@logger.trace {"Record is nil, ignoring the record"}
|
711
770
|
end
|
712
771
|
end
|
713
772
|
@@logger.debug {"records.length:#{records.length}"}
|
773
|
+
|
774
|
+
tag_metrics_set.each do |tag,metricsLabels|
|
775
|
+
latency_avg = (metricsLabels.latency / metricsLabels.records_per_tag).round(3)
|
776
|
+
@@prometheusMetrics.chunk_time_to_receive.observe(latency_avg, labels: { tag: tag})
|
777
|
+
end
|
778
|
+
|
714
779
|
lrpes_for_logGroupId = {}
|
715
780
|
records.group_by{|record|
|
716
781
|
oci_la_log_group_id = record['oci_la_log_group_id']
|
@@ -721,29 +786,62 @@ module Fluent::Plugin
|
|
721
786
|
rescue => ex
|
722
787
|
@@logger.error {"Error occurred while grouping records by oci_la_log_group_id:#{ex.inspect}"}
|
723
788
|
end
|
724
|
-
return incoming_records_per_tag,invalid_records_per_tag,tags_per_logGroupId,lrpes_for_logGroupId
|
789
|
+
return incoming_records_per_tag,invalid_records_per_tag,tag_metrics_set,logGroup_labels_set,tags_per_logGroupId,lrpes_for_logGroupId
|
725
790
|
end
|
726
791
|
# main entry point for FluentD's flush_threads, which get invoked
|
727
792
|
# when a chunk is ready for flushing (see chunk limits and flush_intervals)
|
728
793
|
def write(chunk)
|
729
794
|
@@logger.info {"Received new chunk, started processing ..."}
|
730
|
-
|
731
|
-
metrics["count"] = 0
|
732
|
-
metrics["event"] = "zipping"
|
733
|
-
metrics["bytes_in"] = chunk.bytesize
|
734
|
-
metrics['records_dropped'] = 0
|
795
|
+
#@@prometheusMetrics.bytes_received.set(chunk.bytesize, labels: { tag: nil})
|
735
796
|
begin
|
736
797
|
# 1) Create an in-memory zipfile for the given FluentD chunk
|
737
798
|
# 2) Synchronization has been removed. See EMCLAS-28675
|
738
799
|
|
739
800
|
begin
|
740
801
|
lrpes_for_logGroupId = {}
|
741
|
-
incoming_records_per_tag,invalid_records_per_tag,tags_per_logGroupId,lrpes_for_logGroupId = group_by_logGroupId(chunk)
|
802
|
+
incoming_records_per_tag,invalid_records_per_tag,tag_metrics_set,logGroup_labels_set,tags_per_logGroupId,lrpes_for_logGroupId = group_by_logGroupId(chunk)
|
742
803
|
valid_message_per_tag = Hash.new
|
804
|
+
logGroup_metrics_map = Hash.new
|
805
|
+
metricsLabels_array = []
|
806
|
+
|
743
807
|
incoming_records_per_tag.each do |key,value|
|
744
808
|
dropped_messages = (invalid_records_per_tag.has_key?(key)) ? invalid_records_per_tag[key].to_i : 0
|
745
809
|
valid_messages = value.to_i - dropped_messages
|
746
810
|
valid_message_per_tag[key] = valid_messages
|
811
|
+
|
812
|
+
metricsLabels = tag_metrics_set[key]
|
813
|
+
if metricsLabels == nil
|
814
|
+
metricsLabels = MetricsLabels.new
|
815
|
+
end
|
816
|
+
metricsLabels.records_valid = valid_messages
|
817
|
+
# logGroup_metrics_map will have logGroupId as key and metricsLabels_array as value.
|
818
|
+
# In a chunk we can have different logGroupIds but we are creating payloads based on logGroupId and that can internally have different logSourceName and tag data.
|
819
|
+
# Using logGroup_metrics_map, for a given chunk, we can produce the metrics with proper logGroupId and its corresponding values.
|
820
|
+
if metricsLabels.logGroupId != nil
|
821
|
+
if logGroup_metrics_map.has_key?(metricsLabels.logGroupId)
|
822
|
+
metricsLabels_array = logGroup_metrics_map[metricsLabels.logGroupId]
|
823
|
+
else
|
824
|
+
metricsLabels_array = []
|
825
|
+
end
|
826
|
+
metricsLabels_array.push(metricsLabels)
|
827
|
+
logGroup_metrics_map[metricsLabels.logGroupId] = metricsLabels_array
|
828
|
+
end
|
829
|
+
|
830
|
+
@@prometheusMetrics.records_received.set(value.to_i, labels: { tag: key,
|
831
|
+
oci_la_log_group_id: metricsLabels.logGroupId,
|
832
|
+
oci_la_log_source_name: metricsLabels.logSourceName,
|
833
|
+
oci_la_log_set: metricsLabels.logSet})
|
834
|
+
|
835
|
+
@@prometheusMetrics.records_invalid.set(dropped_messages, labels: { tag: key,
|
836
|
+
oci_la_log_group_id: metricsLabels.logGroupId,
|
837
|
+
oci_la_log_source_name: metricsLabels.logSourceName,
|
838
|
+
oci_la_log_set: metricsLabels.logSet,
|
839
|
+
reason: metricsLabels.invalid_reason})
|
840
|
+
@@prometheusMetrics.records_valid.set(valid_messages, labels: { tag: key,
|
841
|
+
oci_la_log_group_id: metricsLabels.logGroupId,
|
842
|
+
oci_la_log_source_name: metricsLabels.logSourceName,
|
843
|
+
oci_la_log_set: metricsLabels.logSet})
|
844
|
+
|
747
845
|
if dropped_messages > 0
|
748
846
|
@@logger.info {"Messages: #{value.to_i} \t Valid: #{valid_messages} \t Invalid: #{dropped_messages} \t tag:#{key}"}
|
749
847
|
end
|
@@ -758,17 +856,27 @@ module Fluent::Plugin
|
|
758
856
|
zippedstream = nil
|
759
857
|
oci_la_log_set = nil
|
760
858
|
logSets_per_logGroupId_map = Hash.new
|
859
|
+
|
860
|
+
metricsLabels_array = logGroup_metrics_map[oci_la_log_group_id]
|
861
|
+
|
761
862
|
# Only MAX_FILES_PER_ZIP (100) files are allowed, which will be grouped and zipped.
|
762
863
|
# Due to MAX_FILES_PER_ZIP constraint, for a oci_la_log_group_id, we can get more than one zip file and those many api calls will be made.
|
763
864
|
logSets_per_logGroupId_map, oci_la_global_metadata = get_logSets_map_per_logGroupId(oci_la_log_group_id,records_per_logGroupId)
|
764
865
|
if logSets_per_logGroupId_map != nil
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
866
|
+
bytes_out = 0
|
867
|
+
records_out = 0
|
868
|
+
chunk_upload_time_taken = nil
|
869
|
+
chunk_upload_time_taken = Benchmark.measure {
|
870
|
+
logSets_per_logGroupId_map.each do |file_count,records_per_logSet_map|
|
871
|
+
zippedstream,number_of_records = get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map)
|
872
|
+
if zippedstream != nil
|
873
|
+
zippedstream.rewind #reposition buffer pointer to the beginning
|
874
|
+
upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream, metricsLabels_array)
|
875
|
+
end
|
876
|
+
end
|
877
|
+
}.real.round(3)
|
878
|
+
@@prometheusMetrics.chunk_time_to_upload.observe(chunk_upload_time_taken, labels: { oci_la_log_group_id: oci_la_log_group_id})
|
879
|
+
|
772
880
|
end
|
773
881
|
ensure
|
774
882
|
zippedstream&.close
|
@@ -882,8 +990,10 @@ module Fluent::Plugin
|
|
882
990
|
end
|
883
991
|
|
884
992
|
# upload zipped stream to oci
|
885
|
-
def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream)
|
993
|
+
def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream, metricsLabels_array)
|
886
994
|
begin
|
995
|
+
error_reason = nil
|
996
|
+
error_code = nil
|
887
997
|
opts = {payload_type: "ZIP"}
|
888
998
|
|
889
999
|
response = @@loganalytics_client.upload_log_events_file(namespace_name=@namespace,
|
@@ -892,6 +1002,19 @@ module Fluent::Plugin
|
|
892
1002
|
opts)
|
893
1003
|
if !response.nil? && response.status == 200 then
|
894
1004
|
headers = response.headers
|
1005
|
+
|
1006
|
+
metricsLabels_array.each { |metricsLabels|
|
1007
|
+
@@prometheusMetrics.records_posted.set(metricsLabels.records_valid, labels: { tag: metricsLabels.tag,
|
1008
|
+
oci_la_log_group_id: metricsLabels.logGroupId,
|
1009
|
+
oci_la_log_source_name: metricsLabels.logSourceName,
|
1010
|
+
oci_la_log_set: metricsLabels.logSet})
|
1011
|
+
}
|
1012
|
+
|
1013
|
+
#zippedstream.rewind #reposition buffer pointer to the beginning
|
1014
|
+
#zipfile = zippedstream&.sysread&.dup
|
1015
|
+
#bytes_out = zipfile&.length
|
1016
|
+
#@@prometheusMetrics.bytes_posted.set(bytes_out, labels: { oci_la_log_group_id: oci_la_log_group_id})
|
1017
|
+
|
895
1018
|
@@logger.info {"The payload has been successfully uploaded to logAnalytics -
|
896
1019
|
oci_la_log_group_id: #{oci_la_log_group_id},
|
897
1020
|
ConsumedRecords: #{number_of_records},
|
@@ -901,13 +1024,16 @@ module Fluent::Plugin
|
|
901
1024
|
opc-object-id: #{headers['opc-object-id']}"}
|
902
1025
|
end
|
903
1026
|
rescue OCI::Errors::ServiceError => serviceError
|
1027
|
+
error_code = serviceError.status_code
|
904
1028
|
case serviceError.status_code
|
905
1029
|
when 400
|
1030
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_400
|
906
1031
|
@@logger.error {"oci upload exception : Error while uploading the payload. Invalid/Incorrect/missing Parameter - opc-request-id:#{serviceError.request_id}"}
|
907
1032
|
if plugin_retry_on_4xx
|
908
1033
|
raise serviceError
|
909
1034
|
end
|
910
1035
|
when 401
|
1036
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_401
|
911
1037
|
@@logger.error {"oci upload exception : Error while uploading the payload. Not Authenticated.
|
912
1038
|
opc-request-id:#{serviceError.request_id}
|
913
1039
|
message: #{serviceError.message}"}
|
@@ -915,6 +1041,7 @@ module Fluent::Plugin
|
|
915
1041
|
raise serviceError
|
916
1042
|
end
|
917
1043
|
when 404
|
1044
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_404
|
918
1045
|
@@logger.error {"oci upload exception : Error while uploading the payload. Authorization failed for given oci_la_log_group_id against given Tenancy Namespace.
|
919
1046
|
oci_la_log_group_id: #{oci_la_log_group_id}
|
920
1047
|
Namespace: #{@namespace}
|
@@ -924,33 +1051,52 @@ module Fluent::Plugin
|
|
924
1051
|
raise serviceError
|
925
1052
|
end
|
926
1053
|
when 429
|
1054
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_429
|
927
1055
|
@@logger.error {"oci upload exception : Error while uploading the payload. Too Many Requests - opc-request-id:#{serviceError.request_id}"}
|
928
1056
|
raise serviceError
|
929
1057
|
when 500
|
1058
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_500
|
930
1059
|
@@logger.error {"oci upload exception : Error while uploading the payload. Internal Server Error - opc-request-id:#{serviceError.request_id}"}
|
931
1060
|
raise serviceError
|
932
1061
|
|
933
1062
|
when 502
|
1063
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_502
|
934
1064
|
@@logger.error {"oci upload exception : Error while uploading the payload. Bad Gateway - opc-request-id:#{serviceError.request_id}"}
|
935
1065
|
raise serviceError
|
936
1066
|
|
937
1067
|
when 503
|
1068
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_503
|
938
1069
|
@@logger.error {"oci upload exception : Error while uploading the payload. Service unavailable - opc-request-id:#{serviceError.request_id}"}
|
939
1070
|
raise serviceError
|
940
1071
|
|
941
1072
|
when 504
|
1073
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_504
|
942
1074
|
@@logger.error {"oci upload exception : Error while uploading the payload. Gateway Timeout - opc-request-id:#{serviceError.request_id}"}
|
943
1075
|
raise serviceError
|
944
1076
|
|
945
1077
|
when 505
|
1078
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_505
|
946
1079
|
@@logger.error {"oci upload exception : Error while uploading the payload. HTTP Version Not Supported - opc-request-id:#{serviceError.request_id}"}
|
947
1080
|
raise serviceError
|
948
1081
|
else
|
1082
|
+
error_reason = OutOracleOCILogAnalytics::METRICS_SERVICE_ERROR_REASON_UNKNOWN
|
949
1083
|
@@logger.error {"oci upload exception : Error while uploading the payload #{serviceError.message}"}
|
950
1084
|
raise serviceError
|
951
1085
|
end
|
952
1086
|
rescue => ex
|
1087
|
+
error_reason = ex
|
953
1088
|
@@logger.error {"oci upload exception : Error while uploading the payload. #{ex}"}
|
1089
|
+
ensure
|
1090
|
+
if error_reason != nil
|
1091
|
+
metricsLabels_array.each { |metricsLabels|
|
1092
|
+
@@prometheusMetrics.records_error.set(metricsLabels.records_valid, labels: { tag: metricsLabels.tag,
|
1093
|
+
oci_la_log_group_id: metricsLabels.logGroupId,
|
1094
|
+
oci_la_log_source_name: metricsLabels.logSourceName,
|
1095
|
+
oci_la_log_set: metricsLabels.logSet,
|
1096
|
+
error_code: error_code,
|
1097
|
+
reason: error_reason})
|
1098
|
+
}
|
1099
|
+
end
|
954
1100
|
end
|
955
1101
|
end
|
956
1102
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-oci-logging-analytics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oracle
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-04-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -79,14 +79,28 @@ dependencies:
|
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '2.
|
82
|
+
version: '2.16'
|
83
83
|
type: :runtime
|
84
84
|
prerelease: false
|
85
85
|
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '2.
|
89
|
+
version: '2.16'
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: prometheus-client
|
92
|
+
requirement: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 2.1.0
|
97
|
+
type: :runtime
|
98
|
+
prerelease: false
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 2.1.0
|
90
104
|
description: OCI Logging Analytics Fluentd output plugin for ingesting the collected
|
91
105
|
log events to OCI Logging Analytics.
|
92
106
|
email:
|
@@ -97,6 +111,8 @@ extra_rdoc_files: []
|
|
97
111
|
files:
|
98
112
|
- lib/fluent/dto/logEvents.rb
|
99
113
|
- lib/fluent/dto/logEventsJson.rb
|
114
|
+
- lib/fluent/metrics/metricsLabels.rb
|
115
|
+
- lib/fluent/metrics/prometheusMetrics.rb
|
100
116
|
- lib/fluent/plugin/out_oci-logging-analytics.rb
|
101
117
|
homepage:
|
102
118
|
licenses:
|