fluent-plugin-oci-logging-analytics 2.0.4 → 2.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: b2c10b68eefbf373de236156362383500f9746a9
4
- data.tar.gz: 6ad63e3893634f360387c68b03248291d787d33f
2
+ SHA256:
3
+ metadata.gz: b209883985bb866cf22a68d107f378f2345450017ff0273f70742b17bcbc8bfe
4
+ data.tar.gz: 4e7a78239ef46e22991ea524e83dc2f95a0d9300c86753a8a874733a99d30b13
5
5
  SHA512:
6
- metadata.gz: fcb93cdf4da13e23101127320bec26bb96ac3426e2fe34402591febfc1e0ab8494a253e8c089f0bcd13aafea35b740ea353ee164213d89d415fe93a07d8764e2
7
- data.tar.gz: 83115a796e9baf6311c3d3da4f9f137fdbd4cbb74bb52f10a3686cc8441f26315c38522bb32c1acc06652aaae92bd7a1adff643f555df91de11a4b1e84487355
6
+ metadata.gz: 72ef3ff0b97ef16caf94b88f8bfc2f61651992b7f3507f987cefd65e841c3d12f60aff07451916b119177efb5c1f43150978862dd27efbc0d2418ff3b62dfaa2
7
+ data.tar.gz: a690af49fc8e74b3851116121cdb9743ca118f1f38bd69353ea909bc6ed7c0ed5c9912e99f0d920ec231eb78db748b2302a9082f9efc77cc60937bdd72c92ff2
@@ -1,6 +1,7 @@
1
1
  class MetricsLabels
2
- attr_accessor :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
2
+ attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
3
3
  def initialize
4
+ @worker_id = nil
4
5
  @tag = nil
5
6
  @logGroupId = nil
6
7
  @logSourceName = nil
@@ -14,16 +14,15 @@ class PrometheusMetrics
14
14
  end
15
15
  def createMetrics
16
16
  gauge = Prometheus::Client::Gauge
17
- @records_received = gauge.new(:oci_la_fluentd_output_plugin_records_received, docstring: 'Number of records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
18
- @records_valid = gauge.new(:oci_la_fluentd_output_plugin_records_valid, docstring: 'Number of valid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
19
- @records_invalid = gauge.new(:oci_la_fluentd_output_plugin_records_invalid, docstring: 'Number of invalid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:reason])
20
- @records_error = gauge.new(:oci_la_fluentd_output_plugin_records_post_error, docstring: 'Number of records failed posting to OCI Logging Analytics by the Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:error_code, :reason])
21
- @records_posted = gauge.new(:oci_la_fluentd_output_plugin_records_post_success, docstring: 'Number of records posted by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
22
- #@bytes_received = gauge.new(:oci_la_bytes_received, docstring: '...', labels: [:tag])
23
- #@bytes_posted = gauge.new(:oci_la_bytes_posted, docstring: '...', labels: [:oci_la_log_group_id])
17
+ @records_received = gauge.new(:oci_la_fluentd_output_plugin_records_received, docstring: 'Number of records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:worker_id,:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
18
+ @records_valid = gauge.new(:oci_la_fluentd_output_plugin_records_valid, docstring: 'Number of valid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:worker_id,:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
19
+ @records_invalid = gauge.new(:oci_la_fluentd_output_plugin_records_invalid, docstring: 'Number of invalid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:worker_id,:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:reason])
20
+ @records_error = gauge.new(:oci_la_fluentd_output_plugin_records_post_error, docstring: 'Number of records failed posting to OCI Logging Analytics by the Fluentd output plugin.', labels: [:worker_id,:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:error_code, :reason])
21
+ @records_posted = gauge.new(:oci_la_fluentd_output_plugin_records_post_success, docstring: 'Number of records posted by the OCI Logging Analytics Fluentd output plugin.', labels: [:worker_id,:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
22
+
24
23
  histogram = Prometheus::Client::Histogram
25
- @chunk_time_to_receive = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_receive, docstring: 'Average time taken by Fluentd to deliver the collected records from Input plugin to OCI Logging Analytics output plugin.', labels: [:tag])
26
- @chunk_time_to_upload = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_post, docstring: 'Average time taken for posting the received records to OCI Logging Analytics by the Fluentd output plugin.', labels: [:oci_la_log_group_id])
24
+ @chunk_time_to_receive = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_receive, docstring: 'Average time taken by Fluentd to deliver the collected records from Input plugin to OCI Logging Analytics output plugin.', labels: [:worker_id,:tag])
25
+ @chunk_time_to_upload = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_post, docstring: 'Average time taken for posting the received records to OCI Logging Analytics by the Fluentd output plugin.', labels: [:worker_id,:oci_la_log_group_id])
27
26
  end
28
27
 
29
28
  def registerMetrics
@@ -33,8 +32,6 @@ class PrometheusMetrics
33
32
  registry.register(@records_invalid) unless registry.exist?('oci_la_fluentd_output_plugin_records_invalid')
34
33
  registry.register(@records_error) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_error')
35
34
  registry.register(@records_posted) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_success')
36
- #registry.register(@bytes_received) unless registry.exist?('oci_la_bytes_received')
37
- #registry.register(@bytes_posted) unless registry.exist?('oci_la_bytes_valid')
38
35
  registry.register(@chunk_time_to_receive) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_receive')
39
36
  registry.register(@chunk_time_to_upload) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_post')
40
37
  end
@@ -4,6 +4,8 @@
4
4
  require 'fluent/plugin/output'
5
5
  require "benchmark"
6
6
  require 'zip'
7
+ require 'yajl'
8
+ require 'yajl/json_gem'
7
9
 
8
10
  require 'logger'
9
11
  require_relative '../dto/logEventsJson'
@@ -75,6 +77,8 @@ module Fluent::Plugin
75
77
  @@loganalytics_client = nil
76
78
  @@prometheusMetrics = nil
77
79
  @@logger_config_errors = []
80
+ @@worker_id = '0'
81
+ @@encoded_messages_count = 0
78
82
 
79
83
 
80
84
  desc 'OCI Tenancy Namespace.'
@@ -92,7 +96,7 @@ module Fluent::Plugin
92
96
  desc 'Payload zip File Location.'
93
97
  config_param :zip_file_location, :string, :default => nil
94
98
  desc 'The kubernetes_metadata_keys_mapping.'
95
- config_param :kubernetes_metadata_keys_mapping, :hash, :default => {"container_name":"Kubernetes Container Name","namespace_name":"Kubernetes Namespace Name","pod_name":"Kubernetes Pod Name","container_image":"Kubernetes Container Image","host":"Kubernetes Node Name","master_url":"Kubernetes Master Url"}
99
+ config_param :kubernetes_metadata_keys_mapping, :hash, :default => {"container_name":"Container","namespace_name":"Namespace","pod_name":"Pod","container_image":"Container Image Name","host":"Node"}
96
100
 
97
101
 
98
102
  #****************************************************************
@@ -146,7 +150,7 @@ module Fluent::Plugin
146
150
  desc 'The number of threads of output plugins, which is used to write chunks in parallel.'
147
151
  config_set_default :flush_thread_count, 1
148
152
  desc 'The max size of each chunks: events will be written into chunks until the size of chunks become this size.'
149
- config_set_default :chunk_limit_size, 2 * 1024 * 1024 # 2MB
153
+ config_set_default :chunk_limit_size, 4 * 1024 * 1024 # 4MB
150
154
  desc 'The size limitation of this buffer plugin instance.'
151
155
  config_set_default :total_limit_size, 5 * (1024**3) # 5GB
152
156
  desc 'Flush interval'
@@ -170,67 +174,72 @@ module Fluent::Plugin
170
174
  end
171
175
 
172
176
  def initialize_logger()
173
- filename = nil
174
- is_default_log_location = false
175
- if is_valid(@plugin_log_location)
176
- filename = @plugin_log_location[-1] == '/' ? @plugin_log_location : @plugin_log_location +'/'
177
- else
178
- is_default_log_location = true
179
- end
180
- if !is_valid_log_level(@plugin_log_level)
181
- @plugin_log_level = @@default_log_level
182
- end
183
- oci_fluent_output_plugin_log = nil
184
- if is_default_log_location
185
- oci_fluent_output_plugin_log = 'oci-logging-analytics.log'
186
- else
187
- oci_fluent_output_plugin_log = filename+'oci-logging-analytics.log'
188
- end
189
-
190
- logger_config = nil
191
-
192
- if is_valid_number_of_logs(@plugin_log_file_count) && is_valid_log_size(@plugin_log_file_size)
193
- # When customer provided valid log_file_count and log_file_size.
194
- # logger will rotate with max log_file_count with each file having max log_file_size.
195
- # Older logs purged automatically.
196
- @@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_file_count, @@validated_log_size)
197
- logger_config = 'USER_CONFIG'
198
- elsif is_valid_log_rotation(@plugin_log_rotation)
199
- # When customer provided only log_rotation.
200
- # logger will create a new log based on log_rotation (new file everyday if the rotation is daily).
201
- # This will create too many logs over a period of time as log purging is not done.
202
- @@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_rotation)
203
- logger_config = 'FALLBACK_CONFIG'
204
- else
205
- # When customer provided invalid log config, default config is considered.
206
- # logger will rotate with max default log_file_count with each file having max default log_file_size.
207
- # Older logs purged automatically.
208
- @@logger = Logger.new(oci_fluent_output_plugin_log, @@default_number_of_logs, @@default_log_size)
209
- logger_config = 'DEFAULT_CONFIG'
210
- end
211
-
212
- logger_set_level(@plugin_log_level)
177
+ begin
178
+ filename = nil
179
+ is_default_log_location = false
180
+ if is_valid(@plugin_log_location)
181
+ filename = @plugin_log_location[-1] == '/' ? @plugin_log_location : @plugin_log_location +'/'
182
+ else
183
+ @@logger = log
184
+ return
185
+ end
186
+ if !is_valid_log_level(@plugin_log_level)
187
+ @plugin_log_level = @@default_log_level
188
+ end
189
+ oci_fluent_output_plugin_log = nil
190
+ if is_default_log_location
191
+ oci_fluent_output_plugin_log = 'oci-logging-analytics.log'
192
+ else
193
+ oci_fluent_output_plugin_log = filename+'oci-logging-analytics.log'
194
+ end
195
+ logger_config = nil
196
+
197
+ if is_valid_number_of_logs(@plugin_log_file_count) && is_valid_log_size(@plugin_log_file_size)
198
+ # When customer provided valid log_file_count and log_file_size.
199
+ # logger will rotate with max log_file_count with each file having max log_file_size.
200
+ # Older logs purged automatically.
201
+ @@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_file_count, @@validated_log_size)
202
+ logger_config = 'USER_CONFIG'
203
+ elsif is_valid_log_rotation(@plugin_log_rotation)
204
+ # When customer provided only log_rotation.
205
+ # logger will create a new log based on log_rotation (new file everyday if the rotation is daily).
206
+ # This will create too many logs over a period of time as log purging is not done.
207
+ @@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_rotation)
208
+ logger_config = 'FALLBACK_CONFIG'
209
+ else
210
+ # When customer provided invalid log config, default config is considered.
211
+ # logger will rotate with max default log_file_count with each file having max default log_file_size.
212
+ # Older logs purged automatically.
213
+ @@logger = Logger.new(oci_fluent_output_plugin_log, @@default_number_of_logs, @@default_log_size)
214
+ logger_config = 'DEFAULT_CONFIG'
215
+ end
213
216
 
214
- @@logger.info {"Initializing oci-logging-analytics plugin"}
215
- if is_default_log_location
216
- @@logger.info {"plugin_log_location is not specified. oci-logging-analytics.log will be generated under directory from where fluentd is executed."}
217
- end
217
+ logger_set_level(@plugin_log_level)
218
+ @@logger.info {"Initializing oci-logging-analytics plugin"}
219
+ if is_default_log_location
220
+ @@logger.info {"plugin_log_location is not specified. oci-logging-analytics.log will be generated under directory from where fluentd is executed."}
221
+ end
218
222
 
219
- case logger_config
220
- when 'USER_CONFIG'
221
- @@logger.info {"Logger for oci-logging-analytics.log is initialized with config values log size: #{@plugin_log_file_size}, number of logs: #{@plugin_log_file_count}"}
222
- when 'FALLBACK_CONFIG'
223
- @@logger.info {"Logger for oci-logging-analytics.log is initialized with log rotation: #{@plugin_log_rotation}"}
224
- when 'DEFAULT_CONFIG'
225
- @@logger.info {"Logger for oci-logging-analytics.log is initialized with default config values log size: #{@@default_log_size}, number of logs: #{@@default_number_of_logs}"}
226
- end
227
- if @@logger_config_errors.length > 0
228
- @@logger_config_errors. each {|logger_config_error|
229
- @@logger.warn {"#{logger_config_error}"}
230
- }
231
- end
232
- if is_valid_log_age(@plugin_log_age)
233
- @@logger.warn {"'plugin_log_age' field is deprecated. Use 'plugin_log_file_size' and 'plugin_log_file_count' instead."}
223
+ case logger_config
224
+ when 'USER_CONFIG'
225
+ @@logger.info {"Logger for oci-logging-analytics.log is initialized with config values log size: #{@plugin_log_file_size}, number of logs: #{@plugin_log_file_count}"}
226
+ when 'FALLBACK_CONFIG'
227
+ @@logger.info {"Logger for oci-logging-analytics.log is initialized with log rotation: #{@plugin_log_rotation}"}
228
+ when 'DEFAULT_CONFIG'
229
+ @@logger.info {"Logger for oci-logging-analytics.log is initialized with default config values log size: #{@@default_log_size}, number of logs: #{@@default_number_of_logs}"}
230
+ end
231
+ if @@logger_config_errors.length > 0
232
+ @@logger_config_errors. each {|logger_config_error|
233
+ @@logger.warn {"#{logger_config_error}"}
234
+ }
235
+ end
236
+ if is_valid_log_age(@plugin_log_age)
237
+ @@logger.warn {"'plugin_log_age' field is deprecated. Use 'plugin_log_file_size' and 'plugin_log_file_count' instead."}
238
+ end
239
+ rescue => ex
240
+ @@logger = log
241
+ @@logger.error {"Error while initializing logger:#{ex.inspect}"}
242
+ @@logger.info {"Redirecting oci logging analytics logs to STDOUT"}
234
243
  end
235
244
  end
236
245
 
@@ -275,8 +284,8 @@ module Fluent::Plugin
275
284
  initialize_logger
276
285
 
277
286
  initialize_loganalytics_client
278
- @@logger.error {"Error in config file : Buffer plugin must be of @type file."} unless buffer_config['@type'] == 'file'
279
- raise Fluent::ConfigError, "Error in config file : Buffer plugin must be of @type file." unless buffer_config['@type'] == 'file'
287
+ #@@logger.error {"Error in config file : Buffer plugin must be of @type file."} unless buffer_config['@type'] == 'file'
288
+ #raise Fluent::ConfigError, "Error in config file : Buffer plugin must be of @type file." unless buffer_config['@type'] == 'file'
280
289
 
281
290
  is_mandatory_fields_valid,invalid_field_name = mandatory_field_validator
282
291
  if !is_mandatory_fields_valid
@@ -288,8 +297,8 @@ module Fluent::Plugin
288
297
  unless conf.elements(name: 'buffer').empty?
289
298
  buffer_conf = conf.elements(name: 'buffer').first
290
299
  chunk_limit_size_from_conf = buffer_conf['chunk_limit_size']
291
- unless chunk_limit_size_from_conf.nil?
292
- log.debug "chunk limit size as per the configuration file is #{chunk_limit_size_from_conf}"
300
+ unless chunk_limit_size_from_conf.nil? && buffer_config['@type'] != 'file'
301
+ @@logger.debug "chunk limit size as per the configuration file is #{chunk_limit_size_from_conf}"
293
302
  case chunk_limit_size_from_conf.to_s
294
303
  when /([0-9]+)k/i
295
304
  chunk_limit_size_bytes = $~[1].to_i * 1024
@@ -299,13 +308,13 @@ module Fluent::Plugin
299
308
  chunk_limit_size_bytes = $~[1].to_i * (1024 ** 3)
300
309
  when /([0-9]+)t/i
301
310
  chunk_limit_size_bytes = $~[1].to_i * (1024 ** 4)
302
- else
303
- raise Fluent::ConfigError, "error parsing chunk_limit_size"
311
+ #else
312
+ #raise Fluent::ConfigError, "error parsing chunk_limit_size"
304
313
  end
305
314
 
306
- log.debug "chunk limit size in bytes as per the configuration file is #{chunk_limit_size_bytes}"
307
- if !chunk_limit_size_bytes.between?(1048576, 2097152)
308
- raise Fluent::ConfigError, "chunk_limit_size must be between 1MB and 2MB"
315
+ @@logger.debug "chunk limit size in bytes as per the configuration file is #{chunk_limit_size_bytes}"
316
+ if chunk_limit_size_bytes != nil && !chunk_limit_size_bytes.between?(1048576, 4194304)
317
+ raise Fluent::ConfigError, "chunk_limit_size must be between 1MB and 4MB"
309
318
  end
310
319
  end
311
320
  end
@@ -577,7 +586,7 @@ module Fluent::Plugin
577
586
  kubernetes_metadata.each do |key, value|
578
587
  if kubernetes_metadata_keys_mapping.has_key?(key)
579
588
  if !is_valid(oci_la_metadata[kubernetes_metadata_keys_mapping[key]])
580
- oci_la_metadata[kubernetes_metadata_keys_mapping[key]] = json_message_handler(value)
589
+ oci_la_metadata[kubernetes_metadata_keys_mapping[key]] = json_message_handler(key, value)
581
590
  end
582
591
  end
583
592
  end
@@ -588,14 +597,21 @@ module Fluent::Plugin
588
597
  return oci_la_metadata
589
598
  end
590
599
 
591
- def json_message_handler(message)
592
- if message.is_a?(Hash)
593
- return JSON.generate(message)
594
- else
595
- return message
596
- end
597
- rescue => ex
598
- return message
600
+ def json_message_handler(key, message)
601
+ begin
602
+ if !is_valid(message)
603
+ return nil
604
+ end
605
+ if message.is_a?(Hash)
606
+ return Yajl.dump(message) #JSON.generate(message)
607
+ end
608
+ return message
609
+ rescue => ex
610
+ @@logger.error {"Error occured while generating json for
611
+ field: #{key}
612
+ exception : #{ex}"}
613
+ return nil
614
+ end
599
615
  end
600
616
 
601
617
  def group_by_logGroupId(chunk)
@@ -624,6 +640,10 @@ module Fluent::Plugin
624
640
  if !record.nil?
625
641
  begin
626
642
  record_hash = record.keys.map {|x| [x,true]}.to_h
643
+ if record_hash.has_key?("worker_id") && is_valid(record["worker_id"])
644
+ metricsLabels.worker_id = record["worker_id"]||= '0'
645
+ @@worker_id = record["worker_id"]||= '0'
646
+ end
627
647
  is_tag_exists = false
628
648
  if record_hash.has_key?("tag") && is_valid(record["tag"])
629
649
  is_tag_exists = true
@@ -702,22 +722,23 @@ module Fluent::Plugin
702
722
  if record["oci_la_log_set"] != nil
703
723
  metricsLabels.logSet = record["oci_la_log_set"]
704
724
  end
725
+ record["message"] = json_message_handler("message", record["message"])
726
+
727
+
705
728
  #This will check for null or empty messages and only that record will be ignored.
706
729
  if !is_valid(record["message"])
707
730
  metricsLabels.invalid_reason = OutOracleOCILogAnalytics::METRICS_INVALID_REASON_MESSAGE
708
731
  if is_tag_exists
709
- @@logger.warn {"'message' field has empty value, Skipping records associated with tag : #{record["tag"]}."}
710
732
  if invalid_records_per_tag.has_key?(record["tag"])
711
733
  invalid_records_per_tag[record["tag"]] += 1
712
734
  else
713
735
  invalid_records_per_tag[record["tag"]] = 1
736
+ @@logger.warn {"'message' field is empty or encoded, Skipping records associated with tag : #{record["tag"]}."}
714
737
  end
715
738
  else
716
- @@logger.warn {"'message' field has empty value, Skipping record."}
739
+ @@logger.warn {"'message' field is empty or encoded, Skipping record."}
717
740
  end
718
741
  next
719
- else
720
- record["message"] = json_message_handler(record["message"])
721
742
  end
722
743
 
723
744
  if record_hash.has_key?("kubernetes")
@@ -773,7 +794,7 @@ module Fluent::Plugin
773
794
 
774
795
  tag_metrics_set.each do |tag,metricsLabels|
775
796
  latency_avg = (metricsLabels.latency / metricsLabels.records_per_tag).round(3)
776
- @@prometheusMetrics.chunk_time_to_receive.observe(latency_avg, labels: { tag: tag})
797
+ @@prometheusMetrics.chunk_time_to_receive.observe(latency_avg, labels: { worker_id: metricsLabels.worker_id, tag: tag})
777
798
  end
778
799
 
779
800
  lrpes_for_logGroupId = {}
@@ -827,17 +848,20 @@ module Fluent::Plugin
827
848
  logGroup_metrics_map[metricsLabels.logGroupId] = metricsLabels_array
828
849
  end
829
850
 
830
- @@prometheusMetrics.records_received.set(value.to_i, labels: { tag: key,
851
+ @@prometheusMetrics.records_received.set(value.to_i, labels: { worker_id: metricsLabels.worker_id,
852
+ tag: key,
831
853
  oci_la_log_group_id: metricsLabels.logGroupId,
832
854
  oci_la_log_source_name: metricsLabels.logSourceName,
833
855
  oci_la_log_set: metricsLabels.logSet})
834
856
 
835
- @@prometheusMetrics.records_invalid.set(dropped_messages, labels: { tag: key,
857
+ @@prometheusMetrics.records_invalid.set(dropped_messages, labels: { worker_id: metricsLabels.worker_id,
858
+ tag: key,
836
859
  oci_la_log_group_id: metricsLabels.logGroupId,
837
860
  oci_la_log_source_name: metricsLabels.logSourceName,
838
861
  oci_la_log_set: metricsLabels.logSet,
839
862
  reason: metricsLabels.invalid_reason})
840
- @@prometheusMetrics.records_valid.set(valid_messages, labels: { tag: key,
863
+ @@prometheusMetrics.records_valid.set(valid_messages, labels: { worker_id: metricsLabels.worker_id,
864
+ tag: key,
841
865
  oci_la_log_group_id: metricsLabels.logGroupId,
842
866
  oci_la_log_source_name: metricsLabels.logSourceName,
843
867
  oci_la_log_set: metricsLabels.logSet})
@@ -875,7 +899,7 @@ module Fluent::Plugin
875
899
  end
876
900
  end
877
901
  }.real.round(3)
878
- @@prometheusMetrics.chunk_time_to_upload.observe(chunk_upload_time_taken, labels: { oci_la_log_group_id: oci_la_log_group_id})
902
+ @@prometheusMetrics.chunk_time_to_upload.observe(chunk_upload_time_taken, labels: { worker_id: @@worker_id, oci_la_log_group_id: oci_la_log_group_id})
879
903
 
880
904
  end
881
905
  ensure
@@ -956,7 +980,7 @@ module Fluent::Plugin
956
980
  @@logger.debug {"Added entry #{nextEntry} for oci_la_log_set #{oci_la_log_set} into the zip."}
957
981
  zos.put_next_entry(nextEntry)
958
982
  logEventsJsonFinal = LogEventsJson.new(oci_la_global_metadata,lrpes_for_logEvents)
959
- zos.write logEventsJsonFinal.to_hash.to_json
983
+ zos.write Yajl.dump(logEventsJsonFinal.to_hash)
960
984
  end
961
985
  }
962
986
  zippedstream.rewind
@@ -1002,13 +1026,15 @@ module Fluent::Plugin
1002
1026
  opts)
1003
1027
  if !response.nil? && response.status == 200 then
1004
1028
  headers = response.headers
1005
-
1006
- metricsLabels_array.each { |metricsLabels|
1007
- @@prometheusMetrics.records_posted.set(metricsLabels.records_valid, labels: { tag: metricsLabels.tag,
1008
- oci_la_log_group_id: metricsLabels.logGroupId,
1009
- oci_la_log_source_name: metricsLabels.logSourceName,
1010
- oci_la_log_set: metricsLabels.logSet})
1011
- }
1029
+ if metricsLabels_array != nil
1030
+ metricsLabels_array.each { |metricsLabels|
1031
+ @@prometheusMetrics.records_posted.set(metricsLabels.records_valid, labels: { worker_id: metricsLabels.worker_id,
1032
+ tag: metricsLabels.tag,
1033
+ oci_la_log_group_id: metricsLabels.logGroupId,
1034
+ oci_la_log_source_name: metricsLabels.logSourceName,
1035
+ oci_la_log_set: metricsLabels.logSet})
1036
+ }
1037
+ end
1012
1038
 
1013
1039
  #zippedstream.rewind #reposition buffer pointer to the beginning
1014
1040
  #zipfile = zippedstream&.sysread&.dup
@@ -1087,9 +1113,10 @@ module Fluent::Plugin
1087
1113
  error_reason = ex
1088
1114
  @@logger.error {"oci upload exception : Error while uploading the payload. #{ex}"}
1089
1115
  ensure
1090
- if error_reason != nil
1116
+ if error_reason != nil && metricsLabels_array != nil
1091
1117
  metricsLabels_array.each { |metricsLabels|
1092
- @@prometheusMetrics.records_error.set(metricsLabels.records_valid, labels: { tag: metricsLabels.tag,
1118
+ @@prometheusMetrics.records_error.set(metricsLabels.records_valid, labels: {worker_id: metricsLabels.worker_id,
1119
+ tag: metricsLabels.tag,
1093
1120
  oci_la_log_group_id: metricsLabels.logGroupId,
1094
1121
  oci_la_log_source_name: metricsLabels.logSourceName,
1095
1122
  oci_la_log_set: metricsLabels.logSet,
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-oci-logging-analytics
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.4
4
+ version: 2.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oracle
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-06-20 00:00:00.000000000 Z
12
+ date: 2023-04-12 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -101,6 +101,26 @@ dependencies:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
103
  version: 4.0.0
104
+ - !ruby/object:Gem::Dependency
105
+ name: yajl-ruby
106
+ requirement: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.4'
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: 1.4.3
114
+ type: :runtime
115
+ prerelease: false
116
+ version_requirements: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - "~>"
119
+ - !ruby/object:Gem::Version
120
+ version: '1.4'
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: 1.4.3
104
124
  description: OCI Logging Analytics Fluentd output plugin for ingesting the collected
105
125
  log events to OCI Logging Analytics.
106
126
  email:
@@ -136,8 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
136
156
  - !ruby/object:Gem::Version
137
157
  version: '0'
138
158
  requirements: []
139
- rubyforge_project:
140
- rubygems_version: 2.6.13
159
+ rubygems_version: 3.0.3
141
160
  signing_key:
142
161
  specification_version: 4
143
162
  summary: Fluentd output plugin for OCI Logging Analytics.