fluent-plugin-oci-logging-analytics 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: dde9a7a3e6be01a2556efa148f43934c2c723ffce544d3c11af1c50fe46f06b7
|
4
|
+
data.tar.gz: b442ff6c28ef6bc1c3ddc6d74c6b48e30c9921902607f9581c0aa116e5ba9db2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a106761ec570d6c14435544617bf2357846b022869ef7c6154463b848cf72eeedbc96080a7d819911518818864379655aaa7668c4bad46d4d45a80a10dffc40f
|
7
|
+
data.tar.gz: dd7fcd53fd707c1c365cd8c5cbe1f4d6612d27b0de201d9ed2e1a98235322b08cdb7c4d15f968fa3bc667175e226cae1022673c36ba6409ee73bbe0e703417aa
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
class LogEvents
|
3
|
+
attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata
|
4
|
+
def initialize(lrpe_key, fluentd_records)
|
5
|
+
@metadata, @entityId, @entityType, @logSourceName, @logPath = lrpe_key
|
6
|
+
@logRecords = fluentd_records.map{ |record|
|
7
|
+
record['message']
|
8
|
+
}
|
9
|
+
end
|
10
|
+
|
11
|
+
def to_hash
|
12
|
+
{
|
13
|
+
metadata: @metadata,
|
14
|
+
entityId: @entityId,
|
15
|
+
entityType: @entityType,
|
16
|
+
logSourceName: @logSourceName,
|
17
|
+
logPath: @logPath,
|
18
|
+
logRecords: @logRecords
|
19
|
+
}.compact
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
require_relative './logEvents'
|
3
|
+
|
4
|
+
class LogEventsJson
|
5
|
+
attr_accessor :metadata, :LogEvents
|
6
|
+
def initialize(metadata,logEvents)
|
7
|
+
if metadata!= nil || metadata != 'null'
|
8
|
+
@metadata = metadata
|
9
|
+
end
|
10
|
+
@LogEvents = logEvents
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_hash
|
14
|
+
{
|
15
|
+
metadata: @metadata,
|
16
|
+
logEvents: @LogEvents.map do |le|
|
17
|
+
le.to_hash
|
18
|
+
end
|
19
|
+
}.compact
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'prometheus/client'
|
2
|
+
require 'prometheus/client/registry'
|
3
|
+
require 'prometheus/client/gauge'
|
4
|
+
require 'prometheus/client/histogram'
|
5
|
+
require 'singleton'
|
6
|
+
|
7
|
+
class PrometheusMetrics
|
8
|
+
include Singleton
|
9
|
+
attr_accessor :records_received, :records_valid, :records_dropped, :records_error, :records_posted,
|
10
|
+
:bytes_received, :bytes_posted, :latency_avg, :chunk_upload_time_taken
|
11
|
+
def initialize
|
12
|
+
createMetrics
|
13
|
+
registerMetrics
|
14
|
+
end
|
15
|
+
def createMetrics
|
16
|
+
gauge = Prometheus::Client::Gauge
|
17
|
+
@records_received = gauge.new(:oci_logging_analytics_records_received, docstring: '...', labels: [:tag,:logGroupId,:logSourceName,:logSet])
|
18
|
+
@records_valid = gauge.new(:oci_logging_analytics_records_valid, docstring: '...', labels: [:tag,:logGroupId,:logSourceName,:logSet])
|
19
|
+
@records_dropped = gauge.new(:oci_logging_analytics_records_dropped, docstring: '...', labels: [:tag,:logGroupId,:logSourceName,:logSet,:drop_reason])
|
20
|
+
@records_error = gauge.new(:oci_logging_analytics_records_error, docstring: '...', labels: [:tag,:logGroupId,:logSourceName,:logSet,:error_code, :error_reason])
|
21
|
+
@records_posted = gauge.new(:oci_logging_analytics_records_posted, docstring: '...', labels: [:tag,:logGroupId,:logSourceName,:logSet])
|
22
|
+
@bytes_received = gauge.new(:oci_logging_analytics_bytes_received, docstring: '...', labels: [:tag])
|
23
|
+
@bytes_posted = gauge.new(:oci_logging_analytics_bytes_posted, docstring: '...', labels: [:logGroupId])
|
24
|
+
histogram = Prometheus::Client::Histogram
|
25
|
+
@latency_avg = histogram.new(:oci_logging_analytics_network_latency_avg, docstring: '...', labels: [:tag])
|
26
|
+
@chunk_upload_time_taken = histogram.new(:oci_logging_analytics_chunk_upload_time_taken, docstring: '...', labels: [:logGroupId])
|
27
|
+
end
|
28
|
+
|
29
|
+
def registerMetrics
|
30
|
+
registry = Prometheus::Client.registry
|
31
|
+
registry.register(@records_received) unless registry.exist?('oci_logging_analytics_records_received')
|
32
|
+
registry.register(@records_valid) unless registry.exist?('oci_logging_analytics_records_valid')
|
33
|
+
registry.register(@records_dropped) unless registry.exist?('oci_logging_analytics_records_dropped')
|
34
|
+
registry.register(@records_error) unless registry.exist?('oci_logging_analytics_records_error')
|
35
|
+
registry.register(@records_posted) unless registry.exist?('oci_logging_analytics_records_posted')
|
36
|
+
registry.register(@bytes_received) unless registry.exist?('oci_logging_analytics_bytes_received')
|
37
|
+
registry.register(@bytes_posted) unless registry.exist?('oci_logging_analytics_bytes_valid')
|
38
|
+
registry.register(@latency_avg) unless registry.exist?('oci_logging_analytics_network_latency_avg')
|
39
|
+
registry.register(@chunk_upload_time_taken) unless registry.exist?('oci_logging_analytics_chunk_upload_time_taken')
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,919 @@
|
|
1
|
+
## Copyright (c) 2021 Oracle and/or its affiliates.
|
2
|
+
## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
3
|
+
|
4
|
+
require 'fluent/plugin/output'
|
5
|
+
require "benchmark"
|
6
|
+
require 'zip'
|
7
|
+
require 'oci'
|
8
|
+
require 'logger'
|
9
|
+
require_relative '../dto/logEventsJson'
|
10
|
+
require_relative '../dto/logEvents'
|
11
|
+
|
12
|
+
module Fluent::Plugin
|
13
|
+
class OutOracleOCILogAnalytics < Output
|
14
|
+
Fluent::Plugin.register_output('oci-logging-analytics', self)
|
15
|
+
helpers :thread, :event_emitter
|
16
|
+
|
17
|
+
MAX_FILES_PER_ZIP = 100
|
18
|
+
@@logger = nil
|
19
|
+
@@loganalytics_client = nil
|
20
|
+
@@logger_config_errors = []
|
21
|
+
|
22
|
+
|
23
|
+
desc 'OCI Tenancy Namespace.'
|
24
|
+
config_param :namespace, :string, :default => nil
|
25
|
+
desc 'OCI config file location.'
|
26
|
+
config_param :config_file_location, :string, :default => nil
|
27
|
+
desc 'Name of the profile to be used.'
|
28
|
+
config_param :profile_name, :string, :default => 'DEFAULT'
|
29
|
+
desc 'OCI endpoint.'
|
30
|
+
config_param :endpoint, :string, :default => nil
|
31
|
+
desc 'AuthType to be used.'
|
32
|
+
config_param :auth_type, :string, :default => 'InstancePrincipal'
|
33
|
+
desc 'Enable local payload dump.'
|
34
|
+
config_param :dump_zip_file, :bool, :default => false
|
35
|
+
desc 'Payload zip File Location.'
|
36
|
+
config_param :zip_file_location, :string, :default => nil
|
37
|
+
desc 'The kubernetes_metadata_keys_mapping.'
|
38
|
+
config_param :kubernetes_metadata_keys_mapping, :hash, :default => {"container_name":"Kubernetes Container Name","namespace_name":"Kubernetes Namespace Name","pod_name":"Kubernetes Pod Name","container_image":"Kubernetes Container Image","host":"Kubernetes Node Name","master_url":"Kubernetes Master Url"}
|
39
|
+
|
40
|
+
|
41
|
+
#****************************************************************
|
42
|
+
desc 'The http proxy to be used.'
|
43
|
+
config_param :http_proxy, :string, :default => nil
|
44
|
+
desc 'The proxy_ip to be used.'
|
45
|
+
config_param :proxy_ip, :string, :default => nil
|
46
|
+
desc 'The proxy_port to be used.'
|
47
|
+
config_param :proxy_port, :integer, :default => 80
|
48
|
+
desc 'The proxy_username to be used.'
|
49
|
+
config_param :proxy_username, :string, :default => nil
|
50
|
+
desc 'The proxy_password to be used.'
|
51
|
+
config_param :proxy_password, :string, :default => nil
|
52
|
+
|
53
|
+
desc 'OCI Output plugin log location.'
|
54
|
+
config_param :plugin_log_location, :string, :default => nil
|
55
|
+
desc 'OCI Output plugin log level.'
|
56
|
+
config_param :plugin_log_level, :string, :default => nil
|
57
|
+
desc 'OCI Output plugin log rotation.'
|
58
|
+
config_param :plugin_log_rotation, :string, :default => nil
|
59
|
+
desc 'OCI Output plugin log age.'
|
60
|
+
config_param :plugin_log_age, :string, :default => nil # Deprecated
|
61
|
+
desc 'The maximum log file size at which point the log file to be rotated.'
|
62
|
+
config_param :plugin_log_file_size, :string, :default => nil
|
63
|
+
desc 'The number of archived/rotated log files to keep.'
|
64
|
+
config_param :plugin_log_file_count, :integer, :default => 10
|
65
|
+
|
66
|
+
desc 'OCI Output plugin 4xx exception handling.' # Except '429'
|
67
|
+
config_param :plugin_retry_on_4xx, :bool, :default => false
|
68
|
+
|
69
|
+
@@default_log_level = 'info'
|
70
|
+
@@default_log_rotation = 'daily'
|
71
|
+
@@validated_log_size = nil
|
72
|
+
@@default_log_size = 1 * 1024 * 1024 # 1MB
|
73
|
+
@@default_number_of_logs = 10
|
74
|
+
|
75
|
+
#************************************************************************
|
76
|
+
# following params are only for internal testing.
|
77
|
+
|
78
|
+
desc 'Default is false. When true, prohibits HTTP requests to oci.'
|
79
|
+
config_param :test_flag, :bool, :default => false
|
80
|
+
desc 'Sets the environment. Default is prod.'
|
81
|
+
config_param :environment, :string, :default => "prod"
|
82
|
+
desc 'Default log group'
|
83
|
+
config_param :default_log_group,:string, :default => nil
|
84
|
+
#*************************************************************************
|
85
|
+
# define default buffer section
|
86
|
+
config_section :buffer do
|
87
|
+
config_set_default :type, 'file'
|
88
|
+
config_set_default :chunk_keys, ['oci_la_log_group_id']
|
89
|
+
desc 'The number of threads of output plugins, which is used to write chunks in parallel.'
|
90
|
+
config_set_default :flush_thread_count, 1
|
91
|
+
desc 'The max size of each chunks: events will be written into chunks until the size of chunks become this size.'
|
92
|
+
config_set_default :chunk_limit_size, 2 * 1024 * 1024 # 2MB
|
93
|
+
desc 'The size limitation of this buffer plugin instance.'
|
94
|
+
config_set_default :total_limit_size, 5 * (1024**3) # 5GB
|
95
|
+
desc 'Flush interval'
|
96
|
+
config_set_default :flush_interval, 30 # seconds
|
97
|
+
desc 'The sleep interval of threads to wait next flush trial (when no chunks are waiting).'
|
98
|
+
config_set_default :flush_thread_interval, 0.5
|
99
|
+
desc 'The sleep interval seconds of threads between flushes when output plugin flushes waiting chunks next to next.'
|
100
|
+
config_set_default :flush_thread_burst_interval, 0.05
|
101
|
+
desc 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
|
102
|
+
config_set_default :retry_wait, 2 # seconds
|
103
|
+
desc 'The maximum number of times to retry to flush while failing.'
|
104
|
+
config_set_default :retry_max_times, 17
|
105
|
+
desc 'The base number of exponential backoff for retries.'
|
106
|
+
config_set_default :retry_exponential_backoff_base , 2
|
107
|
+
desc 'retry_forever'
|
108
|
+
config_set_default :retry_forever, true
|
109
|
+
end
|
110
|
+
|
111
|
+
def initialize
|
112
|
+
super
|
113
|
+
end
|
114
|
+
|
115
|
+
def initialize_logger()
|
116
|
+
filename = nil
|
117
|
+
is_default_log_location = false
|
118
|
+
if is_valid(@plugin_log_location)
|
119
|
+
filename = @plugin_log_location[-1] == '/' ? @plugin_log_location : @plugin_log_location +'/'
|
120
|
+
else
|
121
|
+
is_default_log_location = true
|
122
|
+
end
|
123
|
+
if !is_valid_log_level(@plugin_log_level)
|
124
|
+
@plugin_log_level = @@default_log_level
|
125
|
+
end
|
126
|
+
oci_fluent_output_plugin_log = nil
|
127
|
+
if is_default_log_location
|
128
|
+
oci_fluent_output_plugin_log = 'oci-logging-analytics.log'
|
129
|
+
else
|
130
|
+
oci_fluent_output_plugin_log = filename+'oci-logging-analytics.log'
|
131
|
+
end
|
132
|
+
|
133
|
+
logger_config = nil
|
134
|
+
|
135
|
+
if is_valid_number_of_logs(@plugin_log_file_count) && is_valid_log_size(@plugin_log_file_size)
|
136
|
+
# When customer provided valid log_file_count and log_file_size.
|
137
|
+
# logger will rotate with max log_file_count with each file having max log_file_size.
|
138
|
+
# Older logs purged automatically.
|
139
|
+
@@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_file_count, @@validated_log_size)
|
140
|
+
logger_config = 'USER_CONFIG'
|
141
|
+
elsif is_valid_log_rotation(@plugin_log_rotation)
|
142
|
+
# When customer provided only log_rotation.
|
143
|
+
# logger will create a new log based on log_rotation (new file everyday if the rotation is daily).
|
144
|
+
# This will create too many logs over a period of time as log purging is not done.
|
145
|
+
@@logger = Logger.new(oci_fluent_output_plugin_log, @plugin_log_rotation)
|
146
|
+
logger_config = 'FALLBACK_CONFIG'
|
147
|
+
else
|
148
|
+
# When customer provided invalid log config, default config is considered.
|
149
|
+
# logger will rotate with max default log_file_count with each file having max default log_file_size.
|
150
|
+
# Older logs purged automatically.
|
151
|
+
@@logger = Logger.new(oci_fluent_output_plugin_log, @@default_number_of_logs, @@default_log_size)
|
152
|
+
logger_config = 'DEFAULT_CONFIG'
|
153
|
+
end
|
154
|
+
|
155
|
+
logger_set_level(@plugin_log_level)
|
156
|
+
|
157
|
+
@@logger.info {"Initializing oci-logging-analytics plugin"}
|
158
|
+
if is_default_log_location
|
159
|
+
@@logger.info {"plugin_log_location is not specified. oci-logging-analytics.log will be generated under directory from where fluentd is executed."}
|
160
|
+
end
|
161
|
+
|
162
|
+
case logger_config
|
163
|
+
when 'USER_CONFIG'
|
164
|
+
@@logger.info {"Logger for oci-logging-analytics.log is initialized with config values log size: #{@plugin_log_file_size}, number of logs: #{@plugin_log_file_count}"}
|
165
|
+
when 'FALLBACK_CONFIG'
|
166
|
+
@@logger.info {"Logger for oci-logging-analytics.log is initialized with log rotation: #{@plugin_log_rotation}"}
|
167
|
+
when 'DEFAULT_CONFIG'
|
168
|
+
@@logger.info {"Logger for oci-logging-analytics.log is initialized with default config values log size: #{@@default_log_size}, number of logs: #{@@default_number_of_logs}"}
|
169
|
+
end
|
170
|
+
if @@logger_config_errors.length > 0
|
171
|
+
@@logger_config_errors. each {|logger_config_error|
|
172
|
+
@@logger.warn {"#{logger_config_error}"}
|
173
|
+
}
|
174
|
+
end
|
175
|
+
if is_valid_log_age(@plugin_log_age)
|
176
|
+
@@logger.warn {"'plugin_log_age' field is deprecated. Use 'plugin_log_file_size' and 'plugin_log_file_count' instead."}
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
def initialize_loganalytics_client()
|
181
|
+
if is_valid(@config_file_location)
|
182
|
+
@auth_type = "ConfigFile"
|
183
|
+
end
|
184
|
+
case @auth_type
|
185
|
+
when "InstancePrincipal"
|
186
|
+
instance_principals_signer = OCI::Auth::Signers::InstancePrincipalsSecurityTokenSigner.new
|
187
|
+
@@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: instance_principals_signer)
|
188
|
+
when "ConfigFile"
|
189
|
+
my_config = OCI::ConfigFileLoader.load_config(config_file_location: @config_file_location, profile_name: @profile_name)
|
190
|
+
if is_valid(endpoint)
|
191
|
+
@@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config:my_config, endpoint:@endpoint)
|
192
|
+
@@logger.info {"loganalytics_client initialised with endpoint: #{@endpoint}"}
|
193
|
+
else
|
194
|
+
@@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config:my_config)
|
195
|
+
end
|
196
|
+
else
|
197
|
+
raise Fluent::ConfigError, "Invalid authType @auth_type, authType must be either InstancePrincipal or ConfigFile."
|
198
|
+
abort
|
199
|
+
end
|
200
|
+
|
201
|
+
if is_valid(@proxy_ip) && is_number(@proxy_port)
|
202
|
+
if is_valid(@proxy_username) && is_valid(@proxy_password)
|
203
|
+
@@loganalytics_client.api_client.proxy_settings = OCI::ApiClientProxySettings.new(@proxy_ip, @proxy_port, @proxy_username, @proxy_password)
|
204
|
+
else
|
205
|
+
@@loganalytics_client.api_client.proxy_settings = OCI::ApiClientProxySettings.new(@proxy_ip, @proxy_port)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
rescue => ex
|
210
|
+
@@logger.error {"Error occurred while initializing LogAnalytics Client:
|
211
|
+
authType: #{@auth_type},
|
212
|
+
errorMessage: #{ex}"}
|
213
|
+
end
|
214
|
+
|
215
|
+
def configure(conf)
|
216
|
+
super
|
217
|
+
initialize_logger
|
218
|
+
|
219
|
+
initialize_loganalytics_client
|
220
|
+
@@logger.error {"Error in config file : Buffer plugin must be of @type file."} unless buffer_config['@type'] == 'file'
|
221
|
+
raise Fluent::ConfigError, "Error in config file : Buffer plugin must be of @type file." unless buffer_config['@type'] == 'file'
|
222
|
+
|
223
|
+
is_mandatory_fields_valid,invalid_field_name = mandatory_field_validator
|
224
|
+
if !is_mandatory_fields_valid
|
225
|
+
@@logger.error {"Error in config file : invalid #{invalid_field_name}"}
|
226
|
+
raise Fluent::ConfigError, "Error in config file : invalid #{invalid_field_name}"
|
227
|
+
end
|
228
|
+
|
229
|
+
# Get the chunk_limit_size from conf as it's not available in the buffer_config
|
230
|
+
unless conf.elements(name: 'buffer').empty?
|
231
|
+
buffer_conf = conf.elements(name: 'buffer').first
|
232
|
+
chunk_limit_size_from_conf = buffer_conf['chunk_limit_size']
|
233
|
+
unless chunk_limit_size_from_conf.nil?
|
234
|
+
log.debug "chunk limit size as per the configuration file is #{chunk_limit_size_from_conf}"
|
235
|
+
case chunk_limit_size_from_conf.to_s
|
236
|
+
when /([0-9]+)k/i
|
237
|
+
chunk_limit_size_bytes = $~[1].to_i * 1024
|
238
|
+
when /([0-9]+)m/i
|
239
|
+
chunk_limit_size_bytes = $~[1].to_i * (1024 ** 2)
|
240
|
+
when /([0-9]+)g/i
|
241
|
+
chunk_limit_size_bytes = $~[1].to_i * (1024 ** 3)
|
242
|
+
when /([0-9]+)t/i
|
243
|
+
chunk_limit_size_bytes = $~[1].to_i * (1024 ** 4)
|
244
|
+
else
|
245
|
+
raise Fluent::ConfigError, "error parsing chunk_limit_size"
|
246
|
+
end
|
247
|
+
|
248
|
+
log.debug "chunk limit size in bytes as per the configuration file is #{chunk_limit_size_bytes}"
|
249
|
+
if !chunk_limit_size_bytes.between?(1048576, 2097152)
|
250
|
+
raise Fluent::ConfigError, "chunk_limit_size must be between 1MB and 2MB"
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
if buffer_config.flush_interval < 10
|
256
|
+
raise Fluent::ConfigError, "flush_interval must be greater than or equal to 10sec"
|
257
|
+
end
|
258
|
+
@mutex = Mutex.new
|
259
|
+
@num_flush_threads = Float(buffer_config.flush_thread_count)
|
260
|
+
max_chunk_lifespan = (buffer_config.retry_type == :exponential_backoff) ?
|
261
|
+
buffer_config.retry_wait * buffer_config.retry_exponential_backoff_base**(buffer_config.retry_max_times+1) - 1 :
|
262
|
+
buffer_config.retry_wait * buffer_config.retry_max_times
|
263
|
+
end
|
264
|
+
|
265
|
+
def get_or_parse_logSet(unparsed_logSet, record, record_hash, is_tag_exists)
|
266
|
+
oci_la_log_set = nil
|
267
|
+
parsed_logSet = nil
|
268
|
+
if !is_valid(unparsed_logSet)
|
269
|
+
return nil
|
270
|
+
end
|
271
|
+
if record_hash.has_key?("oci_la_log_set_ext_regex") && is_valid(record["oci_la_log_set_ext_regex"])
|
272
|
+
parsed_logSet = unparsed_logSet.match(record["oci_la_log_set_ext_regex"])
|
273
|
+
#*******************************************TO-DO**********************************************************
|
274
|
+
# Based on the observed behaviour, below cases are handled. We need to revisit this section.
|
275
|
+
# When trying to apply regex on a String and getting a matched substring, observed couple of scenarios.
|
276
|
+
# For oci_la_log_set_ext_regex value = '.*\\\\/([^\\\\.]{1,40}).*' this returns an array with both input string and matched pattern
|
277
|
+
# For oci_la_log_set_ext_regex value = '[ \\\\w-]+?(?=\\\\.)' this returns an array with only matched pattern
|
278
|
+
# For few cases, String is returned instead of an array.
|
279
|
+
#*******************************************End of TO-DO***************************************************
|
280
|
+
if parsed_logSet!= nil # Based on the regex pattern, match is returning different outputs for same input.
|
281
|
+
if parsed_logSet.is_a? String
|
282
|
+
oci_la_log_set = parsed_logSet.encode("UTF-8") # When matched String is returned instead of an array.
|
283
|
+
elsif parsed_logSet.length > 1 #oci_la_log_set_ext_regex '.*\\\\/([^\\\\.]{1,40}).*' this returns an array with both input string and matched pattern
|
284
|
+
oci_la_log_set = parsed_logSet[1].encode("UTF-8")
|
285
|
+
elsif parsed_logSet.length > 0 # oci_la_log_set_ext_regex '[ \\\\w-]+?(?=\\\\.)' this returns an array with only matched pattern
|
286
|
+
oci_la_log_set = parsed_logSet[0].encode("UTF-8") #Encoding to handle escape characters
|
287
|
+
else
|
288
|
+
oci_la_log_set = nil
|
289
|
+
end
|
290
|
+
else
|
291
|
+
oci_la_log_set = nil
|
292
|
+
if is_tag_exists
|
293
|
+
@@logger.error {"Error occurred while parsing oci_la_log_set : #{unparsed_logSet} with oci_la_log_set_ext_regex : #{record["oci_la_log_set_ext_regex"]}. Default oci_la_log_set will be assigned to all the records with tag : #{record["tag"]}."}
|
294
|
+
else
|
295
|
+
@@logger.error {"Error occurred while parsing oci_la_log_set : #{unparsed_logSet} with oci_la_log_set_ext_regex : #{record["oci_la_log_set_ext_regex"]}. Default oci_la_log_set will be assigned."}
|
296
|
+
end
|
297
|
+
end
|
298
|
+
else
|
299
|
+
oci_la_log_set = unparsed_logSet.force_encoding('UTF-8').encode("UTF-8")
|
300
|
+
end
|
301
|
+
return oci_la_log_set
|
302
|
+
rescue => ex
|
303
|
+
@@logger.error {"Error occurred while parsing oci_la_log_set : #{ex}. Default oci_la_log_set will be assigned."}
|
304
|
+
return nil
|
305
|
+
end
|
306
|
+
|
307
|
+
def is_valid(field)
|
308
|
+
if field.nil? || field.empty? then
|
309
|
+
return false
|
310
|
+
else
|
311
|
+
return true
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def is_valid_log_rotation(log_rotation)
|
316
|
+
if !is_valid(log_rotation)
|
317
|
+
return false
|
318
|
+
end
|
319
|
+
case log_rotation.downcase
|
320
|
+
when "daily"
|
321
|
+
return true
|
322
|
+
when "weekly"
|
323
|
+
return true
|
324
|
+
when "monthly"
|
325
|
+
return true
|
326
|
+
else
|
327
|
+
@@logger_config_error << "Only 'daily'/'weekly'/'monthly' are supported for 'plugin_log_rotation'."
|
328
|
+
return false
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
def is_valid_log_age(param)
|
333
|
+
if !is_valid(param)
|
334
|
+
return false
|
335
|
+
end
|
336
|
+
case param.downcase
|
337
|
+
when "daily"
|
338
|
+
return true
|
339
|
+
when "weekly"
|
340
|
+
return true
|
341
|
+
when "monthly"
|
342
|
+
return true
|
343
|
+
else
|
344
|
+
return false
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def is_valid_log_level(param)
|
349
|
+
if !is_valid(param)
|
350
|
+
return false
|
351
|
+
end
|
352
|
+
case param.upcase
|
353
|
+
when "DEBUG"
|
354
|
+
return true
|
355
|
+
when "INFO"
|
356
|
+
return true
|
357
|
+
when "WARN"
|
358
|
+
return true
|
359
|
+
when "ERROR"
|
360
|
+
return true
|
361
|
+
when "FATAL"
|
362
|
+
return true
|
363
|
+
when "UNKNOWN"
|
364
|
+
return true
|
365
|
+
else
|
366
|
+
return false
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
def logger_set_level(param)
|
371
|
+
# DEBUG < INFO < WARN < ERROR < FATAL < UNKNOWN
|
372
|
+
case @plugin_log_level.upcase
|
373
|
+
when "DEBUG"
|
374
|
+
@@logger.level = Logger::DEBUG
|
375
|
+
when "INFO"
|
376
|
+
@@logger.level = Logger::INFO
|
377
|
+
when "WARN"
|
378
|
+
@@logger.level = Logger::WARN
|
379
|
+
when "ERROR"
|
380
|
+
@@logger.level = Logger::ERROR
|
381
|
+
when "FATAL"
|
382
|
+
@@logger.level = Logger::FATAL
|
383
|
+
when "UNKNOWN"
|
384
|
+
@@logger.level = Logger::UNKNOWN
|
385
|
+
end
|
386
|
+
end
|
387
|
+
|
388
|
+
def is_number(field)
|
389
|
+
true if Integer(field) rescue false
|
390
|
+
end
|
391
|
+
|
392
|
+
def is_valid_log_size(log_size)
|
393
|
+
if log_size != nil
|
394
|
+
case log_size.to_s
|
395
|
+
when /([0-9]+)k/i
|
396
|
+
log_size = $~[1].to_i * 1024
|
397
|
+
when /([0-9]+)m/i
|
398
|
+
log_size = $~[1].to_i * (1024 ** 2)
|
399
|
+
when /([0-9]+)g/i
|
400
|
+
log_size = $~[1].to_i * (1024 ** 3)
|
401
|
+
else
|
402
|
+
@@logger_config_errors << "plugin_log_file_size must be greater than 1KB."
|
403
|
+
return false
|
404
|
+
end
|
405
|
+
@@validated_log_size = log_size
|
406
|
+
return true
|
407
|
+
else
|
408
|
+
return false
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
def is_valid_number_of_logs(number_of_logs)
|
413
|
+
if !is_number(number_of_logs) || number_of_logs < 1
|
414
|
+
@@logger_config_errors << "plugin_log_file_count must be greater than zero"
|
415
|
+
return false
|
416
|
+
end
|
417
|
+
return true
|
418
|
+
end
|
419
|
+
|
420
|
+
def get_valid_metadata(oci_la_metadata)
|
421
|
+
if oci_la_metadata != nil
|
422
|
+
if oci_la_metadata.is_a?(Hash)
|
423
|
+
valid_metadata = Hash.new
|
424
|
+
invalid_keys = []
|
425
|
+
oci_la_metadata.each do |key, value|
|
426
|
+
if value != nil && !value.is_a?(Hash) && !value.is_a?(Array)
|
427
|
+
if key != nil && !key.is_a?(Hash) && !key.is_a?(Array)
|
428
|
+
valid_metadata[key] = value
|
429
|
+
else
|
430
|
+
invalid_keys << key
|
431
|
+
end
|
432
|
+
else
|
433
|
+
invalid_keys << key
|
434
|
+
end
|
435
|
+
end
|
436
|
+
if invalid_keys.length > 0
|
437
|
+
@@logger.warn {"Skipping the following oci_la_metadata/oci_la_global_metadata keys #{invalid_keys.compact.reject(&:empty?).join(',')} as the corresponding values are in invalid format."}
|
438
|
+
end
|
439
|
+
if valid_metadata.length > 0
|
440
|
+
return valid_metadata
|
441
|
+
else
|
442
|
+
return nil
|
443
|
+
end
|
444
|
+
else
|
445
|
+
@@logger.warn {"Ignoring 'oci_la_metadata'/'oci_la_global_metadata' provided in the record_transformer filter as only key-value pairs are supported."}
|
446
|
+
return nil
|
447
|
+
end
|
448
|
+
else
|
449
|
+
return nil
|
450
|
+
end
|
451
|
+
end
|
452
|
+
|
453
|
+
def mandatory_field_validator
|
454
|
+
begin
|
455
|
+
if !is_valid(@namespace)
|
456
|
+
return false,'namespace'
|
457
|
+
elsif !is_valid(@config_file_location) && @auth_type == 'ConfigFile'
|
458
|
+
return false,'config_file_location'
|
459
|
+
elsif !is_valid(@profile_name) && @auth_type == 'ConfigFile'
|
460
|
+
return false,'profile_name'
|
461
|
+
else
|
462
|
+
return true,nil
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
466
|
+
|
467
|
+
def is_valid_record(record_hash,record)
|
468
|
+
begin
|
469
|
+
if !record_hash.has_key?("message")
|
470
|
+
if record_hash.has_key?("tag")
|
471
|
+
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}. 'message' field is not present in the record."}
|
472
|
+
else
|
473
|
+
@@logger.info {"InvalidRecord: #{record}"}
|
474
|
+
@@logger.warn {"Invalid record. 'message' field is not present in the record."}
|
475
|
+
end
|
476
|
+
return false
|
477
|
+
elsif !record_hash.has_key?("oci_la_log_group_id") || !is_valid(record["oci_la_log_group_id"])
|
478
|
+
if record_hash.has_key?("tag")
|
479
|
+
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}.'oci_la_log_group_id' must not be empty.
|
480
|
+
Skipping all the records associated with the tag"}
|
481
|
+
else
|
482
|
+
@@logger.warn {"Invalid record.'oci_la_log_group_id' must not be empty"}
|
483
|
+
end
|
484
|
+
return false
|
485
|
+
elsif !record_hash.has_key?("oci_la_log_source_name") || !is_valid(record["oci_la_log_source_name"])
|
486
|
+
if record_hash.has_key?("tag")
|
487
|
+
@@logger.warn {"Invalid records associated with tag : #{record["tag"]}.'oci_la_log_source_name' must not be empty.
|
488
|
+
Skipping all the records associated with the tag"}
|
489
|
+
else
|
490
|
+
@@logger.warn {"Invalid record.'oci_la_log_source_name' must not be empty"}
|
491
|
+
end
|
492
|
+
return false
|
493
|
+
else
|
494
|
+
return true
|
495
|
+
end
|
496
|
+
end
|
497
|
+
end
|
498
|
+
|
499
|
+
def flatten(kubernetes_metadata)
|
500
|
+
kubernetes_metadata.each_with_object({}) do |(key, value), hash|
|
501
|
+
hash[key] = value
|
502
|
+
if value.is_a? Hash
|
503
|
+
flatten(value).map do |hash_key, hash_value|
|
504
|
+
hash["#{key}.#{hash_key}"] = hash_value
|
505
|
+
end
|
506
|
+
end
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
510
|
+
def get_kubernetes_metadata(oci_la_metadata,record)
|
511
|
+
if oci_la_metadata == nil
|
512
|
+
oci_la_metadata = {}
|
513
|
+
end
|
514
|
+
kubernetes_metadata = flatten(record["kubernetes"])
|
515
|
+
kubernetes_metadata.each do |key, value|
|
516
|
+
if kubernetes_metadata_keys_mapping.has_key?(key)
|
517
|
+
if !is_valid(oci_la_metadata[kubernetes_metadata_keys_mapping[key]])
|
518
|
+
oci_la_metadata[kubernetes_metadata_keys_mapping[key]] = json_message_handler(value)
|
519
|
+
end
|
520
|
+
end
|
521
|
+
end
|
522
|
+
return oci_la_metadata
|
523
|
+
rescue => ex
|
524
|
+
@@logger.error {"Error occurred while getting kubernetes oci_la_metadata:
|
525
|
+
error message: #{ex}"}
|
526
|
+
return oci_la_metadata
|
527
|
+
end
|
528
|
+
|
529
|
+
def json_message_handler(message)
|
530
|
+
if message.is_a?(Hash)
|
531
|
+
return JSON.generate(message)
|
532
|
+
else
|
533
|
+
return message
|
534
|
+
end
|
535
|
+
rescue => ex
|
536
|
+
return message
|
537
|
+
end
|
538
|
+
|
539
|
+
def group_by_logGroupId(chunk)
|
540
|
+
begin
|
541
|
+
current = Time.now
|
542
|
+
current_f, current_s = current.to_f, current.strftime("%Y%m%dT%H%M%S%9NZ")
|
543
|
+
records = []
|
544
|
+
count = 0
|
545
|
+
|
546
|
+
invalid_tag_set = Set.new
|
547
|
+
incoming_records_per_tag = Hash.new
|
548
|
+
invalid_records_per_tag = Hash.new
|
549
|
+
tags_per_logGroupId = Hash.new
|
550
|
+
tag_logSet_map = Hash.new
|
551
|
+
tag_metadata_map = Hash.new
|
552
|
+
|
553
|
+
chunk.each do |time, record|
|
554
|
+
|
555
|
+
if !record.nil?
|
556
|
+
record_hash = record.keys.map {|x| [x,true]}.to_h
|
557
|
+
is_tag_exists = false
|
558
|
+
if record_hash.has_key?("tag") && is_valid(record["tag"])
|
559
|
+
is_tag_exists = true
|
560
|
+
end
|
561
|
+
|
562
|
+
if is_tag_exists && incoming_records_per_tag.has_key?(record["tag"])
|
563
|
+
incoming_records_per_tag[record["tag"]] += 1
|
564
|
+
elsif is_tag_exists
|
565
|
+
incoming_records_per_tag[record["tag"]] = 1
|
566
|
+
end
|
567
|
+
#For any given tag, if one record fails (mandatory fields validation) then all the records from that source will be ignored
|
568
|
+
if is_tag_exists && invalid_tag_set.include?(record["tag"])
|
569
|
+
invalid_records_per_tag[record["tag"]] += 1
|
570
|
+
next #This tag is already present in the invalid_tag_set, so ignoring the message.
|
571
|
+
end
|
572
|
+
#Setting tag/default value for oci_la_log_path, when not provided in config file.
|
573
|
+
if !record_hash.has_key?("oci_la_log_path") || !is_valid(record["oci_la_log_path"])
|
574
|
+
if is_tag_exists
|
575
|
+
record["oci_la_log_path"] = record["tag"]
|
576
|
+
else
|
577
|
+
record["oci_la_log_path"] = 'UNDEFINED'
|
578
|
+
end
|
579
|
+
end
|
580
|
+
|
581
|
+
#Extracting oci_la_log_set when oci_la_log_set_key and oci_la_log_set_ext_regex is provided.
|
582
|
+
#1) oci_la_log_set param is not provided in config file and above logic not executed.
|
583
|
+
#2) Valid oci_la_log_set_key + No oci_la_log_set_ext_regex
|
584
|
+
#a) Valid key available in record with oci_la_log_set_key corresponding value (oci_la_log_set_key is a key in config file) --> oci_la_log_set
|
585
|
+
#b) No Valid key available in record with oci_la_log_set_key corresponding value --> nil
|
586
|
+
#3) Valid key available in record with oci_la_log_set_key corresponding value + Valid oci_la_log_set_ext_regex
|
587
|
+
#a) Parse success --> parsed oci_la_log_set
|
588
|
+
#b) Parse failure --> nil (as oci_la_log_set value)
|
589
|
+
#4) No oci_la_log_set_key --> do nothing --> nil
|
590
|
+
|
591
|
+
#Extracting oci_la_log_set when oci_la_log_set and oci_la_log_set_ext_regex is provided.
|
592
|
+
#1) Valid oci_la_log_set + No oci_la_log_set_ext_regex --> oci_la_log_set
|
593
|
+
#2) Valid oci_la_log_set + Valid oci_la_log_set_ext_regex
|
594
|
+
#a) Parse success --> parsed oci_la_log_set
|
595
|
+
#b) Parse failure --> nil (as oci_la_log_set value)
|
596
|
+
#3) No oci_la_log_set --> do nothing --> nil
|
597
|
+
|
598
|
+
unparsed_logSet = nil
|
599
|
+
processed_logSet = nil
|
600
|
+
if is_tag_exists && tag_logSet_map.has_key?(record["tag"])
|
601
|
+
record["oci_la_log_set"] = tag_logSet_map[record["tag"]]
|
602
|
+
else
|
603
|
+
if record_hash.has_key?("oci_la_log_set_key")
|
604
|
+
if is_valid(record["oci_la_log_set_key"]) && record_hash.has_key?(record["oci_la_log_set_key"])
|
605
|
+
if is_valid(record[record["oci_la_log_set_key"]])
|
606
|
+
unparsed_logSet = record[record["oci_la_log_set_key"]]
|
607
|
+
processed_logSet = get_or_parse_logSet(unparsed_logSet,record, record_hash,is_tag_exists)
|
608
|
+
end
|
609
|
+
end
|
610
|
+
end
|
611
|
+
if !is_valid(processed_logSet) && record_hash.has_key?("oci_la_log_set")
|
612
|
+
if is_valid(record["oci_la_log_set"])
|
613
|
+
unparsed_logSet = record["oci_la_log_set"]
|
614
|
+
processed_logSet = get_or_parse_logSet(unparsed_logSet,record, record_hash,is_tag_exists)
|
615
|
+
end
|
616
|
+
end
|
617
|
+
record["oci_la_log_set"] = processed_logSet
|
618
|
+
tag_logSet_map[record["tag"]] = processed_logSet
|
619
|
+
end
|
620
|
+
|
621
|
+
unless is_valid_record(record_hash,record)
|
622
|
+
if is_tag_exists
|
623
|
+
invalid_tag_set.add(record["tag"])
|
624
|
+
invalid_records_per_tag[record["tag"]] = 1
|
625
|
+
end
|
626
|
+
next
|
627
|
+
end
|
628
|
+
#This will check for null or empty messages and only that record will be ignored.
|
629
|
+
if !is_valid(record["message"])
|
630
|
+
if is_tag_exists
|
631
|
+
@@logger.warn {"'message' field has empty value, Skipping records associated with tag : #{record["tag"]}."}
|
632
|
+
if invalid_records_per_tag.has_key?(record["tag"])
|
633
|
+
invalid_records_per_tag[record["tag"]] += 1
|
634
|
+
else
|
635
|
+
invalid_records_per_tag[record["tag"]] = 1
|
636
|
+
end
|
637
|
+
else
|
638
|
+
@@logger.warn {"'message' field has empty value, Skipping record."}
|
639
|
+
end
|
640
|
+
next
|
641
|
+
else
|
642
|
+
record["message"] = json_message_handler(record["message"])
|
643
|
+
end
|
644
|
+
|
645
|
+
if record_hash.has_key?("kubernetes")
|
646
|
+
record["oci_la_metadata"] = get_kubernetes_metadata(record["oci_la_metadata"],record)
|
647
|
+
end
|
648
|
+
|
649
|
+
if tag_metadata_map.has_key?(record["tag"])
|
650
|
+
record["oci_la_metadata"] = tag_metadata_map[record["tag"]]
|
651
|
+
else
|
652
|
+
if record_hash.has_key?("oci_la_metadata")
|
653
|
+
record["oci_la_metadata"] = get_valid_metadata(record["oci_la_metadata"])
|
654
|
+
tags_per_logGroupId[record["tag"]] = record["oci_la_metadata"]
|
655
|
+
else
|
656
|
+
tags_per_logGroupId[record["tag"]] = nil
|
657
|
+
end
|
658
|
+
end
|
659
|
+
|
660
|
+
if is_tag_exists
|
661
|
+
if tags_per_logGroupId.has_key?(record["oci_la_log_group_id"])
|
662
|
+
if !tags_per_logGroupId[record["oci_la_log_group_id"]].include?(record["tag"])
|
663
|
+
tags_per_logGroupId[record["oci_la_log_group_id"]] += ", "+record["tag"]
|
664
|
+
end
|
665
|
+
else
|
666
|
+
tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"]
|
667
|
+
end
|
668
|
+
end
|
669
|
+
|
670
|
+
records << record
|
671
|
+
else
|
672
|
+
@@logger.trace {"Record is nil, ignoring the record"}
|
673
|
+
end
|
674
|
+
end
|
675
|
+
@@logger.debug {"records.length:#{records.length}"}
|
676
|
+
lrpes_for_logGroupId = {}
|
677
|
+
records.group_by{|record|
|
678
|
+
oci_la_log_group_id = record['oci_la_log_group_id']
|
679
|
+
(oci_la_log_group_id)
|
680
|
+
}.map {|oci_la_log_group_id, records_per_logGroupId|
|
681
|
+
lrpes_for_logGroupId[oci_la_log_group_id] = records_per_logGroupId
|
682
|
+
}
|
683
|
+
rescue => ex
|
684
|
+
@@logger.error {"Error occurred while grouping records by oci_la_log_group_id:#{ex.inspect}"}
|
685
|
+
end
|
686
|
+
return incoming_records_per_tag,invalid_records_per_tag,tags_per_logGroupId,lrpes_for_logGroupId
|
687
|
+
end
|
688
|
+
# main entry point for FluentD's flush_threads, which get invoked
|
689
|
+
# when a chunk is ready for flushing (see chunk limits and flush_intervals)
|
690
|
+
def write(chunk)
|
691
|
+
@@logger.info {"Received new chunk, started processing ..."}
|
692
|
+
metrics = Hash.new
|
693
|
+
metrics["count"] = 0
|
694
|
+
metrics["event"] = "zipping"
|
695
|
+
metrics["bytes_in"] = chunk.bytesize
|
696
|
+
metrics['records_dropped'] = 0
|
697
|
+
begin
|
698
|
+
# 1) Create an in-memory zipfile for the given FluentD chunk
|
699
|
+
# 2) Synchronization has been removed. See EMCLAS-28675
|
700
|
+
|
701
|
+
begin
|
702
|
+
lrpes_for_logGroupId = {}
|
703
|
+
incoming_records_per_tag,invalid_records_per_tag,tags_per_logGroupId,lrpes_for_logGroupId = group_by_logGroupId(chunk)
|
704
|
+
valid_message_per_tag = Hash.new
|
705
|
+
incoming_records_per_tag.each do |key,value|
|
706
|
+
dropped_messages = (invalid_records_per_tag.has_key?(key)) ? invalid_records_per_tag[key].to_i : 0
|
707
|
+
valid_messages = value.to_i - dropped_messages
|
708
|
+
valid_message_per_tag[key] = valid_messages
|
709
|
+
if dropped_messages > 0
|
710
|
+
@@logger.info {"Messages: #{value.to_i} \t Valid: #{valid_messages} \t Invalid: #{dropped_messages} \t tag:#{key}"}
|
711
|
+
end
|
712
|
+
@@logger.debug {"Messages: #{value.to_i} \t Valid: #{valid_messages} \t Invalid: #{dropped_messages} \t tag:#{key}"}
|
713
|
+
end
|
714
|
+
|
715
|
+
if lrpes_for_logGroupId != nil && lrpes_for_logGroupId.length > 0
|
716
|
+
lrpes_for_logGroupId.each do |oci_la_log_group_id,records_per_logGroupId|
|
717
|
+
begin
|
718
|
+
tags = tags_per_logGroupId.key(oci_la_log_group_id)
|
719
|
+
@@logger.info {"Generating payload with #{records_per_logGroupId.length} records for oci_la_log_group_id: #{oci_la_log_group_id}"}
|
720
|
+
zippedstream = nil
|
721
|
+
oci_la_log_set = nil
|
722
|
+
logSets_per_logGroupId_map = Hash.new
|
723
|
+
# Only MAX_FILES_PER_ZIP (100) files are allowed, which will be grouped and zipped.
|
724
|
+
# Due to MAX_FILES_PER_ZIP constraint, for a oci_la_log_group_id, we can get more than one zip file and those many api calls will be made.
|
725
|
+
logSets_per_logGroupId_map, oci_la_global_metadata = get_logSets_map_per_logGroupId(oci_la_log_group_id,records_per_logGroupId)
|
726
|
+
if logSets_per_logGroupId_map != nil
|
727
|
+
logSets_per_logGroupId_map.each do |file_count,records_per_logSet_map|
|
728
|
+
zippedstream,number_of_records = get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map)
|
729
|
+
if zippedstream != nil
|
730
|
+
zippedstream.rewind #reposition buffer pointer to the beginning
|
731
|
+
upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream)
|
732
|
+
end
|
733
|
+
end
|
734
|
+
end
|
735
|
+
ensure
|
736
|
+
zippedstream&.close
|
737
|
+
|
738
|
+
end
|
739
|
+
end
|
740
|
+
end
|
741
|
+
end
|
742
|
+
end
|
743
|
+
end
|
744
|
+
|
745
|
+
# Each oci_la_log_set will correspond to a separate file in the zip
|
746
|
+
# Only MAX_FILES_PER_ZIP files are allowed per zip.
|
747
|
+
# Here we are grouping logSets so that if file_count reaches MAX_FILES_PER_ZIP, these files will be considered for a separate zip file.
|
748
|
+
def get_logSets_map_per_logGroupId(oci_la_log_group_id,records_per_logGroupId)
|
749
|
+
file_count = 0
|
750
|
+
oci_la_global_metadata = nil
|
751
|
+
is_oci_la_global_metadata_assigned = false
|
752
|
+
oci_la_log_set = nil
|
753
|
+
records_per_logSet_map = Hash.new
|
754
|
+
logSets_per_logGroupId_map = Hash.new
|
755
|
+
|
756
|
+
records_per_logGroupId.group_by { |record|
|
757
|
+
if !is_oci_la_global_metadata_assigned
|
758
|
+
record_hash = record.keys.map {|x| [x,true]}.to_h
|
759
|
+
if record_hash.has_key?("oci_la_global_metadata")
|
760
|
+
oci_la_global_metadata = record['oci_la_global_metadata']
|
761
|
+
end
|
762
|
+
is_oci_la_global_metadata_assigned = true
|
763
|
+
end
|
764
|
+
oci_la_log_set = record['oci_la_log_set']
|
765
|
+
(oci_la_log_set)
|
766
|
+
}.map { |oci_la_log_set, records_per_logSet|
|
767
|
+
if file_count % OutOracleOCILogAnalytics::MAX_FILES_PER_ZIP == 0
|
768
|
+
records_per_logSet_map = Hash.new
|
769
|
+
end
|
770
|
+
records_per_logSet_map[oci_la_log_set] = records_per_logSet
|
771
|
+
file_count += 1
|
772
|
+
if file_count % OutOracleOCILogAnalytics::MAX_FILES_PER_ZIP == 0
|
773
|
+
logSets_per_logGroupId_map[file_count] = records_per_logSet_map
|
774
|
+
end
|
775
|
+
}
|
776
|
+
logSets_per_logGroupId_map[file_count] = records_per_logSet_map
|
777
|
+
return logSets_per_logGroupId_map,oci_la_global_metadata
|
778
|
+
rescue => exc
|
779
|
+
@@logger.error {"Error in mapping records to oci_la_log_set.
|
780
|
+
oci_la_log_group_id: #{oci_la_log_group_id},
|
781
|
+
error message:#{exc}"}
|
782
|
+
end
|
783
|
+
|
784
|
+
# takes a fluentD chunk and converts it to an in-memory zipfile, populating metrics hash provided
|
785
|
+
# Any exception raised is passed into the metrics hash, to be re-thrown from write()
|
786
|
+
def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map)
|
787
|
+
begin
|
788
|
+
current, = Time.now
|
789
|
+
current_f, current_s = current.to_f, current.strftime("%Y%m%dT%H%M%S%9NZ")
|
790
|
+
number_of_records = 0
|
791
|
+
noOfFilesGenerated = 0
|
792
|
+
zippedstream = Zip::OutputStream.write_buffer { |zos|
|
793
|
+
records_per_logSet_map.each do |oci_la_log_set,records_per_logSet|
|
794
|
+
lrpes_for_logEvents = records_per_logSet.group_by { |record| [
|
795
|
+
record['oci_la_metadata'],
|
796
|
+
record['oci_la_entity_id'],
|
797
|
+
record['oci_la_entity_type'],
|
798
|
+
record['oci_la_log_source_name'] ,
|
799
|
+
record['oci_la_log_path']
|
800
|
+
]}.map { |lrpe_key, records_per_lrpe|
|
801
|
+
number_of_records += records_per_lrpe.length
|
802
|
+
LogEvents.new(lrpe_key, records_per_lrpe)
|
803
|
+
}
|
804
|
+
noOfFilesGenerated = noOfFilesGenerated +1
|
805
|
+
if is_valid(oci_la_log_set) then
|
806
|
+
nextEntry = oci_la_log_group_id+ "_#{current_s}" +"_"+ noOfFilesGenerated.to_s + "_logSet=" + oci_la_log_set + ".json" #oci_la_log_group_id + ".json"
|
807
|
+
else
|
808
|
+
nextEntry = oci_la_log_group_id + "_#{current_s}" +"_"+ noOfFilesGenerated.to_s + ".json"
|
809
|
+
end
|
810
|
+
@@logger.debug {"Added entry #{nextEntry} for oci_la_log_set #{oci_la_log_set} into the zip."}
|
811
|
+
zos.put_next_entry(nextEntry)
|
812
|
+
logEventsJsonFinal = LogEventsJson.new(oci_la_global_metadata,lrpes_for_logEvents)
|
813
|
+
zos.write logEventsJsonFinal.to_hash.to_json
|
814
|
+
end
|
815
|
+
}
|
816
|
+
zippedstream.rewind
|
817
|
+
if @dump_zip_file
|
818
|
+
save_zip_to_local(oci_la_log_group_id,zippedstream,current_s)
|
819
|
+
end
|
820
|
+
#zippedstream.rewind if records.length > 0 #reposition buffer pointer to the beginning
|
821
|
+
rescue => exc
|
822
|
+
@@logger.error {"Error in generating payload.
|
823
|
+
oci_la_log_group_id: #{oci_la_log_group_id},
|
824
|
+
error message:#{exc}"}
|
825
|
+
end
|
826
|
+
return zippedstream,number_of_records
|
827
|
+
end
|
828
|
+
|
829
|
+
def save_zip_to_local(oci_la_log_group_id, zippedstream, current_s)
|
830
|
+
begin
|
831
|
+
fileName = oci_la_log_group_id+"_"+current_s+'.zip'
|
832
|
+
fileLocation = @zip_file_location+fileName
|
833
|
+
file = File.open(fileLocation, "w")
|
834
|
+
file.write(zippedstream.sysread)
|
835
|
+
rescue => ex
|
836
|
+
@@logger.error {"Error occurred while saving zip file.
|
837
|
+
oci_la_log_group_id: oci_la_log_group_id,
|
838
|
+
fileLocation: @zip_file_location
|
839
|
+
fileName: fileName
|
840
|
+
error message: #{ex}"}
|
841
|
+
ensure
|
842
|
+
file.close unless file.nil?
|
843
|
+
end
|
844
|
+
end
|
845
|
+
|
846
|
+
# upload zipped stream to oci
|
847
|
+
def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream)
|
848
|
+
begin
|
849
|
+
opts = {payload_type: "ZIP"}
|
850
|
+
|
851
|
+
response = @@loganalytics_client.upload_log_events_file(namespace_name=@namespace,
|
852
|
+
logGroupId=oci_la_log_group_id ,
|
853
|
+
uploadLogEventsFileDetails=zippedstream,
|
854
|
+
opts)
|
855
|
+
if !response.nil? && response.status == 200 then
|
856
|
+
headers = response.headers
|
857
|
+
@@logger.info {"The payload has been successfully uploaded to logAnalytics -
|
858
|
+
oci_la_log_group_id: #{oci_la_log_group_id},
|
859
|
+
ConsumedRecords: #{number_of_records},
|
860
|
+
Date: #{headers['date']},
|
861
|
+
Time: #{headers['timecreated']},
|
862
|
+
opc-request-id: #{headers['opc-request-id']},
|
863
|
+
opc-object-id: #{headers['opc-object-id']}"}
|
864
|
+
end
|
865
|
+
rescue OCI::Errors::ServiceError => serviceError
|
866
|
+
case serviceError.status_code
|
867
|
+
when 400
|
868
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Invalid/Incorrect/missing Parameter - opc-request-id:#{serviceError.request_id}"}
|
869
|
+
if plugin_retry_on_4xx
|
870
|
+
raise serviceError
|
871
|
+
end
|
872
|
+
when 401
|
873
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Not Authenticated.
|
874
|
+
opc-request-id:#{serviceError.request_id}
|
875
|
+
message: #{serviceError.message}"}
|
876
|
+
if plugin_retry_on_4xx
|
877
|
+
raise serviceError
|
878
|
+
end
|
879
|
+
when 404
|
880
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Authorization failed for given oci_la_log_group_id against given Tenancy Namespace.
|
881
|
+
oci_la_log_group_id: #{oci_la_log_group_id}
|
882
|
+
Namespace: #{@namespace}
|
883
|
+
opc-request-id: #{serviceError.request_id}
|
884
|
+
message: #{serviceError.message}"}
|
885
|
+
if plugin_retry_on_4xx
|
886
|
+
raise serviceError
|
887
|
+
end
|
888
|
+
when 429
|
889
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Too Many Requests - opc-request-id:#{serviceError.request_id}"}
|
890
|
+
raise serviceError
|
891
|
+
when 500
|
892
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Internal Server Error - opc-request-id:#{serviceError.request_id}"}
|
893
|
+
raise serviceError
|
894
|
+
|
895
|
+
when 502
|
896
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Bad Gateway - opc-request-id:#{serviceError.request_id}"}
|
897
|
+
raise serviceError
|
898
|
+
|
899
|
+
when 503
|
900
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Service unavailable - opc-request-id:#{serviceError.request_id}"}
|
901
|
+
raise serviceError
|
902
|
+
|
903
|
+
when 504
|
904
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. Gateway Timeout - opc-request-id:#{serviceError.request_id}"}
|
905
|
+
raise serviceError
|
906
|
+
|
907
|
+
when 505
|
908
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. HTTP Version Not Supported - opc-request-id:#{serviceError.request_id}"}
|
909
|
+
raise serviceError
|
910
|
+
else
|
911
|
+
@@logger.error {"oci upload exception : Error while uploading the payload #{serviceError.message}"}
|
912
|
+
raise serviceError
|
913
|
+
end
|
914
|
+
rescue => ex
|
915
|
+
@@logger.error {"oci upload exception : Error while uploading the payload. #{ex}"}
|
916
|
+
end
|
917
|
+
end
|
918
|
+
end
|
919
|
+
end
|
metadata
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-oci-logging-analytics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 2.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- OCI Logging Analytics Team
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-01-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '12.0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '12.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: test-unit
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.14.10
|
48
|
+
- - "<"
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: '2'
|
51
|
+
type: :runtime
|
52
|
+
prerelease: false
|
53
|
+
version_requirements: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 0.14.10
|
58
|
+
- - "<"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '2'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rubyzip
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 2.3.2
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 2.3.2
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: oci
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '2.13'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '2.13'
|
89
|
+
description: Fluentd Output plugin to ship logs/events to OCI Logging Analytics.
|
90
|
+
email:
|
91
|
+
- oci_la_plugins@oracle.com
|
92
|
+
executables: []
|
93
|
+
extensions: []
|
94
|
+
extra_rdoc_files: []
|
95
|
+
files:
|
96
|
+
- lib/fluent/dto/logEvents.rb
|
97
|
+
- lib/fluent/dto/logEventsJson.rb
|
98
|
+
- lib/fluent/metrics/metricsLabels.rb
|
99
|
+
- lib/fluent/metrics/prometheusMetrics.rb
|
100
|
+
- lib/fluent/plugin/out_oci-logging-analytics.rb
|
101
|
+
homepage:
|
102
|
+
licenses:
|
103
|
+
- UPL-1.0
|
104
|
+
metadata: {}
|
105
|
+
post_install_message:
|
106
|
+
rdoc_options: []
|
107
|
+
require_paths:
|
108
|
+
- lib
|
109
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ">="
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
requirements: []
|
120
|
+
rubygems_version: 3.0.8
|
121
|
+
signing_key:
|
122
|
+
specification_version: 4
|
123
|
+
summary: Fluentd Output plugin to ship logs/events to OCI Logging Analytics.
|
124
|
+
test_files: []
|