fluent-plugin-scalyr 0.8.10 → 0.8.15

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,47 +17,46 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
- require 'fluent/plugin/output'
20
- require 'fluent/plugin/scalyr-exceptions'
21
- require 'fluent/plugin_helper/compat_parameters'
22
- require 'json'
23
- require 'net/http'
24
- require 'net/https'
25
- require 'rbzip2'
26
- require 'stringio'
27
- require 'zlib'
28
- require 'securerandom'
29
- require 'socket'
30
- require 'thread'
31
-
20
+ require "fluent/plugin/output"
21
+ require "fluent/plugin/scalyr_exceptions"
22
+ require "fluent/plugin/scalyr_utils"
23
+ require "fluent/plugin_helper/compat_parameters"
24
+ require "json"
25
+ require "net/http"
26
+ require "net/https"
27
+ require "rbzip2"
28
+ require "stringio"
29
+ require "zlib"
30
+ require "securerandom"
31
+ require "socket"
32
32
  module Scalyr
33
33
  class ScalyrOut < Fluent::Plugin::Output
34
- Fluent::Plugin.register_output( 'scalyr', self )
34
+ Fluent::Plugin.register_output("scalyr", self)
35
35
  helpers :compat_parameters
36
36
  helpers :event_emitter
37
37
 
38
38
  config_param :api_write_token, :string
39
- config_param :server_attributes, :hash, :default => nil
40
- config_param :use_hostname_for_serverhost, :bool, :default => true
41
- config_param :scalyr_server, :string, :default => "https://agent.scalyr.com/"
42
- config_param :ssl_ca_bundle_path, :string, :default => "/etc/ssl/certs/ca-bundle.crt"
43
- config_param :ssl_verify_peer, :bool, :default => true
44
- config_param :ssl_verify_depth, :integer, :default => 5
45
- config_param :message_field, :string, :default => "message"
46
- config_param :max_request_buffer, :integer, :default => 5500000
47
- config_param :force_message_encoding, :string, :default => nil
48
- config_param :replace_invalid_utf8, :bool, :default => false
49
- config_param :compression_type, :string, :default => nil #Valid options are bz2, deflate or None. Defaults to None.
50
- config_param :compression_level, :integer, :default => 6 #An int containing the compression level of compression to use, from 1-9. Defaults to 6
39
+ config_param :server_attributes, :hash, default: nil
40
+ config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
41
+ config_param :use_hostname_for_serverhost, :bool, default: true
42
+ config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
43
+ config_param :ssl_ca_bundle_path, :string, default: nil
44
+ config_param :ssl_verify_peer, :bool, default: true
45
+ config_param :ssl_verify_depth, :integer, default: 5
46
+ config_param :message_field, :string, default: "message"
47
+ config_param :max_request_buffer, :integer, default: 5_500_000
48
+ config_param :force_message_encoding, :string, default: nil
49
+ config_param :replace_invalid_utf8, :bool, default: false
50
+ config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
51
+ config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
51
52
 
52
53
  config_section :buffer do
53
- config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
54
- config_set_default :retry_max_interval, 30 #wait a maximum of 30 seconds per retry
55
- config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
56
- config_set_default :flush_interval, 5 #default flush interval of 5 seconds
57
- config_set_default :chunk_limit_size, 2500000 #default chunk size of 2.5mb
58
- config_set_default :queue_limit_length, 1024 #default queue size of 1024
54
+ config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
55
+ config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
56
+ config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
57
+ config_set_default :flush_interval, 5 # default flush interval of 5 seconds
58
+ config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
59
+ config_set_default :queue_limit_length, 1024 # default queue size of 1024
59
60
  end
60
61
 
61
62
  # support for version 0.14.0:
@@ -71,46 +72,45 @@ module Scalyr
71
72
  true
72
73
  end
73
74
 
74
- def configure( conf )
75
-
76
- if conf.elements('buffer').empty?
77
- $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
75
+ def configure(conf)
76
+ if conf.elements("buffer").empty?
77
+ $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
78
78
  end
79
79
 
80
- compat_parameters_buffer( conf, default_chunk_key: '' )
80
+ compat_parameters_buffer(conf, default_chunk_key: "")
81
81
 
82
82
  super
83
83
 
84
- if @buffer.chunk_limit_size > 6000000
85
- $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
84
+ if @buffer.chunk_limit_size > 6_000_000
85
+ $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
86
86
  end
87
87
 
88
- if @max_request_buffer > 6000000
89
- $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
88
+ if @max_request_buffer > 6_000_000
89
+ $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
90
90
  end
91
91
 
92
92
  @message_encoding = nil
93
- if @force_message_encoding.to_s != ''
93
+ if @force_message_encoding.to_s != ""
94
94
  begin
95
- @message_encoding = Encoding.find( @force_message_encoding )
95
+ @message_encoding = Encoding.find(@force_message_encoding)
96
96
  $log.debug "Forcing message encoding to '#{@force_message_encoding}'"
97
97
  rescue ArgumentError
98
98
  $log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
99
99
  end
100
100
  end
101
101
 
102
- #evaluate any statements in string value of the server_attributes object
102
+ # evaluate any statements in string value of the server_attributes object
103
103
  if @server_attributes
104
104
  new_attributes = {}
105
105
  @server_attributes.each do |key, value|
106
- if value.is_a?( String )
107
- m = /^\#{(.*)}$/.match( value )
108
- if m
109
- new_attributes[key] = eval( m[1] )
110
- else
111
- new_attributes[key] = value
112
- end
113
- end
106
+ next unless value.is_a?(String)
107
+
108
+ m = /^\#{(.*)}$/.match(value)
109
+ new_attributes[key] = if m
110
+ eval(m[1]) # rubocop:disable Security/Eval
111
+ else
112
+ value
113
+ end
114
114
  end
115
115
  @server_attributes = new_attributes
116
116
  end
@@ -119,143 +119,131 @@ module Scalyr
119
119
  if @use_hostname_for_serverhost
120
120
 
121
121
  # ensure server_attributes is not nil
122
- if @server_attributes.nil?
123
- @server_attributes = {}
124
- end
122
+ @server_attributes = {} if @server_attributes.nil?
125
123
 
126
124
  # only set serverHost if it doesn't currently exist in server_attributes
127
125
  # Note: Use strings rather than symbols for the key, because keys coming
128
126
  # from the config file will be strings
129
- if !@server_attributes.key? 'serverHost'
130
- @server_attributes['serverHost'] = Socket.gethostname
127
+ unless @server_attributes.key? "serverHost"
128
+ @server_attributes["serverHost"] = Socket.gethostname
131
129
  end
132
130
  end
133
131
 
134
- @scalyr_server << '/' unless @scalyr_server.end_with?('/')
132
+ @scalyr_server << "/" unless @scalyr_server.end_with?("/")
135
133
 
136
134
  @add_events_uri = URI @scalyr_server + "addEvents"
137
135
 
138
136
  num_threads = @buffer_config.flush_thread_count
139
137
 
140
- #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
141
- raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
138
+ # forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
139
+ if num_threads > 1
140
+ raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
141
+ end
142
142
  end
143
143
 
144
144
  def start
145
145
  super
146
- #Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
146
+ # Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
147
147
  @session = SecureRandom.uuid
148
148
 
149
- $log.info "Scalyr Fluentd Plugin ID id=#{self.plugin_id()} worker=#{fluentd_worker_id} session=#{@session}"
150
-
149
+ $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
151
150
  end
152
151
 
153
- def format( tag, time, record )
154
- begin
155
-
156
- if time.nil?
157
- time = Fluent::Engine.now
158
- end
159
-
160
- # handle timestamps that are not EventTime types
161
- if time.is_a?( Integer )
162
- time = Fluent::EventTime.new( time )
163
- elsif time.is_a?( Float )
164
- components = time.divmod 1 #get integer and decimal components
165
- sec = components[0].to_i
166
- nsec = (components[1] * 10**9).to_i
167
- time = Fluent::EventTime.new( sec, nsec )
168
- end
152
+ def format(tag, time, record)
153
+ time = Fluent::Engine.now if time.nil?
154
+
155
+ # handle timestamps that are not EventTime types
156
+ if time.is_a?(Integer)
157
+ time = Fluent::EventTime.new(time)
158
+ elsif time.is_a?(Float)
159
+ components = time.divmod 1 # get integer and decimal components
160
+ sec = components[0].to_i
161
+ nsec = (components[1] * 10**9).to_i
162
+ time = Fluent::EventTime.new(sec, nsec)
163
+ end
169
164
 
170
- if @message_field != "message"
171
- if record.key? @message_field
172
- if record.key? "message"
173
- $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten."
174
- end
175
- record["message"] = record[@message_field]
176
- record.delete( @message_field )
165
+ if @message_field != "message"
166
+ if record.key? @message_field
167
+ if record.key? "message"
168
+ $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
177
169
  end
170
+ record["message"] = record[@message_field]
171
+ record.delete(@message_field)
178
172
  end
173
+ end
179
174
 
180
- if @message_encoding and record.key? "message" and record["message"]
181
- if @replace_invalid_utf8 and @message_encoding == Encoding::UTF_8
182
- record["message"] = record["message"].encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
183
- else
184
- record["message"].force_encoding( @message_encoding )
185
- end
175
+ if @message_encoding && record.key?("message") && record["message"]
176
+ if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
177
+ record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
+ else
179
+ record["message"].force_encoding(@message_encoding)
186
180
  end
187
- [tag, time.sec, time.nsec, record].to_msgpack
188
-
189
- rescue JSON::GeneratorError
190
- $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
191
- raise
192
181
  end
182
+ [tag, time.sec, time.nsec, record].to_msgpack
183
+ rescue JSON::GeneratorError
184
+ $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
185
+ raise
193
186
  end
194
187
 
195
- #called by fluentd when a chunk of log messages is ready
196
- def write( chunk )
197
- begin
198
- $log.debug "Size of chunk is: #{chunk.size}"
199
- requests = self.build_add_events_body( chunk )
200
- $log.debug "Chunk split into #{requests.size} request(s)."
201
-
202
- requests.each_with_index { |request, index|
203
- $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
204
- begin
205
- response = self.post_request( @add_events_uri, request[:body] )
206
- self.handle_response( response )
207
- rescue OpenSSL::SSL::SSLError => e
208
- if e.message.include? "certificate verify failed"
209
- $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
210
- end
211
- $log.warn e.message
212
- $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
213
- rescue Scalyr::Client4xxError => e
214
- $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
188
+ # called by fluentd when a chunk of log messages is ready
189
+ def write(chunk)
190
+ $log.debug "Size of chunk is: #{chunk.size}"
191
+ requests = build_add_events_body(chunk)
192
+ $log.debug "Chunk split into #{requests.size} request(s)."
215
193
 
194
+ requests.each_with_index {|request, index|
195
+ $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
196
+ begin
197
+ response = post_request(@add_events_uri, request[:body])
198
+ handle_response(response)
199
+ rescue OpenSSL::SSL::SSLError => e
200
+ if e.message.include? "certificate verify failed"
201
+ $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
216
202
  end
217
- }
218
-
219
- rescue JSON::GeneratorError
220
- $log.warn "Unable to format message due to JSON::GeneratorError."
221
- raise
222
- end
203
+ $log.warn e.message
204
+ $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
205
+ rescue Scalyr::Client4xxError => e
206
+ $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
207
+ end
208
+ }
209
+ rescue JSON::GeneratorError
210
+ $log.warn "Unable to format message due to JSON::GeneratorError."
211
+ raise
223
212
  end
224
213
 
225
-
226
- #explicit function to convert to nanoseconds
227
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
228
- def to_nanos( seconds, nsec )
214
+ # explicit function to convert to nanoseconds
215
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
216
+ def to_nanos(seconds, nsec)
229
217
  (seconds * 10**9) + nsec
230
218
  end
231
219
 
232
- #explicit function to convert to milliseconds
233
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
234
- def to_millis( timestamp )
220
+ # explicit function to convert to milliseconds
221
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
222
+ def to_millis(timestamp)
235
223
  (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
236
224
  end
237
225
 
238
- def post_request( uri, body )
239
-
240
- https = Net::HTTP.new( uri.host, uri.port )
226
+ def post_request(uri, body)
227
+ https = Net::HTTP.new(uri.host, uri.port)
241
228
  https.use_ssl = true
242
229
 
243
- #verify peers to prevent potential MITM attacks
230
+ # verify peers to prevent potential MITM attacks
244
231
  if @ssl_verify_peer
245
- https.ca_file = @ssl_ca_bundle_path
232
+ https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
233
+ https.ssl_version = :TLSv1_2
246
234
  https.verify_mode = OpenSSL::SSL::VERIFY_PEER
247
235
  https.verify_depth = @ssl_verify_depth
248
236
  end
249
237
 
250
- #use compression if enabled
238
+ # use compression if enabled
251
239
  encoding = nil
252
240
 
253
241
  if @compression_type
254
- if @compression_type == 'deflate'
255
- encoding = 'deflate'
242
+ if @compression_type == "deflate"
243
+ encoding = "deflate"
256
244
  body = Zlib::Deflate.deflate(body, @compression_level)
257
- elsif @compression_type == 'bz2'
258
- encoding = 'bz2'
245
+ elsif @compression_type == "bz2"
246
+ encoding = "bz2"
259
247
  io = StringIO.new
260
248
  bz2 = RBzip2.default_adapter::Compressor.new io
261
249
  bz2.write body
@@ -265,164 +253,185 @@ module Scalyr
265
253
  end
266
254
 
267
255
  post = Net::HTTP::Post.new uri.path
268
- post.add_field( 'Content-Type', 'application/json' )
256
+ post.add_field("Content-Type", "application/json")
269
257
 
270
- if @compression_type
271
- post.add_field( 'Content-Encoding', encoding )
272
- end
258
+ post.add_field("Content-Encoding", encoding) if @compression_type
273
259
 
274
260
  post.body = body
275
261
 
276
- https.request( post )
277
-
262
+ https.request(post)
278
263
  end
279
264
 
280
- def handle_response( response )
265
+ def handle_response(response)
281
266
  $log.debug "Response Code: #{response.code}"
282
267
  $log.debug "Response Body: #{response.body}"
283
268
 
284
- response_hash = Hash.new
269
+ response_hash = {}
285
270
 
286
271
  begin
287
- response_hash = JSON.parse( response.body )
288
- rescue
272
+ response_hash = JSON.parse(response.body)
273
+ rescue StandardError
289
274
  response_hash["status"] = "Invalid JSON response from server"
290
275
  end
291
276
 
292
- #make sure the JSON reponse has a "status" field
293
- if !response_hash.key? "status"
277
+ # make sure the JSON reponse has a "status" field
278
+ unless response_hash.key? "status"
294
279
  $log.debug "JSON response does not contain status message"
295
280
  raise Scalyr::ServerError.new "JSON response does not contain status message"
296
281
  end
297
282
 
298
283
  status = response_hash["status"]
299
284
 
300
- #4xx codes are handled separately
285
+ # 4xx codes are handled separately
301
286
  if response.code =~ /^4\d\d/
302
287
  raise Scalyr::Client4xxError.new status
303
288
  else
304
- if status != "success"
289
+ if status != "success" # rubocop:disable Style/IfInsideElse
305
290
  if status =~ /discardBuffer/
306
291
  $log.warn "Received 'discardBuffer' message from server. Buffer dropped."
307
- elsif status =~ %r"/client/"i
292
+ elsif status =~ %r{/client/}i
308
293
  raise Scalyr::ClientError.new status
309
- else #don't check specifically for server, we assume all non-client errors are server errors
294
+ else # don't check specifically for server, we assume all non-client errors are server errors
310
295
  raise Scalyr::ServerError.new status
311
296
  end
312
- elsif !response.code.include? "200" #response code is a string not an int
297
+ elsif !response.code.include? "200" # response code is a string not an int
313
298
  raise Scalyr::ServerError
314
299
  end
315
300
  end
316
-
317
301
  end
318
302
 
319
- def build_add_events_body( chunk )
303
+ def build_add_events_body(chunk)
304
+ # requests
305
+ requests = []
320
306
 
321
- #requests
322
- requests = Array.new
307
+ # set of unique scalyr threads for this chunk
308
+ current_threads = {}
323
309
 
324
- #set of unique scalyr threads for this chunk
325
- current_threads = Hash.new
326
-
327
- #byte count
310
+ # byte count
328
311
  total_bytes = 0
329
312
 
330
- #create a Scalyr event object for each record in the chunk
331
- events = Array.new
332
- chunk.msgpack_each {|(tag, sec, nsec, record)|
333
-
334
- timestamp = self.to_nanos( sec, nsec )
313
+ # create a Scalyr event object for each record in the chunk
314
+ events = []
315
+ chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
316
+ timestamp = to_nanos(sec, nsec)
335
317
 
336
318
  thread_id = tag
337
319
 
338
- #then update the map of threads for this chunk
320
+ # then update the map of threads for this chunk
339
321
  current_threads[tag] = thread_id
340
322
 
341
- #add a logfile field if one doesn't exist
342
- if !record.key? "logfile"
343
- record["logfile"] = "/fluentd/#{tag}"
344
- end
323
+ # add a logfile field if one doesn't exist
324
+ record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
345
325
 
346
- #append to list of events
347
- event = { :thread => thread_id.to_s,
348
- :ts => timestamp,
349
- :attrs => record
350
- }
326
+ # set per-event parser if it is configured
327
+ record["parser"] = @parser unless @parser.nil?
351
328
 
352
- #get json string of event to keep track of how many bytes we are sending
329
+ # append to list of events
330
+ event = {thread: thread_id.to_s,
331
+ ts: timestamp,
332
+ attrs: record}
333
+
334
+ # get json string of event to keep track of how many bytes we are sending
353
335
 
354
336
  begin
355
337
  event_json = event.to_json
356
338
  rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
357
- $log.warn "#{e.class}: #{e.message}"
339
+ $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
358
340
 
359
341
  # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
360
- time = Fluent::EventTime.new( sec, nsec )
342
+ time = Fluent::EventTime.new(sec, nsec)
361
343
  router.emit_error_event(tag, time, record, e)
362
344
 
345
+ # Print attribute values for debugging / troubleshooting purposes
346
+ $log.debug "Event attributes:"
347
+
363
348
  event[:attrs].each do |key, value|
364
- $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
365
- event[:attrs][key] = value.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
349
+ # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
350
+ $log.debug "\t#{key} (#{value.class}): '#{value}'"
366
351
  end
352
+
353
+ # Recursively re-encode and sanitize potentially bad string values
354
+ event[:attrs] = sanitize_and_reencode_value(event[:attrs])
367
355
  event_json = event.to_json
368
356
  end
369
357
 
370
- #generate new request if json size of events in the array exceed maximum request buffer size
358
+ # generate new request if json size of events in the array exceed maximum request buffer size
371
359
  append_event = true
372
360
  if total_bytes + event_json.bytesize > @max_request_buffer
373
- #make sure we always have at least one event
374
- if events.size == 0
375
- events << event
361
+ # the case where a single event causes us to exceed the @max_request_buffer
362
+ if events.empty?
363
+ # if we are able to truncate the content (and append an ellipsis)
364
+ # inside the @message_field we do so here
365
+ if record.key?(@message_field) &&
366
+ record[@message_field].is_a?(String) &&
367
+ record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
368
+ record[@message_field].bytesize >= 3
369
+
370
+ @log.warn "Received a record that cannot fit within max_request_buffer "\
371
+ "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
372
+ "is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
373
+ max_msg_size = @max_request_buffer - event_json.bytesize - 3
374
+ truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
375
+ event[:attrs][@message_field] = truncated_msg
376
+ events << event
377
+
378
+ # otherwise we drop the event and save ourselves hitting a 4XX response from the server
379
+ else
380
+ @log.warn "Received a record that cannot fit within max_request_buffer "\
381
+ "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
382
+ "is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
383
+ "dropping event."
384
+ end
376
385
  append_event = false
377
386
  end
378
- request = self.create_request( events, current_threads )
379
- requests << request
387
+
388
+ unless events.empty?
389
+ request = create_request(events, current_threads)
390
+ requests << request
391
+ end
380
392
 
381
393
  total_bytes = 0
382
- current_threads = Hash.new
383
- events = Array.new
394
+ current_threads = {}
395
+ events = []
384
396
  end
385
397
 
386
- #if we haven't consumed the current event already
387
- #add it to the end of our array and keep track of the json bytesize
398
+ # if we haven't consumed the current event already
399
+ # add it to the end of our array and keep track of the json bytesize
388
400
  if append_event
389
401
  events << event
390
402
  total_bytes += event_json.bytesize
391
403
  end
392
-
393
404
  }
394
405
 
395
- #create a final request with any left over events
396
- request = self.create_request( events, current_threads )
397
- requests << request
406
+ # create a final request with any left over events
407
+ unless events.empty?
408
+ request = create_request(events, current_threads)
409
+ requests << request
410
+ end
398
411
 
412
+ requests
399
413
  end
400
414
 
401
- def create_request( events, current_threads )
402
- #build the scalyr thread objects
403
- threads = Array.new
415
+ def create_request(events, current_threads)
416
+ # build the scalyr thread objects
417
+ threads = []
404
418
  current_threads.each do |tag, id|
405
- threads << { :id => id.to_s,
406
- :name => "Fluentd: #{tag}"
407
- }
419
+ threads << {id: id.to_s,
420
+ name: "Fluentd: #{tag}"}
408
421
  end
409
422
 
410
- current_time = self.to_millis( Fluent::Engine.now )
423
+ current_time = to_millis(Fluent::Engine.now)
411
424
 
412
- body = { :token => @api_write_token,
413
- :client_timestamp => current_time.to_s,
414
- :session => @session,
415
- :events => events,
416
- :threads => threads
417
- }
425
+ body = {token: @api_write_token,
426
+ client_timestamp: current_time.to_s,
427
+ session: @session,
428
+ events: events,
429
+ threads: threads}
418
430
 
419
- #add server_attributes hash if it exists
420
- if @server_attributes
421
- body[:sessionInfo] = @server_attributes
422
- end
431
+ # add server_attributes hash if it exists
432
+ body[:sessionInfo] = @server_attributes if @server_attributes
423
433
 
424
- { :body => body.to_json, :record_count => events.size }
434
+ {body: body.to_json, record_count: events.size}
425
435
  end
426
-
427
436
  end
428
437
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,8 +17,6 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
-
20
20
  module Scalyr
21
21
  class ClientError < StandardError; end
22
22
  class Client4xxError < StandardError; end