fluent-plugin-scalyr 0.8.10 → 0.8.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,47 +17,46 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
- require 'fluent/plugin/output'
20
- require 'fluent/plugin/scalyr-exceptions'
21
- require 'fluent/plugin_helper/compat_parameters'
22
- require 'json'
23
- require 'net/http'
24
- require 'net/https'
25
- require 'rbzip2'
26
- require 'stringio'
27
- require 'zlib'
28
- require 'securerandom'
29
- require 'socket'
30
- require 'thread'
31
-
20
+ require "fluent/plugin/output"
21
+ require "fluent/plugin/scalyr_exceptions"
22
+ require "fluent/plugin/scalyr_utils"
23
+ require "fluent/plugin_helper/compat_parameters"
24
+ require "json"
25
+ require "net/http"
26
+ require "net/https"
27
+ require "rbzip2"
28
+ require "stringio"
29
+ require "zlib"
30
+ require "securerandom"
31
+ require "socket"
32
32
  module Scalyr
33
33
  class ScalyrOut < Fluent::Plugin::Output
34
- Fluent::Plugin.register_output( 'scalyr', self )
34
+ Fluent::Plugin.register_output("scalyr", self)
35
35
  helpers :compat_parameters
36
36
  helpers :event_emitter
37
37
 
38
38
  config_param :api_write_token, :string
39
- config_param :server_attributes, :hash, :default => nil
40
- config_param :use_hostname_for_serverhost, :bool, :default => true
41
- config_param :scalyr_server, :string, :default => "https://agent.scalyr.com/"
42
- config_param :ssl_ca_bundle_path, :string, :default => "/etc/ssl/certs/ca-bundle.crt"
43
- config_param :ssl_verify_peer, :bool, :default => true
44
- config_param :ssl_verify_depth, :integer, :default => 5
45
- config_param :message_field, :string, :default => "message"
46
- config_param :max_request_buffer, :integer, :default => 5500000
47
- config_param :force_message_encoding, :string, :default => nil
48
- config_param :replace_invalid_utf8, :bool, :default => false
49
- config_param :compression_type, :string, :default => nil #Valid options are bz2, deflate or None. Defaults to None.
50
- config_param :compression_level, :integer, :default => 6 #An int containing the compression level of compression to use, from 1-9. Defaults to 6
39
+ config_param :server_attributes, :hash, default: nil
40
+ config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
41
+ config_param :use_hostname_for_serverhost, :bool, default: true
42
+ config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
43
+ config_param :ssl_ca_bundle_path, :string, default: nil
44
+ config_param :ssl_verify_peer, :bool, default: true
45
+ config_param :ssl_verify_depth, :integer, default: 5
46
+ config_param :message_field, :string, default: "message"
47
+ config_param :max_request_buffer, :integer, default: 5_500_000
48
+ config_param :force_message_encoding, :string, default: nil
49
+ config_param :replace_invalid_utf8, :bool, default: false
50
+ config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
51
+ config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
51
52
 
52
53
  config_section :buffer do
53
- config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
54
- config_set_default :retry_max_interval, 30 #wait a maximum of 30 seconds per retry
55
- config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
56
- config_set_default :flush_interval, 5 #default flush interval of 5 seconds
57
- config_set_default :chunk_limit_size, 2500000 #default chunk size of 2.5mb
58
- config_set_default :queue_limit_length, 1024 #default queue size of 1024
54
+ config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
55
+ config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
56
+ config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
57
+ config_set_default :flush_interval, 5 # default flush interval of 5 seconds
58
+ config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
59
+ config_set_default :queue_limit_length, 1024 # default queue size of 1024
59
60
  end
60
61
 
61
62
  # support for version 0.14.0:
@@ -71,46 +72,45 @@ module Scalyr
71
72
  true
72
73
  end
73
74
 
74
- def configure( conf )
75
-
76
- if conf.elements('buffer').empty?
77
- $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
75
+ def configure(conf)
76
+ if conf.elements("buffer").empty?
77
+ $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
78
78
  end
79
79
 
80
- compat_parameters_buffer( conf, default_chunk_key: '' )
80
+ compat_parameters_buffer(conf, default_chunk_key: "")
81
81
 
82
82
  super
83
83
 
84
- if @buffer.chunk_limit_size > 6000000
85
- $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
84
+ if @buffer.chunk_limit_size > 6_000_000
85
+ $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
86
86
  end
87
87
 
88
- if @max_request_buffer > 6000000
89
- $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
88
+ if @max_request_buffer > 6_000_000
89
+ $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
90
90
  end
91
91
 
92
92
  @message_encoding = nil
93
- if @force_message_encoding.to_s != ''
93
+ if @force_message_encoding.to_s != ""
94
94
  begin
95
- @message_encoding = Encoding.find( @force_message_encoding )
95
+ @message_encoding = Encoding.find(@force_message_encoding)
96
96
  $log.debug "Forcing message encoding to '#{@force_message_encoding}'"
97
97
  rescue ArgumentError
98
98
  $log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
99
99
  end
100
100
  end
101
101
 
102
- #evaluate any statements in string value of the server_attributes object
102
+ # evaluate any statements in string value of the server_attributes object
103
103
  if @server_attributes
104
104
  new_attributes = {}
105
105
  @server_attributes.each do |key, value|
106
- if value.is_a?( String )
107
- m = /^\#{(.*)}$/.match( value )
108
- if m
109
- new_attributes[key] = eval( m[1] )
110
- else
111
- new_attributes[key] = value
112
- end
113
- end
106
+ next unless value.is_a?(String)
107
+
108
+ m = /^\#{(.*)}$/.match(value)
109
+ new_attributes[key] = if m
110
+ eval(m[1]) # rubocop:disable Security/Eval
111
+ else
112
+ value
113
+ end
114
114
  end
115
115
  @server_attributes = new_attributes
116
116
  end
@@ -119,143 +119,131 @@ module Scalyr
119
119
  if @use_hostname_for_serverhost
120
120
 
121
121
  # ensure server_attributes is not nil
122
- if @server_attributes.nil?
123
- @server_attributes = {}
124
- end
122
+ @server_attributes = {} if @server_attributes.nil?
125
123
 
126
124
  # only set serverHost if it doesn't currently exist in server_attributes
127
125
  # Note: Use strings rather than symbols for the key, because keys coming
128
126
  # from the config file will be strings
129
- if !@server_attributes.key? 'serverHost'
130
- @server_attributes['serverHost'] = Socket.gethostname
127
+ unless @server_attributes.key? "serverHost"
128
+ @server_attributes["serverHost"] = Socket.gethostname
131
129
  end
132
130
  end
133
131
 
134
- @scalyr_server << '/' unless @scalyr_server.end_with?('/')
132
+ @scalyr_server << "/" unless @scalyr_server.end_with?("/")
135
133
 
136
134
  @add_events_uri = URI @scalyr_server + "addEvents"
137
135
 
138
136
  num_threads = @buffer_config.flush_thread_count
139
137
 
140
- #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
141
- raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
138
+ # forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
139
+ if num_threads > 1
140
+ raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
141
+ end
142
142
  end
143
143
 
144
144
  def start
145
145
  super
146
- #Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
146
+ # Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
147
147
  @session = SecureRandom.uuid
148
148
 
149
- $log.info "Scalyr Fluentd Plugin ID id=#{self.plugin_id()} worker=#{fluentd_worker_id} session=#{@session}"
150
-
149
+ $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
151
150
  end
152
151
 
153
- def format( tag, time, record )
154
- begin
155
-
156
- if time.nil?
157
- time = Fluent::Engine.now
158
- end
159
-
160
- # handle timestamps that are not EventTime types
161
- if time.is_a?( Integer )
162
- time = Fluent::EventTime.new( time )
163
- elsif time.is_a?( Float )
164
- components = time.divmod 1 #get integer and decimal components
165
- sec = components[0].to_i
166
- nsec = (components[1] * 10**9).to_i
167
- time = Fluent::EventTime.new( sec, nsec )
168
- end
152
+ def format(tag, time, record)
153
+ time = Fluent::Engine.now if time.nil?
154
+
155
+ # handle timestamps that are not EventTime types
156
+ if time.is_a?(Integer)
157
+ time = Fluent::EventTime.new(time)
158
+ elsif time.is_a?(Float)
159
+ components = time.divmod 1 # get integer and decimal components
160
+ sec = components[0].to_i
161
+ nsec = (components[1] * 10**9).to_i
162
+ time = Fluent::EventTime.new(sec, nsec)
163
+ end
169
164
 
170
- if @message_field != "message"
171
- if record.key? @message_field
172
- if record.key? "message"
173
- $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten."
174
- end
175
- record["message"] = record[@message_field]
176
- record.delete( @message_field )
165
+ if @message_field != "message"
166
+ if record.key? @message_field
167
+ if record.key? "message"
168
+ $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
177
169
  end
170
+ record["message"] = record[@message_field]
171
+ record.delete(@message_field)
178
172
  end
173
+ end
179
174
 
180
- if @message_encoding and record.key? "message" and record["message"]
181
- if @replace_invalid_utf8 and @message_encoding == Encoding::UTF_8
182
- record["message"] = record["message"].encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
183
- else
184
- record["message"].force_encoding( @message_encoding )
185
- end
175
+ if @message_encoding && record.key?("message") && record["message"]
176
+ if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
177
+ record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
+ else
179
+ record["message"].force_encoding(@message_encoding)
186
180
  end
187
- [tag, time.sec, time.nsec, record].to_msgpack
188
-
189
- rescue JSON::GeneratorError
190
- $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
191
- raise
192
181
  end
182
+ [tag, time.sec, time.nsec, record].to_msgpack
183
+ rescue JSON::GeneratorError
184
+ $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
185
+ raise
193
186
  end
194
187
 
195
- #called by fluentd when a chunk of log messages is ready
196
- def write( chunk )
197
- begin
198
- $log.debug "Size of chunk is: #{chunk.size}"
199
- requests = self.build_add_events_body( chunk )
200
- $log.debug "Chunk split into #{requests.size} request(s)."
201
-
202
- requests.each_with_index { |request, index|
203
- $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
204
- begin
205
- response = self.post_request( @add_events_uri, request[:body] )
206
- self.handle_response( response )
207
- rescue OpenSSL::SSL::SSLError => e
208
- if e.message.include? "certificate verify failed"
209
- $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
210
- end
211
- $log.warn e.message
212
- $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
213
- rescue Scalyr::Client4xxError => e
214
- $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
188
+ # called by fluentd when a chunk of log messages is ready
189
+ def write(chunk)
190
+ $log.debug "Size of chunk is: #{chunk.size}"
191
+ requests = build_add_events_body(chunk)
192
+ $log.debug "Chunk split into #{requests.size} request(s)."
215
193
 
194
+ requests.each_with_index {|request, index|
195
+ $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
196
+ begin
197
+ response = post_request(@add_events_uri, request[:body])
198
+ handle_response(response)
199
+ rescue OpenSSL::SSL::SSLError => e
200
+ if e.message.include? "certificate verify failed"
201
+ $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
216
202
  end
217
- }
218
-
219
- rescue JSON::GeneratorError
220
- $log.warn "Unable to format message due to JSON::GeneratorError."
221
- raise
222
- end
203
+ $log.warn e.message
204
+ $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
205
+ rescue Scalyr::Client4xxError => e
206
+ $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
207
+ end
208
+ }
209
+ rescue JSON::GeneratorError
210
+ $log.warn "Unable to format message due to JSON::GeneratorError."
211
+ raise
223
212
  end
224
213
 
225
-
226
- #explicit function to convert to nanoseconds
227
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
228
- def to_nanos( seconds, nsec )
214
+ # explicit function to convert to nanoseconds
215
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
216
+ def to_nanos(seconds, nsec)
229
217
  (seconds * 10**9) + nsec
230
218
  end
231
219
 
232
- #explicit function to convert to milliseconds
233
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
234
- def to_millis( timestamp )
220
+ # explicit function to convert to milliseconds
221
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
222
+ def to_millis(timestamp)
235
223
  (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
236
224
  end
237
225
 
238
- def post_request( uri, body )
239
-
240
- https = Net::HTTP.new( uri.host, uri.port )
226
+ def post_request(uri, body)
227
+ https = Net::HTTP.new(uri.host, uri.port)
241
228
  https.use_ssl = true
242
229
 
243
- #verify peers to prevent potential MITM attacks
230
+ # verify peers to prevent potential MITM attacks
244
231
  if @ssl_verify_peer
245
- https.ca_file = @ssl_ca_bundle_path
232
+ https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
233
+ https.ssl_version = :TLSv1_2
246
234
  https.verify_mode = OpenSSL::SSL::VERIFY_PEER
247
235
  https.verify_depth = @ssl_verify_depth
248
236
  end
249
237
 
250
- #use compression if enabled
238
+ # use compression if enabled
251
239
  encoding = nil
252
240
 
253
241
  if @compression_type
254
- if @compression_type == 'deflate'
255
- encoding = 'deflate'
242
+ if @compression_type == "deflate"
243
+ encoding = "deflate"
256
244
  body = Zlib::Deflate.deflate(body, @compression_level)
257
- elsif @compression_type == 'bz2'
258
- encoding = 'bz2'
245
+ elsif @compression_type == "bz2"
246
+ encoding = "bz2"
259
247
  io = StringIO.new
260
248
  bz2 = RBzip2.default_adapter::Compressor.new io
261
249
  bz2.write body
@@ -265,164 +253,185 @@ module Scalyr
265
253
  end
266
254
 
267
255
  post = Net::HTTP::Post.new uri.path
268
- post.add_field( 'Content-Type', 'application/json' )
256
+ post.add_field("Content-Type", "application/json")
269
257
 
270
- if @compression_type
271
- post.add_field( 'Content-Encoding', encoding )
272
- end
258
+ post.add_field("Content-Encoding", encoding) if @compression_type
273
259
 
274
260
  post.body = body
275
261
 
276
- https.request( post )
277
-
262
+ https.request(post)
278
263
  end
279
264
 
280
- def handle_response( response )
265
+ def handle_response(response)
281
266
  $log.debug "Response Code: #{response.code}"
282
267
  $log.debug "Response Body: #{response.body}"
283
268
 
284
- response_hash = Hash.new
269
+ response_hash = {}
285
270
 
286
271
  begin
287
- response_hash = JSON.parse( response.body )
288
- rescue
272
+ response_hash = JSON.parse(response.body)
273
+ rescue StandardError
289
274
  response_hash["status"] = "Invalid JSON response from server"
290
275
  end
291
276
 
292
- #make sure the JSON reponse has a "status" field
293
- if !response_hash.key? "status"
277
+ # make sure the JSON reponse has a "status" field
278
+ unless response_hash.key? "status"
294
279
  $log.debug "JSON response does not contain status message"
295
280
  raise Scalyr::ServerError.new "JSON response does not contain status message"
296
281
  end
297
282
 
298
283
  status = response_hash["status"]
299
284
 
300
- #4xx codes are handled separately
285
+ # 4xx codes are handled separately
301
286
  if response.code =~ /^4\d\d/
302
287
  raise Scalyr::Client4xxError.new status
303
288
  else
304
- if status != "success"
289
+ if status != "success" # rubocop:disable Style/IfInsideElse
305
290
  if status =~ /discardBuffer/
306
291
  $log.warn "Received 'discardBuffer' message from server. Buffer dropped."
307
- elsif status =~ %r"/client/"i
292
+ elsif status =~ %r{/client/}i
308
293
  raise Scalyr::ClientError.new status
309
- else #don't check specifically for server, we assume all non-client errors are server errors
294
+ else # don't check specifically for server, we assume all non-client errors are server errors
310
295
  raise Scalyr::ServerError.new status
311
296
  end
312
- elsif !response.code.include? "200" #response code is a string not an int
297
+ elsif !response.code.include? "200" # response code is a string not an int
313
298
  raise Scalyr::ServerError
314
299
  end
315
300
  end
316
-
317
301
  end
318
302
 
319
- def build_add_events_body( chunk )
303
+ def build_add_events_body(chunk)
304
+ # requests
305
+ requests = []
320
306
 
321
- #requests
322
- requests = Array.new
307
+ # set of unique scalyr threads for this chunk
308
+ current_threads = {}
323
309
 
324
- #set of unique scalyr threads for this chunk
325
- current_threads = Hash.new
326
-
327
- #byte count
310
+ # byte count
328
311
  total_bytes = 0
329
312
 
330
- #create a Scalyr event object for each record in the chunk
331
- events = Array.new
332
- chunk.msgpack_each {|(tag, sec, nsec, record)|
333
-
334
- timestamp = self.to_nanos( sec, nsec )
313
+ # create a Scalyr event object for each record in the chunk
314
+ events = []
315
+ chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
316
+ timestamp = to_nanos(sec, nsec)
335
317
 
336
318
  thread_id = tag
337
319
 
338
- #then update the map of threads for this chunk
320
+ # then update the map of threads for this chunk
339
321
  current_threads[tag] = thread_id
340
322
 
341
- #add a logfile field if one doesn't exist
342
- if !record.key? "logfile"
343
- record["logfile"] = "/fluentd/#{tag}"
344
- end
323
+ # add a logfile field if one doesn't exist
324
+ record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
345
325
 
346
- #append to list of events
347
- event = { :thread => thread_id.to_s,
348
- :ts => timestamp,
349
- :attrs => record
350
- }
326
+ # set per-event parser if it is configured
327
+ record["parser"] = @parser unless @parser.nil?
351
328
 
352
- #get json string of event to keep track of how many bytes we are sending
329
+ # append to list of events
330
+ event = {thread: thread_id.to_s,
331
+ ts: timestamp,
332
+ attrs: record}
333
+
334
+ # get json string of event to keep track of how many bytes we are sending
353
335
 
354
336
  begin
355
337
  event_json = event.to_json
356
338
  rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
357
- $log.warn "#{e.class}: #{e.message}"
339
+ $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
358
340
 
359
341
  # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
360
- time = Fluent::EventTime.new( sec, nsec )
342
+ time = Fluent::EventTime.new(sec, nsec)
361
343
  router.emit_error_event(tag, time, record, e)
362
344
 
345
+ # Print attribute values for debugging / troubleshooting purposes
346
+ $log.debug "Event attributes:"
347
+
363
348
  event[:attrs].each do |key, value|
364
- $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
365
- event[:attrs][key] = value.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
349
+ # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
350
+ $log.debug "\t#{key} (#{value.class}): '#{value}'"
366
351
  end
352
+
353
+ # Recursively re-encode and sanitize potentially bad string values
354
+ event[:attrs] = sanitize_and_reencode_value(event[:attrs])
367
355
  event_json = event.to_json
368
356
  end
369
357
 
370
- #generate new request if json size of events in the array exceed maximum request buffer size
358
+ # generate new request if json size of events in the array exceed maximum request buffer size
371
359
  append_event = true
372
360
  if total_bytes + event_json.bytesize > @max_request_buffer
373
- #make sure we always have at least one event
374
- if events.size == 0
375
- events << event
361
+ # the case where a single event causes us to exceed the @max_request_buffer
362
+ if events.empty?
363
+ # if we are able to truncate the content (and append an ellipsis)
364
+ # inside the @message_field we do so here
365
+ if record.key?(@message_field) &&
366
+ record[@message_field].is_a?(String) &&
367
+ record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
368
+ record[@message_field].bytesize >= 3
369
+
370
+ @log.warn "Received a record that cannot fit within max_request_buffer "\
371
+ "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
372
+ "is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
373
+ max_msg_size = @max_request_buffer - event_json.bytesize - 3
374
+ truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
375
+ event[:attrs][@message_field] = truncated_msg
376
+ events << event
377
+
378
+ # otherwise we drop the event and save ourselves hitting a 4XX response from the server
379
+ else
380
+ @log.warn "Received a record that cannot fit within max_request_buffer "\
381
+ "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
382
+ "is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
383
+ "dropping event."
384
+ end
376
385
  append_event = false
377
386
  end
378
- request = self.create_request( events, current_threads )
379
- requests << request
387
+
388
+ unless events.empty?
389
+ request = create_request(events, current_threads)
390
+ requests << request
391
+ end
380
392
 
381
393
  total_bytes = 0
382
- current_threads = Hash.new
383
- events = Array.new
394
+ current_threads = {}
395
+ events = []
384
396
  end
385
397
 
386
- #if we haven't consumed the current event already
387
- #add it to the end of our array and keep track of the json bytesize
398
+ # if we haven't consumed the current event already
399
+ # add it to the end of our array and keep track of the json bytesize
388
400
  if append_event
389
401
  events << event
390
402
  total_bytes += event_json.bytesize
391
403
  end
392
-
393
404
  }
394
405
 
395
- #create a final request with any left over events
396
- request = self.create_request( events, current_threads )
397
- requests << request
406
+ # create a final request with any left over events
407
+ unless events.empty?
408
+ request = create_request(events, current_threads)
409
+ requests << request
410
+ end
398
411
 
412
+ requests
399
413
  end
400
414
 
401
- def create_request( events, current_threads )
402
- #build the scalyr thread objects
403
- threads = Array.new
415
+ def create_request(events, current_threads)
416
+ # build the scalyr thread objects
417
+ threads = []
404
418
  current_threads.each do |tag, id|
405
- threads << { :id => id.to_s,
406
- :name => "Fluentd: #{tag}"
407
- }
419
+ threads << {id: id.to_s,
420
+ name: "Fluentd: #{tag}"}
408
421
  end
409
422
 
410
- current_time = self.to_millis( Fluent::Engine.now )
423
+ current_time = to_millis(Fluent::Engine.now)
411
424
 
412
- body = { :token => @api_write_token,
413
- :client_timestamp => current_time.to_s,
414
- :session => @session,
415
- :events => events,
416
- :threads => threads
417
- }
425
+ body = {token: @api_write_token,
426
+ client_timestamp: current_time.to_s,
427
+ session: @session,
428
+ events: events,
429
+ threads: threads}
418
430
 
419
- #add server_attributes hash if it exists
420
- if @server_attributes
421
- body[:sessionInfo] = @server_attributes
422
- end
431
+ # add server_attributes hash if it exists
432
+ body[:sessionInfo] = @server_attributes if @server_attributes
423
433
 
424
- { :body => body.to_json, :record_count => events.size }
434
+ {body: body.to_json, record_count: events.size}
425
435
  end
426
-
427
436
  end
428
437
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,8 +17,6 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
-
20
20
  module Scalyr
21
21
  class ClientError < StandardError; end
22
22
  class Client4xxError < StandardError; end