fluent-plugin-scalyr 0.8.9 → 0.8.14

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,47 +17,46 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
- require 'fluent/plugin/output'
20
- require 'fluent/plugin/scalyr-exceptions'
21
- require 'fluent/plugin_helper/compat_parameters'
22
- require 'json'
23
- require 'net/http'
24
- require 'net/https'
25
- require 'rbzip2'
26
- require 'stringio'
27
- require 'zlib'
28
- require 'securerandom'
29
- require 'socket'
30
- require 'thread'
31
-
20
+ require "fluent/plugin/output"
21
+ require "fluent/plugin/scalyr_exceptions"
22
+ require "fluent/plugin/scalyr_utils"
23
+ require "fluent/plugin_helper/compat_parameters"
24
+ require "json"
25
+ require "net/http"
26
+ require "net/https"
27
+ require "rbzip2"
28
+ require "stringio"
29
+ require "zlib"
30
+ require "securerandom"
31
+ require "socket"
32
32
  module Scalyr
33
33
  class ScalyrOut < Fluent::Plugin::Output
34
- Fluent::Plugin.register_output( 'scalyr', self )
34
+ Fluent::Plugin.register_output("scalyr", self)
35
35
  helpers :compat_parameters
36
36
  helpers :event_emitter
37
37
 
38
38
  config_param :api_write_token, :string
39
- config_param :server_attributes, :hash, :default => nil
40
- config_param :use_hostname_for_serverhost, :bool, :default => true
41
- config_param :scalyr_server, :string, :default => "https://agent.scalyr.com/"
42
- config_param :ssl_ca_bundle_path, :string, :default => "/etc/ssl/certs/ca-bundle.crt"
43
- config_param :ssl_verify_peer, :bool, :default => true
44
- config_param :ssl_verify_depth, :integer, :default => 5
45
- config_param :message_field, :string, :default => "message"
46
- config_param :max_request_buffer, :integer, :default => 3000000
47
- config_param :force_message_encoding, :string, :default => nil
48
- config_param :replace_invalid_utf8, :bool, :default => false
49
- config_param :compression_type, :string, :default => nil #Valid options are bz2, deflate or None. Defaults to None.
50
- config_param :compression_level, :integer, :default => 9 #An int containing the compression level of compression to use, from 1-9. Defaults to 9 (max)
39
+ config_param :server_attributes, :hash, default: nil
40
+ config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
41
+ config_param :use_hostname_for_serverhost, :bool, default: true
42
+ config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
43
+ config_param :ssl_ca_bundle_path, :string, default: nil
44
+ config_param :ssl_verify_peer, :bool, default: true
45
+ config_param :ssl_verify_depth, :integer, default: 5
46
+ config_param :message_field, :string, default: "message"
47
+ config_param :max_request_buffer, :integer, default: 5_500_000
48
+ config_param :force_message_encoding, :string, default: nil
49
+ config_param :replace_invalid_utf8, :bool, default: false
50
+ config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
51
+ config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
51
52
 
52
53
  config_section :buffer do
53
- config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
54
- config_set_default :retry_max_interval, 30 #wait a maximum of 30 seconds per retry
55
- config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
56
- config_set_default :flush_interval, 5 #default flush interval of 5 seconds
57
- config_set_default :chunk_limit_size, 2500000 #default chunk size of 2.5mb
58
- config_set_default :queue_limit_length, 1024 #default queue size of 1024
54
+ config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
55
+ config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
56
+ config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
57
+ config_set_default :flush_interval, 5 # default flush interval of 5 seconds
58
+ config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
59
+ config_set_default :queue_limit_length, 1024 # default queue size of 1024
59
60
  end
60
61
 
61
62
  # support for version 0.14.0:
@@ -67,46 +68,49 @@ module Scalyr
67
68
  true
68
69
  end
69
70
 
70
- def configure( conf )
71
+ def multi_workers_ready?
72
+ true
73
+ end
71
74
 
72
- if conf.elements('buffer').empty?
73
- $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
75
+ def configure(conf)
76
+ if conf.elements("buffer").empty?
77
+ $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
74
78
  end
75
79
 
76
- compat_parameters_buffer( conf, default_chunk_key: '' )
80
+ compat_parameters_buffer(conf, default_chunk_key: "")
77
81
 
78
82
  super
79
83
 
80
- if @buffer.chunk_limit_size > 6000000
81
- $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
84
+ if @buffer.chunk_limit_size > 6_000_000
85
+ $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
82
86
  end
83
87
 
84
- if @max_request_buffer > 6000000
85
- $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
88
+ if @max_request_buffer > 6_000_000
89
+ $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
86
90
  end
87
91
 
88
92
  @message_encoding = nil
89
- if @force_message_encoding.to_s != ''
93
+ if @force_message_encoding.to_s != ""
90
94
  begin
91
- @message_encoding = Encoding.find( @force_message_encoding )
95
+ @message_encoding = Encoding.find(@force_message_encoding)
92
96
  $log.debug "Forcing message encoding to '#{@force_message_encoding}'"
93
97
  rescue ArgumentError
94
98
  $log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
95
99
  end
96
100
  end
97
101
 
98
- #evaluate any statements in string value of the server_attributes object
102
+ # evaluate any statements in string value of the server_attributes object
99
103
  if @server_attributes
100
104
  new_attributes = {}
101
105
  @server_attributes.each do |key, value|
102
- if value.is_a?( String )
103
- m = /^\#{(.*)}$/.match( value )
104
- if m
105
- new_attributes[key] = eval( m[1] )
106
- else
107
- new_attributes[key] = value
108
- end
109
- end
106
+ next unless value.is_a?(String)
107
+
108
+ m = /^\#{(.*)}$/.match(value)
109
+ new_attributes[key] = if m
110
+ eval(m[1]) # rubocop:disable Security/Eval
111
+ else
112
+ value
113
+ end
110
114
  end
111
115
  @server_attributes = new_attributes
112
116
  end
@@ -115,148 +119,131 @@ module Scalyr
115
119
  if @use_hostname_for_serverhost
116
120
 
117
121
  # ensure server_attributes is not nil
118
- if @server_attributes.nil?
119
- @server_attributes = {}
120
- end
122
+ @server_attributes = {} if @server_attributes.nil?
121
123
 
122
124
  # only set serverHost if it doesn't currently exist in server_attributes
123
125
  # Note: Use strings rather than symbols for the key, because keys coming
124
126
  # from the config file will be strings
125
- if !@server_attributes.key? 'serverHost'
126
- @server_attributes['serverHost'] = Socket.gethostname
127
+ unless @server_attributes.key? "serverHost"
128
+ @server_attributes["serverHost"] = Socket.gethostname
127
129
  end
128
130
  end
129
131
 
130
- @scalyr_server << '/' unless @scalyr_server.end_with?('/')
132
+ @scalyr_server << "/" unless @scalyr_server.end_with?("/")
131
133
 
132
134
  @add_events_uri = URI @scalyr_server + "addEvents"
133
135
 
134
136
  num_threads = @buffer_config.flush_thread_count
135
137
 
136
- #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
137
- raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
138
+ # forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
139
+ if num_threads > 1
140
+ raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
141
+ end
138
142
  end
139
143
 
140
144
  def start
141
145
  super
142
- $log.info "Scalyr Fluentd Plugin ID - #{self.plugin_id()}"
143
- #Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
146
+ # Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
144
147
  @session = SecureRandom.uuid
145
148
 
146
- @sync = Mutex.new
147
- #the following variables are all under the control of the above mutex
148
- @thread_ids = Hash.new #hash of tags -> id
149
- @next_id = 1 #incrementing thread id for the session
150
- @last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
151
-
149
+ $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
152
150
  end
153
151
 
154
- def format( tag, time, record )
155
- begin
156
-
157
- if time.nil?
158
- time = Fluent::Engine.now
159
- end
160
-
161
- # handle timestamps that are not EventTime types
162
- if time.is_a?( Integer )
163
- time = Fluent::EventTime.new( time )
164
- elsif time.is_a?( Float )
165
- components = time.divmod 1 #get integer and decimal components
166
- sec = components[0].to_i
167
- nsec = (components[1] * 10**9).to_i
168
- time = Fluent::EventTime.new( sec, nsec )
169
- end
152
+ def format(tag, time, record)
153
+ time = Fluent::Engine.now if time.nil?
154
+
155
+ # handle timestamps that are not EventTime types
156
+ if time.is_a?(Integer)
157
+ time = Fluent::EventTime.new(time)
158
+ elsif time.is_a?(Float)
159
+ components = time.divmod 1 # get integer and decimal components
160
+ sec = components[0].to_i
161
+ nsec = (components[1] * 10**9).to_i
162
+ time = Fluent::EventTime.new(sec, nsec)
163
+ end
170
164
 
171
- if @message_field != "message"
172
- if record.key? @message_field
173
- if record.key? "message"
174
- $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten."
175
- end
176
- record["message"] = record[@message_field]
177
- record.delete( @message_field )
165
+ if @message_field != "message"
166
+ if record.key? @message_field
167
+ if record.key? "message"
168
+ $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
169
  end
170
+ record["message"] = record[@message_field]
171
+ record.delete(@message_field)
179
172
  end
173
+ end
180
174
 
181
- if @message_encoding and record.key? "message" and record["message"]
182
- if @replace_invalid_utf8 and @message_encoding == Encoding::UTF_8
183
- record["message"] = record["message"].encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
184
- else
185
- record["message"].force_encoding( @message_encoding )
186
- end
175
+ if @message_encoding && record.key?("message") && record["message"]
176
+ if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
177
+ record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
+ else
179
+ record["message"].force_encoding(@message_encoding)
187
180
  end
188
- [tag, time.sec, time.nsec, record].to_msgpack
189
-
190
- rescue JSON::GeneratorError
191
- $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
192
- raise
193
181
  end
182
+ [tag, time.sec, time.nsec, record].to_msgpack
183
+ rescue JSON::GeneratorError
184
+ $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
185
+ raise
194
186
  end
195
187
 
196
- #called by fluentd when a chunk of log messages is ready
197
- def write( chunk )
198
- begin
199
- $log.debug "Size of chunk is: #{chunk.size}"
200
- requests = self.build_add_events_body( chunk )
201
- $log.debug "Chunk split into #{requests.size} request(s)."
202
-
203
- requests.each_with_index { |request, index|
204
- $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
205
- begin
206
- response = self.post_request( @add_events_uri, request[:body] )
207
- self.handle_response( response )
208
- rescue OpenSSL::SSL::SSLError => e
209
- if e.message.include? "certificate verify failed"
210
- $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
211
- end
212
- $log.warn e.message
213
- $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
214
- rescue Scalyr::Client4xxError => e
215
- $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
188
+ # called by fluentd when a chunk of log messages is ready
189
+ def write(chunk)
190
+ $log.debug "Size of chunk is: #{chunk.size}"
191
+ requests = build_add_events_body(chunk)
192
+ $log.debug "Chunk split into #{requests.size} request(s)."
216
193
 
194
+ requests.each_with_index {|request, index|
195
+ $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
196
+ begin
197
+ response = post_request(@add_events_uri, request[:body])
198
+ handle_response(response)
199
+ rescue OpenSSL::SSL::SSLError => e
200
+ if e.message.include? "certificate verify failed"
201
+ $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
217
202
  end
218
- }
219
-
220
- rescue JSON::GeneratorError
221
- $log.warn "Unable to format message due to JSON::GeneratorError."
222
- raise
223
- end
203
+ $log.warn e.message
204
+ $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
205
+ rescue Scalyr::Client4xxError => e
206
+ $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
207
+ end
208
+ }
209
+ rescue JSON::GeneratorError
210
+ $log.warn "Unable to format message due to JSON::GeneratorError."
211
+ raise
224
212
  end
225
213
 
226
-
227
- #explicit function to convert to nanoseconds
228
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
229
- def to_nanos( seconds, nsec )
214
+ # explicit function to convert to nanoseconds
215
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
216
+ def to_nanos(seconds, nsec)
230
217
  (seconds * 10**9) + nsec
231
218
  end
232
219
 
233
- #explicit function to convert to milliseconds
234
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
235
- def to_millis( timestamp )
220
+ # explicit function to convert to milliseconds
221
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
222
+ def to_millis(timestamp)
236
223
  (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
237
224
  end
238
225
 
239
- def post_request( uri, body )
240
-
241
- https = Net::HTTP.new( uri.host, uri.port )
226
+ def post_request(uri, body)
227
+ https = Net::HTTP.new(uri.host, uri.port)
242
228
  https.use_ssl = true
243
229
 
244
- #verify peers to prevent potential MITM attacks
230
+ # verify peers to prevent potential MITM attacks
245
231
  if @ssl_verify_peer
246
- https.ca_file = @ssl_ca_bundle_path
232
+ https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
233
+ https.ssl_version = :TLSv1_2
247
234
  https.verify_mode = OpenSSL::SSL::VERIFY_PEER
248
235
  https.verify_depth = @ssl_verify_depth
249
236
  end
250
237
 
251
- #use compression if enabled
238
+ # use compression if enabled
252
239
  encoding = nil
253
240
 
254
241
  if @compression_type
255
- if @compression_type == 'deflate'
256
- encoding = 'deflate'
242
+ if @compression_type == "deflate"
243
+ encoding = "deflate"
257
244
  body = Zlib::Deflate.deflate(body, @compression_level)
258
- elsif @compression_type == 'bz2'
259
- encoding = 'bz2'
245
+ elsif @compression_type == "bz2"
246
+ encoding = "bz2"
260
247
  io = StringIO.new
261
248
  bz2 = RBzip2.default_adapter::Compressor.new io
262
249
  bz2.write body
@@ -266,179 +253,157 @@ module Scalyr
266
253
  end
267
254
 
268
255
  post = Net::HTTP::Post.new uri.path
269
- post.add_field( 'Content-Type', 'application/json' )
256
+ post.add_field("Content-Type", "application/json")
270
257
 
271
- if @compression_type
272
- post.add_field( 'Content-Encoding', encoding )
273
- end
258
+ post.add_field("Content-Encoding", encoding) if @compression_type
274
259
 
275
260
  post.body = body
276
261
 
277
- https.request( post )
278
-
262
+ https.request(post)
279
263
  end
280
264
 
281
- def handle_response( response )
265
+ def handle_response(response)
282
266
  $log.debug "Response Code: #{response.code}"
283
267
  $log.debug "Response Body: #{response.body}"
284
268
 
285
- response_hash = Hash.new
269
+ response_hash = {}
286
270
 
287
271
  begin
288
- response_hash = JSON.parse( response.body )
289
- rescue
272
+ response_hash = JSON.parse(response.body)
273
+ rescue StandardError
290
274
  response_hash["status"] = "Invalid JSON response from server"
291
275
  end
292
276
 
293
- #make sure the JSON reponse has a "status" field
294
- if !response_hash.key? "status"
277
+ # make sure the JSON reponse has a "status" field
278
+ unless response_hash.key? "status"
295
279
  $log.debug "JSON response does not contain status message"
296
280
  raise Scalyr::ServerError.new "JSON response does not contain status message"
297
281
  end
298
282
 
299
283
  status = response_hash["status"]
300
284
 
301
- #4xx codes are handled separately
285
+ # 4xx codes are handled separately
302
286
  if response.code =~ /^4\d\d/
303
287
  raise Scalyr::Client4xxError.new status
304
288
  else
305
- if status != "success"
289
+ if status != "success" # rubocop:disable Style/IfInsideElse
306
290
  if status =~ /discardBuffer/
307
291
  $log.warn "Received 'discardBuffer' message from server. Buffer dropped."
308
- elsif status =~ %r"/client/"i
292
+ elsif status =~ %r{/client/}i
309
293
  raise Scalyr::ClientError.new status
310
- else #don't check specifically for server, we assume all non-client errors are server errors
294
+ else # don't check specifically for server, we assume all non-client errors are server errors
311
295
  raise Scalyr::ServerError.new status
312
296
  end
313
- elsif !response.code.include? "200" #response code is a string not an int
297
+ elsif !response.code.include? "200" # response code is a string not an int
314
298
  raise Scalyr::ServerError
315
299
  end
316
300
  end
317
-
318
301
  end
319
302
 
320
- def build_add_events_body( chunk )
321
-
322
- #requests
323
- requests = Array.new
303
+ def build_add_events_body(chunk)
304
+ # requests
305
+ requests = []
324
306
 
325
- #set of unique scalyr threads for this chunk
326
- current_threads = Hash.new
307
+ # set of unique scalyr threads for this chunk
308
+ current_threads = {}
327
309
 
328
- #byte count
310
+ # byte count
329
311
  total_bytes = 0
330
312
 
331
- #create a Scalyr event object for each record in the chunk
332
- events = Array.new
333
- chunk.msgpack_each {|(tag, sec, nsec, record)|
334
-
335
- timestamp = self.to_nanos( sec, nsec )
336
-
337
- thread_id = 0
313
+ # create a Scalyr event object for each record in the chunk
314
+ events = []
315
+ chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
316
+ timestamp = to_nanos(sec, nsec)
338
317
 
339
- @sync.synchronize {
340
- #ensure timestamp is at least 1 nanosecond greater than the last one
341
- timestamp = [timestamp, @last_timestamp + 1].max
342
- @last_timestamp = timestamp
318
+ thread_id = tag
343
319
 
344
- #get thread id or add a new one if we haven't seen this tag before
345
- if @thread_ids.key? tag
346
- thread_id = @thread_ids[tag]
347
- else
348
- thread_id = @next_id
349
- @thread_ids[tag] = thread_id
350
- @next_id += 1
351
- end
352
- }
353
-
354
- #then update the map of threads for this chunk
320
+ # then update the map of threads for this chunk
355
321
  current_threads[tag] = thread_id
356
322
 
357
- #add a logfile field if one doesn't exist
358
- if !record.key? "logfile"
359
- record["logfile"] = "/fluentd/#{tag}"
360
- end
323
+ # add a logfile field if one doesn't exist
324
+ record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
361
325
 
362
- #append to list of events
363
- event = { :thread => thread_id.to_s,
364
- :ts => timestamp,
365
- :attrs => record
366
- }
326
+ # set per-event parser if it is configured
327
+ record["parser"] = @parser unless @parser.nil?
367
328
 
368
- #get json string of event to keep track of how many bytes we are sending
329
+ # append to list of events
330
+ event = {thread: thread_id.to_s,
331
+ ts: timestamp,
332
+ attrs: record}
333
+
334
+ # get json string of event to keep track of how many bytes we are sending
369
335
 
370
336
  begin
371
337
  event_json = event.to_json
372
338
  rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
373
- $log.warn "#{e.class}: #{e.message}"
339
+ $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
374
340
 
375
341
  # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
376
- time = Fluent::EventTime.new( sec, nsec )
342
+ time = Fluent::EventTime.new(sec, nsec)
377
343
  router.emit_error_event(tag, time, record, e)
378
344
 
345
+ # Print attribute values for debugging / troubleshooting purposes
346
+ $log.debug "Event attributes:"
347
+
379
348
  event[:attrs].each do |key, value|
380
- $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
381
- event[:attrs][key] = value.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
349
+ # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
350
+ $log.debug "\t#{key} (#{value.class}): '#{value}'"
382
351
  end
352
+
353
+ # Recursively re-encode and sanitize potentially bad string values
354
+ event[:attrs] = sanitize_and_reencode_value(event[:attrs])
383
355
  event_json = event.to_json
384
356
  end
385
357
 
386
- #generate new request if json size of events in the array exceed maximum request buffer size
358
+ # generate new request if json size of events in the array exceed maximum request buffer size
387
359
  append_event = true
388
360
  if total_bytes + event_json.bytesize > @max_request_buffer
389
- #make sure we always have at least one event
390
- if events.size == 0
361
+ # make sure we always have at least one event
362
+ if events.empty?
391
363
  events << event
392
364
  append_event = false
393
365
  end
394
- request = self.create_request( events, current_threads )
366
+ request = create_request(events, current_threads)
395
367
  requests << request
396
368
 
397
369
  total_bytes = 0
398
- current_threads = Hash.new
399
- events = Array.new
370
+ current_threads = {}
371
+ events = []
400
372
  end
401
373
 
402
- #if we haven't consumed the current event already
403
- #add it to the end of our array and keep track of the json bytesize
374
+ # if we haven't consumed the current event already
375
+ # add it to the end of our array and keep track of the json bytesize
404
376
  if append_event
405
377
  events << event
406
378
  total_bytes += event_json.bytesize
407
379
  end
408
-
409
380
  }
410
381
 
411
- #create a final request with any left over events
412
- request = self.create_request( events, current_threads )
382
+ # create a final request with any left over events
383
+ request = create_request(events, current_threads)
413
384
  requests << request
414
-
415
385
  end
416
386
 
417
- def create_request( events, current_threads )
418
- #build the scalyr thread objects
419
- threads = Array.new
387
+ def create_request(events, current_threads)
388
+ # build the scalyr thread objects
389
+ threads = []
420
390
  current_threads.each do |tag, id|
421
- threads << { :id => id.to_s,
422
- :name => "Fluentd: #{tag}"
423
- }
391
+ threads << {id: id.to_s,
392
+ name: "Fluentd: #{tag}"}
424
393
  end
425
394
 
426
- current_time = self.to_millis( Fluent::Engine.now )
395
+ current_time = to_millis(Fluent::Engine.now)
427
396
 
428
- body = { :token => @api_write_token,
429
- :client_timestamp => current_time.to_s,
430
- :session => @session,
431
- :events => events,
432
- :threads => threads
433
- }
397
+ body = {token: @api_write_token,
398
+ client_timestamp: current_time.to_s,
399
+ session: @session,
400
+ events: events,
401
+ threads: threads}
434
402
 
435
- #add server_attributes hash if it exists
436
- if @server_attributes
437
- body[:sessionInfo] = @server_attributes
438
- end
403
+ # add server_attributes hash if it exists
404
+ body[:sessionInfo] = @server_attributes if @server_attributes
439
405
 
440
- { :body => body.to_json, :record_count => events.size }
406
+ {body: body.to_json, record_count: events.size}
441
407
  end
442
-
443
408
  end
444
409
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,8 +17,6 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
-
20
20
  module Scalyr
21
21
  class ClientError < StandardError; end
22
22
  class Client4xxError < StandardError; end