fluent-plugin-scalyr 0.8.9 → 0.8.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,47 +17,46 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
- require 'fluent/plugin/output'
20
- require 'fluent/plugin/scalyr-exceptions'
21
- require 'fluent/plugin_helper/compat_parameters'
22
- require 'json'
23
- require 'net/http'
24
- require 'net/https'
25
- require 'rbzip2'
26
- require 'stringio'
27
- require 'zlib'
28
- require 'securerandom'
29
- require 'socket'
30
- require 'thread'
31
-
20
+ require "fluent/plugin/output"
21
+ require "fluent/plugin/scalyr_exceptions"
22
+ require "fluent/plugin/scalyr_utils"
23
+ require "fluent/plugin_helper/compat_parameters"
24
+ require "json"
25
+ require "net/http"
26
+ require "net/https"
27
+ require "rbzip2"
28
+ require "stringio"
29
+ require "zlib"
30
+ require "securerandom"
31
+ require "socket"
32
32
  module Scalyr
33
33
  class ScalyrOut < Fluent::Plugin::Output
34
- Fluent::Plugin.register_output( 'scalyr', self )
34
+ Fluent::Plugin.register_output("scalyr", self)
35
35
  helpers :compat_parameters
36
36
  helpers :event_emitter
37
37
 
38
38
  config_param :api_write_token, :string
39
- config_param :server_attributes, :hash, :default => nil
40
- config_param :use_hostname_for_serverhost, :bool, :default => true
41
- config_param :scalyr_server, :string, :default => "https://agent.scalyr.com/"
42
- config_param :ssl_ca_bundle_path, :string, :default => "/etc/ssl/certs/ca-bundle.crt"
43
- config_param :ssl_verify_peer, :bool, :default => true
44
- config_param :ssl_verify_depth, :integer, :default => 5
45
- config_param :message_field, :string, :default => "message"
46
- config_param :max_request_buffer, :integer, :default => 3000000
47
- config_param :force_message_encoding, :string, :default => nil
48
- config_param :replace_invalid_utf8, :bool, :default => false
49
- config_param :compression_type, :string, :default => nil #Valid options are bz2, deflate or None. Defaults to None.
50
- config_param :compression_level, :integer, :default => 9 #An int containing the compression level of compression to use, from 1-9. Defaults to 9 (max)
39
+ config_param :server_attributes, :hash, default: nil
40
+ config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
41
+ config_param :use_hostname_for_serverhost, :bool, default: true
42
+ config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
43
+ config_param :ssl_ca_bundle_path, :string, default: nil
44
+ config_param :ssl_verify_peer, :bool, default: true
45
+ config_param :ssl_verify_depth, :integer, default: 5
46
+ config_param :message_field, :string, default: "message"
47
+ config_param :max_request_buffer, :integer, default: 5_500_000
48
+ config_param :force_message_encoding, :string, default: nil
49
+ config_param :replace_invalid_utf8, :bool, default: false
50
+ config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
51
+ config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
51
52
 
52
53
  config_section :buffer do
53
- config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
54
- config_set_default :retry_max_interval, 30 #wait a maximum of 30 seconds per retry
55
- config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
56
- config_set_default :flush_interval, 5 #default flush interval of 5 seconds
57
- config_set_default :chunk_limit_size, 2500000 #default chunk size of 2.5mb
58
- config_set_default :queue_limit_length, 1024 #default queue size of 1024
54
+ config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
55
+ config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
56
+ config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
57
+ config_set_default :flush_interval, 5 # default flush interval of 5 seconds
58
+ config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
59
+ config_set_default :queue_limit_length, 1024 # default queue size of 1024
59
60
  end
60
61
 
61
62
  # support for version 0.14.0:
@@ -67,46 +68,49 @@ module Scalyr
67
68
  true
68
69
  end
69
70
 
70
- def configure( conf )
71
+ def multi_workers_ready?
72
+ true
73
+ end
71
74
 
72
- if conf.elements('buffer').empty?
73
- $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
75
+ def configure(conf)
76
+ if conf.elements("buffer").empty?
77
+ $log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
74
78
  end
75
79
 
76
- compat_parameters_buffer( conf, default_chunk_key: '' )
80
+ compat_parameters_buffer(conf, default_chunk_key: "")
77
81
 
78
82
  super
79
83
 
80
- if @buffer.chunk_limit_size > 6000000
81
- $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
84
+ if @buffer.chunk_limit_size > 6_000_000
85
+ $log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
82
86
  end
83
87
 
84
- if @max_request_buffer > 6000000
85
- $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
88
+ if @max_request_buffer > 6_000_000
89
+ $log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
86
90
  end
87
91
 
88
92
  @message_encoding = nil
89
- if @force_message_encoding.to_s != ''
93
+ if @force_message_encoding.to_s != ""
90
94
  begin
91
- @message_encoding = Encoding.find( @force_message_encoding )
95
+ @message_encoding = Encoding.find(@force_message_encoding)
92
96
  $log.debug "Forcing message encoding to '#{@force_message_encoding}'"
93
97
  rescue ArgumentError
94
98
  $log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
95
99
  end
96
100
  end
97
101
 
98
- #evaluate any statements in string value of the server_attributes object
102
+ # evaluate any statements in string value of the server_attributes object
99
103
  if @server_attributes
100
104
  new_attributes = {}
101
105
  @server_attributes.each do |key, value|
102
- if value.is_a?( String )
103
- m = /^\#{(.*)}$/.match( value )
104
- if m
105
- new_attributes[key] = eval( m[1] )
106
- else
107
- new_attributes[key] = value
108
- end
109
- end
106
+ next unless value.is_a?(String)
107
+
108
+ m = /^\#{(.*)}$/.match(value)
109
+ new_attributes[key] = if m
110
+ eval(m[1]) # rubocop:disable Security/Eval
111
+ else
112
+ value
113
+ end
110
114
  end
111
115
  @server_attributes = new_attributes
112
116
  end
@@ -115,148 +119,131 @@ module Scalyr
115
119
  if @use_hostname_for_serverhost
116
120
 
117
121
  # ensure server_attributes is not nil
118
- if @server_attributes.nil?
119
- @server_attributes = {}
120
- end
122
+ @server_attributes = {} if @server_attributes.nil?
121
123
 
122
124
  # only set serverHost if it doesn't currently exist in server_attributes
123
125
  # Note: Use strings rather than symbols for the key, because keys coming
124
126
  # from the config file will be strings
125
- if !@server_attributes.key? 'serverHost'
126
- @server_attributes['serverHost'] = Socket.gethostname
127
+ unless @server_attributes.key? "serverHost"
128
+ @server_attributes["serverHost"] = Socket.gethostname
127
129
  end
128
130
  end
129
131
 
130
- @scalyr_server << '/' unless @scalyr_server.end_with?('/')
132
+ @scalyr_server << "/" unless @scalyr_server.end_with?("/")
131
133
 
132
134
  @add_events_uri = URI @scalyr_server + "addEvents"
133
135
 
134
136
  num_threads = @buffer_config.flush_thread_count
135
137
 
136
- #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
137
- raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
138
+ # forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
139
+ if num_threads > 1
140
+ raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
141
+ end
138
142
  end
139
143
 
140
144
  def start
141
145
  super
142
- $log.info "Scalyr Fluentd Plugin ID - #{self.plugin_id()}"
143
- #Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
146
+ # Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
144
147
  @session = SecureRandom.uuid
145
148
 
146
- @sync = Mutex.new
147
- #the following variables are all under the control of the above mutex
148
- @thread_ids = Hash.new #hash of tags -> id
149
- @next_id = 1 #incrementing thread id for the session
150
- @last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
151
-
149
+ $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
152
150
  end
153
151
 
154
- def format( tag, time, record )
155
- begin
156
-
157
- if time.nil?
158
- time = Fluent::Engine.now
159
- end
160
-
161
- # handle timestamps that are not EventTime types
162
- if time.is_a?( Integer )
163
- time = Fluent::EventTime.new( time )
164
- elsif time.is_a?( Float )
165
- components = time.divmod 1 #get integer and decimal components
166
- sec = components[0].to_i
167
- nsec = (components[1] * 10**9).to_i
168
- time = Fluent::EventTime.new( sec, nsec )
169
- end
152
+ def format(tag, time, record)
153
+ time = Fluent::Engine.now if time.nil?
154
+
155
+ # handle timestamps that are not EventTime types
156
+ if time.is_a?(Integer)
157
+ time = Fluent::EventTime.new(time)
158
+ elsif time.is_a?(Float)
159
+ components = time.divmod 1 # get integer and decimal components
160
+ sec = components[0].to_i
161
+ nsec = (components[1] * 10**9).to_i
162
+ time = Fluent::EventTime.new(sec, nsec)
163
+ end
170
164
 
171
- if @message_field != "message"
172
- if record.key? @message_field
173
- if record.key? "message"
174
- $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten."
175
- end
176
- record["message"] = record[@message_field]
177
- record.delete( @message_field )
165
+ if @message_field != "message"
166
+ if record.key? @message_field
167
+ if record.key? "message"
168
+ $log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
169
  end
170
+ record["message"] = record[@message_field]
171
+ record.delete(@message_field)
179
172
  end
173
+ end
180
174
 
181
- if @message_encoding and record.key? "message" and record["message"]
182
- if @replace_invalid_utf8 and @message_encoding == Encoding::UTF_8
183
- record["message"] = record["message"].encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
184
- else
185
- record["message"].force_encoding( @message_encoding )
186
- end
175
+ if @message_encoding && record.key?("message") && record["message"]
176
+ if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
177
+ record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
178
+ else
179
+ record["message"].force_encoding(@message_encoding)
187
180
  end
188
- [tag, time.sec, time.nsec, record].to_msgpack
189
-
190
- rescue JSON::GeneratorError
191
- $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
192
- raise
193
181
  end
182
+ [tag, time.sec, time.nsec, record].to_msgpack
183
+ rescue JSON::GeneratorError
184
+ $log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
185
+ raise
194
186
  end
195
187
 
196
- #called by fluentd when a chunk of log messages is ready
197
- def write( chunk )
198
- begin
199
- $log.debug "Size of chunk is: #{chunk.size}"
200
- requests = self.build_add_events_body( chunk )
201
- $log.debug "Chunk split into #{requests.size} request(s)."
202
-
203
- requests.each_with_index { |request, index|
204
- $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
205
- begin
206
- response = self.post_request( @add_events_uri, request[:body] )
207
- self.handle_response( response )
208
- rescue OpenSSL::SSL::SSLError => e
209
- if e.message.include? "certificate verify failed"
210
- $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
211
- end
212
- $log.warn e.message
213
- $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
214
- rescue Scalyr::Client4xxError => e
215
- $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
188
+ # called by fluentd when a chunk of log messages is ready
189
+ def write(chunk)
190
+ $log.debug "Size of chunk is: #{chunk.size}"
191
+ requests = build_add_events_body(chunk)
192
+ $log.debug "Chunk split into #{requests.size} request(s)."
216
193
 
194
+ requests.each_with_index {|request, index|
195
+ $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
196
+ begin
197
+ response = post_request(@add_events_uri, request[:body])
198
+ handle_response(response)
199
+ rescue OpenSSL::SSL::SSLError => e
200
+ if e.message.include? "certificate verify failed"
201
+ $log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
217
202
  end
218
- }
219
-
220
- rescue JSON::GeneratorError
221
- $log.warn "Unable to format message due to JSON::GeneratorError."
222
- raise
223
- end
203
+ $log.warn e.message
204
+ $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
205
+ rescue Scalyr::Client4xxError => e
206
+ $log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
207
+ end
208
+ }
209
+ rescue JSON::GeneratorError
210
+ $log.warn "Unable to format message due to JSON::GeneratorError."
211
+ raise
224
212
  end
225
213
 
226
-
227
- #explicit function to convert to nanoseconds
228
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
229
- def to_nanos( seconds, nsec )
214
+ # explicit function to convert to nanoseconds
215
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
216
+ def to_nanos(seconds, nsec)
230
217
  (seconds * 10**9) + nsec
231
218
  end
232
219
 
233
- #explicit function to convert to milliseconds
234
- #will make things easier to maintain if/when fluentd supports higher than second resolutions
235
- def to_millis( timestamp )
220
+ # explicit function to convert to milliseconds
221
+ # will make things easier to maintain if/when fluentd supports higher than second resolutions
222
+ def to_millis(timestamp)
236
223
  (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
237
224
  end
238
225
 
239
- def post_request( uri, body )
240
-
241
- https = Net::HTTP.new( uri.host, uri.port )
226
+ def post_request(uri, body)
227
+ https = Net::HTTP.new(uri.host, uri.port)
242
228
  https.use_ssl = true
243
229
 
244
- #verify peers to prevent potential MITM attacks
230
+ # verify peers to prevent potential MITM attacks
245
231
  if @ssl_verify_peer
246
- https.ca_file = @ssl_ca_bundle_path
232
+ https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
233
+ https.ssl_version = :TLSv1_2
247
234
  https.verify_mode = OpenSSL::SSL::VERIFY_PEER
248
235
  https.verify_depth = @ssl_verify_depth
249
236
  end
250
237
 
251
- #use compression if enabled
238
+ # use compression if enabled
252
239
  encoding = nil
253
240
 
254
241
  if @compression_type
255
- if @compression_type == 'deflate'
256
- encoding = 'deflate'
242
+ if @compression_type == "deflate"
243
+ encoding = "deflate"
257
244
  body = Zlib::Deflate.deflate(body, @compression_level)
258
- elsif @compression_type == 'bz2'
259
- encoding = 'bz2'
245
+ elsif @compression_type == "bz2"
246
+ encoding = "bz2"
260
247
  io = StringIO.new
261
248
  bz2 = RBzip2.default_adapter::Compressor.new io
262
249
  bz2.write body
@@ -266,179 +253,157 @@ module Scalyr
266
253
  end
267
254
 
268
255
  post = Net::HTTP::Post.new uri.path
269
- post.add_field( 'Content-Type', 'application/json' )
256
+ post.add_field("Content-Type", "application/json")
270
257
 
271
- if @compression_type
272
- post.add_field( 'Content-Encoding', encoding )
273
- end
258
+ post.add_field("Content-Encoding", encoding) if @compression_type
274
259
 
275
260
  post.body = body
276
261
 
277
- https.request( post )
278
-
262
+ https.request(post)
279
263
  end
280
264
 
281
- def handle_response( response )
265
+ def handle_response(response)
282
266
  $log.debug "Response Code: #{response.code}"
283
267
  $log.debug "Response Body: #{response.body}"
284
268
 
285
- response_hash = Hash.new
269
+ response_hash = {}
286
270
 
287
271
  begin
288
- response_hash = JSON.parse( response.body )
289
- rescue
272
+ response_hash = JSON.parse(response.body)
273
+ rescue StandardError
290
274
  response_hash["status"] = "Invalid JSON response from server"
291
275
  end
292
276
 
293
- #make sure the JSON reponse has a "status" field
294
- if !response_hash.key? "status"
277
+ # make sure the JSON reponse has a "status" field
278
+ unless response_hash.key? "status"
295
279
  $log.debug "JSON response does not contain status message"
296
280
  raise Scalyr::ServerError.new "JSON response does not contain status message"
297
281
  end
298
282
 
299
283
  status = response_hash["status"]
300
284
 
301
- #4xx codes are handled separately
285
+ # 4xx codes are handled separately
302
286
  if response.code =~ /^4\d\d/
303
287
  raise Scalyr::Client4xxError.new status
304
288
  else
305
- if status != "success"
289
+ if status != "success" # rubocop:disable Style/IfInsideElse
306
290
  if status =~ /discardBuffer/
307
291
  $log.warn "Received 'discardBuffer' message from server. Buffer dropped."
308
- elsif status =~ %r"/client/"i
292
+ elsif status =~ %r{/client/}i
309
293
  raise Scalyr::ClientError.new status
310
- else #don't check specifically for server, we assume all non-client errors are server errors
294
+ else # don't check specifically for server, we assume all non-client errors are server errors
311
295
  raise Scalyr::ServerError.new status
312
296
  end
313
- elsif !response.code.include? "200" #response code is a string not an int
297
+ elsif !response.code.include? "200" # response code is a string not an int
314
298
  raise Scalyr::ServerError
315
299
  end
316
300
  end
317
-
318
301
  end
319
302
 
320
- def build_add_events_body( chunk )
321
-
322
- #requests
323
- requests = Array.new
303
+ def build_add_events_body(chunk)
304
+ # requests
305
+ requests = []
324
306
 
325
- #set of unique scalyr threads for this chunk
326
- current_threads = Hash.new
307
+ # set of unique scalyr threads for this chunk
308
+ current_threads = {}
327
309
 
328
- #byte count
310
+ # byte count
329
311
  total_bytes = 0
330
312
 
331
- #create a Scalyr event object for each record in the chunk
332
- events = Array.new
333
- chunk.msgpack_each {|(tag, sec, nsec, record)|
334
-
335
- timestamp = self.to_nanos( sec, nsec )
336
-
337
- thread_id = 0
313
+ # create a Scalyr event object for each record in the chunk
314
+ events = []
315
+ chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
316
+ timestamp = to_nanos(sec, nsec)
338
317
 
339
- @sync.synchronize {
340
- #ensure timestamp is at least 1 nanosecond greater than the last one
341
- timestamp = [timestamp, @last_timestamp + 1].max
342
- @last_timestamp = timestamp
318
+ thread_id = tag
343
319
 
344
- #get thread id or add a new one if we haven't seen this tag before
345
- if @thread_ids.key? tag
346
- thread_id = @thread_ids[tag]
347
- else
348
- thread_id = @next_id
349
- @thread_ids[tag] = thread_id
350
- @next_id += 1
351
- end
352
- }
353
-
354
- #then update the map of threads for this chunk
320
+ # then update the map of threads for this chunk
355
321
  current_threads[tag] = thread_id
356
322
 
357
- #add a logfile field if one doesn't exist
358
- if !record.key? "logfile"
359
- record["logfile"] = "/fluentd/#{tag}"
360
- end
323
+ # add a logfile field if one doesn't exist
324
+ record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
361
325
 
362
- #append to list of events
363
- event = { :thread => thread_id.to_s,
364
- :ts => timestamp,
365
- :attrs => record
366
- }
326
+ # set per-event parser if it is configured
327
+ record["parser"] = @parser unless @parser.nil?
367
328
 
368
- #get json string of event to keep track of how many bytes we are sending
329
+ # append to list of events
330
+ event = {thread: thread_id.to_s,
331
+ ts: timestamp,
332
+ attrs: record}
333
+
334
+ # get json string of event to keep track of how many bytes we are sending
369
335
 
370
336
  begin
371
337
  event_json = event.to_json
372
338
  rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
373
- $log.warn "#{e.class}: #{e.message}"
339
+ $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
374
340
 
375
341
  # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
376
- time = Fluent::EventTime.new( sec, nsec )
342
+ time = Fluent::EventTime.new(sec, nsec)
377
343
  router.emit_error_event(tag, time, record, e)
378
344
 
345
+ # Print attribute values for debugging / troubleshooting purposes
346
+ $log.debug "Event attributes:"
347
+
379
348
  event[:attrs].each do |key, value|
380
- $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
381
- event[:attrs][key] = value.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
349
+ # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
350
+ $log.debug "\t#{key} (#{value.class}): '#{value}'"
382
351
  end
352
+
353
+ # Recursively re-encode and sanitize potentially bad string values
354
+ event[:attrs] = sanitize_and_reencode_value(event[:attrs])
383
355
  event_json = event.to_json
384
356
  end
385
357
 
386
- #generate new request if json size of events in the array exceed maximum request buffer size
358
+ # generate new request if json size of events in the array exceed maximum request buffer size
387
359
  append_event = true
388
360
  if total_bytes + event_json.bytesize > @max_request_buffer
389
- #make sure we always have at least one event
390
- if events.size == 0
361
+ # make sure we always have at least one event
362
+ if events.empty?
391
363
  events << event
392
364
  append_event = false
393
365
  end
394
- request = self.create_request( events, current_threads )
366
+ request = create_request(events, current_threads)
395
367
  requests << request
396
368
 
397
369
  total_bytes = 0
398
- current_threads = Hash.new
399
- events = Array.new
370
+ current_threads = {}
371
+ events = []
400
372
  end
401
373
 
402
- #if we haven't consumed the current event already
403
- #add it to the end of our array and keep track of the json bytesize
374
+ # if we haven't consumed the current event already
375
+ # add it to the end of our array and keep track of the json bytesize
404
376
  if append_event
405
377
  events << event
406
378
  total_bytes += event_json.bytesize
407
379
  end
408
-
409
380
  }
410
381
 
411
- #create a final request with any left over events
412
- request = self.create_request( events, current_threads )
382
+ # create a final request with any left over events
383
+ request = create_request(events, current_threads)
413
384
  requests << request
414
-
415
385
  end
416
386
 
417
- def create_request( events, current_threads )
418
- #build the scalyr thread objects
419
- threads = Array.new
387
+ def create_request(events, current_threads)
388
+ # build the scalyr thread objects
389
+ threads = []
420
390
  current_threads.each do |tag, id|
421
- threads << { :id => id.to_s,
422
- :name => "Fluentd: #{tag}"
423
- }
391
+ threads << {id: id.to_s,
392
+ name: "Fluentd: #{tag}"}
424
393
  end
425
394
 
426
- current_time = self.to_millis( Fluent::Engine.now )
395
+ current_time = to_millis(Fluent::Engine.now)
427
396
 
428
- body = { :token => @api_write_token,
429
- :client_timestamp => current_time.to_s,
430
- :session => @session,
431
- :events => events,
432
- :threads => threads
433
- }
397
+ body = {token: @api_write_token,
398
+ client_timestamp: current_time.to_s,
399
+ session: @session,
400
+ events: events,
401
+ threads: threads}
434
402
 
435
- #add server_attributes hash if it exists
436
- if @server_attributes
437
- body[:sessionInfo] = @server_attributes
438
- end
403
+ # add server_attributes hash if it exists
404
+ body[:sessionInfo] = @server_attributes if @server_attributes
439
405
 
440
- { :body => body.to_json, :record_count => events.size }
406
+ {body: body.to_json, record_count: events.size}
441
407
  end
442
-
443
408
  end
444
409
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Scalyr Output Plugin for Fluentd
3
5
  #
@@ -15,8 +17,6 @@
15
17
  # See the License for the specific language governing permissions and
16
18
  # limitations under the License.
17
19
 
18
-
19
-
20
20
  module Scalyr
21
21
  class ClientError < StandardError; end
22
22
  class Client4xxError < StandardError; end