fluent-plugin-scalyr 0.8.10 → 0.8.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/README.md +49 -17
- data/Rakefile +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +11 -7
- data/lib/fluent/plugin/out_scalyr.rb +228 -219
- data/lib/fluent/plugin/{scalyr-exceptions.rb → scalyr_exceptions.rb} +2 -2
- data/lib/fluent/plugin/scalyr_utils.rb +65 -0
- data/test/helper.rb +12 -6
- data/test/test_config.rb +24 -30
- data/test/test_events.rb +297 -141
- data/test/test_handle_response.rb +34 -35
- data/test/test_ssl_verify.rb +101 -10
- data/test/test_utils.rb +100 -0
- metadata +50 -33
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,47 +17,46 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
require
|
25
|
-
require
|
26
|
-
require
|
27
|
-
require
|
28
|
-
require
|
29
|
-
require
|
30
|
-
require 'thread'
|
31
|
-
|
20
|
+
require "fluent/plugin/output"
|
21
|
+
require "fluent/plugin/scalyr_exceptions"
|
22
|
+
require "fluent/plugin/scalyr_utils"
|
23
|
+
require "fluent/plugin_helper/compat_parameters"
|
24
|
+
require "json"
|
25
|
+
require "net/http"
|
26
|
+
require "net/https"
|
27
|
+
require "rbzip2"
|
28
|
+
require "stringio"
|
29
|
+
require "zlib"
|
30
|
+
require "securerandom"
|
31
|
+
require "socket"
|
32
32
|
module Scalyr
|
33
33
|
class ScalyrOut < Fluent::Plugin::Output
|
34
|
-
Fluent::Plugin.register_output(
|
34
|
+
Fluent::Plugin.register_output("scalyr", self)
|
35
35
|
helpers :compat_parameters
|
36
36
|
helpers :event_emitter
|
37
37
|
|
38
38
|
config_param :api_write_token, :string
|
39
|
-
config_param :server_attributes, :hash, :
|
40
|
-
config_param :
|
41
|
-
config_param :
|
42
|
-
config_param :
|
43
|
-
config_param :
|
44
|
-
config_param :
|
45
|
-
config_param :
|
46
|
-
config_param :
|
47
|
-
config_param :
|
48
|
-
config_param :
|
49
|
-
config_param :
|
50
|
-
config_param :
|
39
|
+
config_param :server_attributes, :hash, default: nil
|
40
|
+
config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
|
41
|
+
config_param :use_hostname_for_serverhost, :bool, default: true
|
42
|
+
config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
|
43
|
+
config_param :ssl_ca_bundle_path, :string, default: nil
|
44
|
+
config_param :ssl_verify_peer, :bool, default: true
|
45
|
+
config_param :ssl_verify_depth, :integer, default: 5
|
46
|
+
config_param :message_field, :string, default: "message"
|
47
|
+
config_param :max_request_buffer, :integer, default: 5_500_000
|
48
|
+
config_param :force_message_encoding, :string, default: nil
|
49
|
+
config_param :replace_invalid_utf8, :bool, default: false
|
50
|
+
config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
|
51
|
+
config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
|
51
52
|
|
52
53
|
config_section :buffer do
|
53
|
-
config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
|
54
|
-
config_set_default :retry_max_interval,
|
55
|
-
config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
|
56
|
-
config_set_default :flush_interval, 5 #default flush interval of 5 seconds
|
57
|
-
config_set_default :chunk_limit_size,
|
58
|
-
config_set_default :queue_limit_length, 1024 #default queue size of 1024
|
54
|
+
config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
|
55
|
+
config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
|
56
|
+
config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
|
57
|
+
config_set_default :flush_interval, 5 # default flush interval of 5 seconds
|
58
|
+
config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
|
59
|
+
config_set_default :queue_limit_length, 1024 # default queue size of 1024
|
59
60
|
end
|
60
61
|
|
61
62
|
# support for version 0.14.0:
|
@@ -71,46 +72,45 @@ module Scalyr
|
|
71
72
|
true
|
72
73
|
end
|
73
74
|
|
74
|
-
def configure(
|
75
|
-
|
76
|
-
|
77
|
-
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
|
75
|
+
def configure(conf)
|
76
|
+
if conf.elements("buffer").empty?
|
77
|
+
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
78
78
|
end
|
79
79
|
|
80
|
-
compat_parameters_buffer(
|
80
|
+
compat_parameters_buffer(conf, default_chunk_key: "")
|
81
81
|
|
82
82
|
super
|
83
83
|
|
84
|
-
if @buffer.chunk_limit_size >
|
85
|
-
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
|
84
|
+
if @buffer.chunk_limit_size > 6_000_000
|
85
|
+
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
86
86
|
end
|
87
87
|
|
88
|
-
if @max_request_buffer >
|
89
|
-
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
|
88
|
+
if @max_request_buffer > 6_000_000
|
89
|
+
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
90
90
|
end
|
91
91
|
|
92
92
|
@message_encoding = nil
|
93
|
-
if @force_message_encoding.to_s !=
|
93
|
+
if @force_message_encoding.to_s != ""
|
94
94
|
begin
|
95
|
-
@message_encoding = Encoding.find(
|
95
|
+
@message_encoding = Encoding.find(@force_message_encoding)
|
96
96
|
$log.debug "Forcing message encoding to '#{@force_message_encoding}'"
|
97
97
|
rescue ArgumentError
|
98
98
|
$log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
|
99
99
|
end
|
100
100
|
end
|
101
101
|
|
102
|
-
#evaluate any statements in string value of the server_attributes object
|
102
|
+
# evaluate any statements in string value of the server_attributes object
|
103
103
|
if @server_attributes
|
104
104
|
new_attributes = {}
|
105
105
|
@server_attributes.each do |key, value|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
106
|
+
next unless value.is_a?(String)
|
107
|
+
|
108
|
+
m = /^\#{(.*)}$/.match(value)
|
109
|
+
new_attributes[key] = if m
|
110
|
+
eval(m[1]) # rubocop:disable Security/Eval
|
111
|
+
else
|
112
|
+
value
|
113
|
+
end
|
114
114
|
end
|
115
115
|
@server_attributes = new_attributes
|
116
116
|
end
|
@@ -119,143 +119,131 @@ module Scalyr
|
|
119
119
|
if @use_hostname_for_serverhost
|
120
120
|
|
121
121
|
# ensure server_attributes is not nil
|
122
|
-
if @server_attributes.nil?
|
123
|
-
@server_attributes = {}
|
124
|
-
end
|
122
|
+
@server_attributes = {} if @server_attributes.nil?
|
125
123
|
|
126
124
|
# only set serverHost if it doesn't currently exist in server_attributes
|
127
125
|
# Note: Use strings rather than symbols for the key, because keys coming
|
128
126
|
# from the config file will be strings
|
129
|
-
|
130
|
-
@server_attributes[
|
127
|
+
unless @server_attributes.key? "serverHost"
|
128
|
+
@server_attributes["serverHost"] = Socket.gethostname
|
131
129
|
end
|
132
130
|
end
|
133
131
|
|
134
|
-
@scalyr_server <<
|
132
|
+
@scalyr_server << "/" unless @scalyr_server.end_with?("/")
|
135
133
|
|
136
134
|
@add_events_uri = URI @scalyr_server + "addEvents"
|
137
135
|
|
138
136
|
num_threads = @buffer_config.flush_thread_count
|
139
137
|
|
140
|
-
#forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
141
|
-
|
138
|
+
# forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
139
|
+
if num_threads > 1
|
140
|
+
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
|
141
|
+
end
|
142
142
|
end
|
143
143
|
|
144
144
|
def start
|
145
145
|
super
|
146
|
-
#Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
146
|
+
# Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
147
147
|
@session = SecureRandom.uuid
|
148
148
|
|
149
|
-
$log.info "Scalyr Fluentd Plugin ID id=#{
|
150
|
-
|
149
|
+
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
151
150
|
end
|
152
151
|
|
153
|
-
def format(
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
sec = components[0].to_i
|
166
|
-
nsec = (components[1] * 10**9).to_i
|
167
|
-
time = Fluent::EventTime.new( sec, nsec )
|
168
|
-
end
|
152
|
+
def format(tag, time, record)
|
153
|
+
time = Fluent::Engine.now if time.nil?
|
154
|
+
|
155
|
+
# handle timestamps that are not EventTime types
|
156
|
+
if time.is_a?(Integer)
|
157
|
+
time = Fluent::EventTime.new(time)
|
158
|
+
elsif time.is_a?(Float)
|
159
|
+
components = time.divmod 1 # get integer and decimal components
|
160
|
+
sec = components[0].to_i
|
161
|
+
nsec = (components[1] * 10**9).to_i
|
162
|
+
time = Fluent::EventTime.new(sec, nsec)
|
163
|
+
end
|
169
164
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
end
|
175
|
-
record["message"] = record[@message_field]
|
176
|
-
record.delete( @message_field )
|
165
|
+
if @message_field != "message"
|
166
|
+
if record.key? @message_field
|
167
|
+
if record.key? "message"
|
168
|
+
$log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
177
169
|
end
|
170
|
+
record["message"] = record[@message_field]
|
171
|
+
record.delete(@message_field)
|
178
172
|
end
|
173
|
+
end
|
179
174
|
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
end
|
175
|
+
if @message_encoding && record.key?("message") && record["message"]
|
176
|
+
if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
|
177
|
+
record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
|
+
else
|
179
|
+
record["message"].force_encoding(@message_encoding)
|
186
180
|
end
|
187
|
-
[tag, time.sec, time.nsec, record].to_msgpack
|
188
|
-
|
189
|
-
rescue JSON::GeneratorError
|
190
|
-
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
|
191
|
-
raise
|
192
181
|
end
|
182
|
+
[tag, time.sec, time.nsec, record].to_msgpack
|
183
|
+
rescue JSON::GeneratorError
|
184
|
+
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
|
185
|
+
raise
|
193
186
|
end
|
194
187
|
|
195
|
-
#called by fluentd when a chunk of log messages is ready
|
196
|
-
def write(
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
$log.debug "Chunk split into #{requests.size} request(s)."
|
201
|
-
|
202
|
-
requests.each_with_index { |request, index|
|
203
|
-
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
204
|
-
begin
|
205
|
-
response = self.post_request( @add_events_uri, request[:body] )
|
206
|
-
self.handle_response( response )
|
207
|
-
rescue OpenSSL::SSL::SSLError => e
|
208
|
-
if e.message.include? "certificate verify failed"
|
209
|
-
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
|
210
|
-
end
|
211
|
-
$log.warn e.message
|
212
|
-
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
213
|
-
rescue Scalyr::Client4xxError => e
|
214
|
-
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
|
188
|
+
# called by fluentd when a chunk of log messages is ready
|
189
|
+
def write(chunk)
|
190
|
+
$log.debug "Size of chunk is: #{chunk.size}"
|
191
|
+
requests = build_add_events_body(chunk)
|
192
|
+
$log.debug "Chunk split into #{requests.size} request(s)."
|
215
193
|
|
194
|
+
requests.each_with_index {|request, index|
|
195
|
+
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
196
|
+
begin
|
197
|
+
response = post_request(@add_events_uri, request[:body])
|
198
|
+
handle_response(response)
|
199
|
+
rescue OpenSSL::SSL::SSLError => e
|
200
|
+
if e.message.include? "certificate verify failed"
|
201
|
+
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
216
202
|
end
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
203
|
+
$log.warn e.message
|
204
|
+
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
205
|
+
rescue Scalyr::Client4xxError => e
|
206
|
+
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
|
207
|
+
end
|
208
|
+
}
|
209
|
+
rescue JSON::GeneratorError
|
210
|
+
$log.warn "Unable to format message due to JSON::GeneratorError."
|
211
|
+
raise
|
223
212
|
end
|
224
213
|
|
225
|
-
|
226
|
-
#
|
227
|
-
|
228
|
-
def to_nanos( seconds, nsec )
|
214
|
+
# explicit function to convert to nanoseconds
|
215
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
216
|
+
def to_nanos(seconds, nsec)
|
229
217
|
(seconds * 10**9) + nsec
|
230
218
|
end
|
231
219
|
|
232
|
-
#explicit function to convert to milliseconds
|
233
|
-
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
234
|
-
def to_millis(
|
220
|
+
# explicit function to convert to milliseconds
|
221
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
222
|
+
def to_millis(timestamp)
|
235
223
|
(timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
|
236
224
|
end
|
237
225
|
|
238
|
-
def post_request(
|
239
|
-
|
240
|
-
https = Net::HTTP.new( uri.host, uri.port )
|
226
|
+
def post_request(uri, body)
|
227
|
+
https = Net::HTTP.new(uri.host, uri.port)
|
241
228
|
https.use_ssl = true
|
242
229
|
|
243
|
-
#verify peers to prevent potential MITM attacks
|
230
|
+
# verify peers to prevent potential MITM attacks
|
244
231
|
if @ssl_verify_peer
|
245
|
-
https.ca_file = @ssl_ca_bundle_path
|
232
|
+
https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
|
233
|
+
https.ssl_version = :TLSv1_2
|
246
234
|
https.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
247
235
|
https.verify_depth = @ssl_verify_depth
|
248
236
|
end
|
249
237
|
|
250
|
-
#use compression if enabled
|
238
|
+
# use compression if enabled
|
251
239
|
encoding = nil
|
252
240
|
|
253
241
|
if @compression_type
|
254
|
-
if @compression_type ==
|
255
|
-
encoding =
|
242
|
+
if @compression_type == "deflate"
|
243
|
+
encoding = "deflate"
|
256
244
|
body = Zlib::Deflate.deflate(body, @compression_level)
|
257
|
-
elsif @compression_type ==
|
258
|
-
encoding =
|
245
|
+
elsif @compression_type == "bz2"
|
246
|
+
encoding = "bz2"
|
259
247
|
io = StringIO.new
|
260
248
|
bz2 = RBzip2.default_adapter::Compressor.new io
|
261
249
|
bz2.write body
|
@@ -265,164 +253,185 @@ module Scalyr
|
|
265
253
|
end
|
266
254
|
|
267
255
|
post = Net::HTTP::Post.new uri.path
|
268
|
-
post.add_field(
|
256
|
+
post.add_field("Content-Type", "application/json")
|
269
257
|
|
270
|
-
if @compression_type
|
271
|
-
post.add_field( 'Content-Encoding', encoding )
|
272
|
-
end
|
258
|
+
post.add_field("Content-Encoding", encoding) if @compression_type
|
273
259
|
|
274
260
|
post.body = body
|
275
261
|
|
276
|
-
https.request(
|
277
|
-
|
262
|
+
https.request(post)
|
278
263
|
end
|
279
264
|
|
280
|
-
def handle_response(
|
265
|
+
def handle_response(response)
|
281
266
|
$log.debug "Response Code: #{response.code}"
|
282
267
|
$log.debug "Response Body: #{response.body}"
|
283
268
|
|
284
|
-
response_hash =
|
269
|
+
response_hash = {}
|
285
270
|
|
286
271
|
begin
|
287
|
-
response_hash = JSON.parse(
|
288
|
-
rescue
|
272
|
+
response_hash = JSON.parse(response.body)
|
273
|
+
rescue StandardError
|
289
274
|
response_hash["status"] = "Invalid JSON response from server"
|
290
275
|
end
|
291
276
|
|
292
|
-
#make sure the JSON reponse has a "status" field
|
293
|
-
|
277
|
+
# make sure the JSON reponse has a "status" field
|
278
|
+
unless response_hash.key? "status"
|
294
279
|
$log.debug "JSON response does not contain status message"
|
295
280
|
raise Scalyr::ServerError.new "JSON response does not contain status message"
|
296
281
|
end
|
297
282
|
|
298
283
|
status = response_hash["status"]
|
299
284
|
|
300
|
-
#4xx codes are handled separately
|
285
|
+
# 4xx codes are handled separately
|
301
286
|
if response.code =~ /^4\d\d/
|
302
287
|
raise Scalyr::Client4xxError.new status
|
303
288
|
else
|
304
|
-
if status != "success"
|
289
|
+
if status != "success" # rubocop:disable Style/IfInsideElse
|
305
290
|
if status =~ /discardBuffer/
|
306
291
|
$log.warn "Received 'discardBuffer' message from server. Buffer dropped."
|
307
|
-
elsif status =~ %r
|
292
|
+
elsif status =~ %r{/client/}i
|
308
293
|
raise Scalyr::ClientError.new status
|
309
|
-
else #don't check specifically for server, we assume all non-client errors are server errors
|
294
|
+
else # don't check specifically for server, we assume all non-client errors are server errors
|
310
295
|
raise Scalyr::ServerError.new status
|
311
296
|
end
|
312
|
-
elsif !response.code.include? "200" #response code is a string not an int
|
297
|
+
elsif !response.code.include? "200" # response code is a string not an int
|
313
298
|
raise Scalyr::ServerError
|
314
299
|
end
|
315
300
|
end
|
316
|
-
|
317
301
|
end
|
318
302
|
|
319
|
-
def build_add_events_body(
|
303
|
+
def build_add_events_body(chunk)
|
304
|
+
# requests
|
305
|
+
requests = []
|
320
306
|
|
321
|
-
#
|
322
|
-
|
307
|
+
# set of unique scalyr threads for this chunk
|
308
|
+
current_threads = {}
|
323
309
|
|
324
|
-
#
|
325
|
-
current_threads = Hash.new
|
326
|
-
|
327
|
-
#byte count
|
310
|
+
# byte count
|
328
311
|
total_bytes = 0
|
329
312
|
|
330
|
-
#create a Scalyr event object for each record in the chunk
|
331
|
-
events =
|
332
|
-
chunk.msgpack_each {|(tag, sec, nsec, record)|
|
333
|
-
|
334
|
-
timestamp = self.to_nanos( sec, nsec )
|
313
|
+
# create a Scalyr event object for each record in the chunk
|
314
|
+
events = []
|
315
|
+
chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
|
316
|
+
timestamp = to_nanos(sec, nsec)
|
335
317
|
|
336
318
|
thread_id = tag
|
337
319
|
|
338
|
-
#then update the map of threads for this chunk
|
320
|
+
# then update the map of threads for this chunk
|
339
321
|
current_threads[tag] = thread_id
|
340
322
|
|
341
|
-
#add a logfile field if one doesn't exist
|
342
|
-
|
343
|
-
record["logfile"] = "/fluentd/#{tag}"
|
344
|
-
end
|
323
|
+
# add a logfile field if one doesn't exist
|
324
|
+
record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
|
345
325
|
|
346
|
-
#
|
347
|
-
|
348
|
-
:ts => timestamp,
|
349
|
-
:attrs => record
|
350
|
-
}
|
326
|
+
# set per-event parser if it is configured
|
327
|
+
record["parser"] = @parser unless @parser.nil?
|
351
328
|
|
352
|
-
#
|
329
|
+
# append to list of events
|
330
|
+
event = {thread: thread_id.to_s,
|
331
|
+
ts: timestamp,
|
332
|
+
attrs: record}
|
333
|
+
|
334
|
+
# get json string of event to keep track of how many bytes we are sending
|
353
335
|
|
354
336
|
begin
|
355
337
|
event_json = event.to_json
|
356
338
|
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
357
|
-
$log.warn "#{e.class}: #{e.message}"
|
339
|
+
$log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
|
358
340
|
|
359
341
|
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
360
|
-
time = Fluent::EventTime.new(
|
342
|
+
time = Fluent::EventTime.new(sec, nsec)
|
361
343
|
router.emit_error_event(tag, time, record, e)
|
362
344
|
|
345
|
+
# Print attribute values for debugging / troubleshooting purposes
|
346
|
+
$log.debug "Event attributes:"
|
347
|
+
|
363
348
|
event[:attrs].each do |key, value|
|
364
|
-
|
365
|
-
|
349
|
+
# NOTE: value doesn't always value.encoding attribute so we use .class which is always available
|
350
|
+
$log.debug "\t#{key} (#{value.class}): '#{value}'"
|
366
351
|
end
|
352
|
+
|
353
|
+
# Recursively re-encode and sanitize potentially bad string values
|
354
|
+
event[:attrs] = sanitize_and_reencode_value(event[:attrs])
|
367
355
|
event_json = event.to_json
|
368
356
|
end
|
369
357
|
|
370
|
-
#generate new request if json size of events in the array exceed maximum request buffer size
|
358
|
+
# generate new request if json size of events in the array exceed maximum request buffer size
|
371
359
|
append_event = true
|
372
360
|
if total_bytes + event_json.bytesize > @max_request_buffer
|
373
|
-
#
|
374
|
-
if events.
|
375
|
-
|
361
|
+
# the case where a single event causes us to exceed the @max_request_buffer
|
362
|
+
if events.empty?
|
363
|
+
# if we are able to truncate the content (and append an ellipsis)
|
364
|
+
# inside the @message_field we do so here
|
365
|
+
if record.key?(@message_field) &&
|
366
|
+
record[@message_field].is_a?(String) &&
|
367
|
+
record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
|
368
|
+
record[@message_field].bytesize >= 3
|
369
|
+
|
370
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
371
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
372
|
+
"is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
|
373
|
+
max_msg_size = @max_request_buffer - event_json.bytesize - 3
|
374
|
+
truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
|
375
|
+
event[:attrs][@message_field] = truncated_msg
|
376
|
+
events << event
|
377
|
+
|
378
|
+
# otherwise we drop the event and save ourselves hitting a 4XX response from the server
|
379
|
+
else
|
380
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
381
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
382
|
+
"is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
|
383
|
+
"dropping event."
|
384
|
+
end
|
376
385
|
append_event = false
|
377
386
|
end
|
378
|
-
|
379
|
-
|
387
|
+
|
388
|
+
unless events.empty?
|
389
|
+
request = create_request(events, current_threads)
|
390
|
+
requests << request
|
391
|
+
end
|
380
392
|
|
381
393
|
total_bytes = 0
|
382
|
-
current_threads =
|
383
|
-
events =
|
394
|
+
current_threads = {}
|
395
|
+
events = []
|
384
396
|
end
|
385
397
|
|
386
|
-
#if we haven't consumed the current event already
|
387
|
-
#add it to the end of our array and keep track of the json bytesize
|
398
|
+
# if we haven't consumed the current event already
|
399
|
+
# add it to the end of our array and keep track of the json bytesize
|
388
400
|
if append_event
|
389
401
|
events << event
|
390
402
|
total_bytes += event_json.bytesize
|
391
403
|
end
|
392
|
-
|
393
404
|
}
|
394
405
|
|
395
|
-
#create a final request with any left over events
|
396
|
-
|
397
|
-
|
406
|
+
# create a final request with any left over events
|
407
|
+
unless events.empty?
|
408
|
+
request = create_request(events, current_threads)
|
409
|
+
requests << request
|
410
|
+
end
|
398
411
|
|
412
|
+
requests
|
399
413
|
end
|
400
414
|
|
401
|
-
def create_request(
|
402
|
-
#build the scalyr thread objects
|
403
|
-
threads =
|
415
|
+
def create_request(events, current_threads)
|
416
|
+
# build the scalyr thread objects
|
417
|
+
threads = []
|
404
418
|
current_threads.each do |tag, id|
|
405
|
-
threads << {
|
406
|
-
|
407
|
-
}
|
419
|
+
threads << {id: id.to_s,
|
420
|
+
name: "Fluentd: #{tag}"}
|
408
421
|
end
|
409
422
|
|
410
|
-
current_time =
|
423
|
+
current_time = to_millis(Fluent::Engine.now)
|
411
424
|
|
412
|
-
body = {
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
}
|
425
|
+
body = {token: @api_write_token,
|
426
|
+
client_timestamp: current_time.to_s,
|
427
|
+
session: @session,
|
428
|
+
events: events,
|
429
|
+
threads: threads}
|
418
430
|
|
419
|
-
#add server_attributes hash if it exists
|
420
|
-
if @server_attributes
|
421
|
-
body[:sessionInfo] = @server_attributes
|
422
|
-
end
|
431
|
+
# add server_attributes hash if it exists
|
432
|
+
body[:sessionInfo] = @server_attributes if @server_attributes
|
423
433
|
|
424
|
-
{
|
434
|
+
{body: body.to_json, record_count: events.size}
|
425
435
|
end
|
426
|
-
|
427
436
|
end
|
428
437
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,8 +17,6 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
|
20
20
|
module Scalyr
|
21
21
|
class ClientError < StandardError; end
|
22
22
|
class Client4xxError < StandardError; end
|