fluent-plugin-scalyr 0.8.10 → 0.8.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/README.md +49 -17
- data/Rakefile +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +11 -7
- data/lib/fluent/plugin/out_scalyr.rb +228 -219
- data/lib/fluent/plugin/{scalyr-exceptions.rb → scalyr_exceptions.rb} +2 -2
- data/lib/fluent/plugin/scalyr_utils.rb +65 -0
- data/test/helper.rb +12 -6
- data/test/test_config.rb +24 -30
- data/test/test_events.rb +297 -141
- data/test/test_handle_response.rb +34 -35
- data/test/test_ssl_verify.rb +101 -10
- data/test/test_utils.rb +100 -0
- metadata +50 -33
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,47 +17,46 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
require
|
25
|
-
require
|
26
|
-
require
|
27
|
-
require
|
28
|
-
require
|
29
|
-
require
|
30
|
-
require 'thread'
|
31
|
-
|
20
|
+
require "fluent/plugin/output"
|
21
|
+
require "fluent/plugin/scalyr_exceptions"
|
22
|
+
require "fluent/plugin/scalyr_utils"
|
23
|
+
require "fluent/plugin_helper/compat_parameters"
|
24
|
+
require "json"
|
25
|
+
require "net/http"
|
26
|
+
require "net/https"
|
27
|
+
require "rbzip2"
|
28
|
+
require "stringio"
|
29
|
+
require "zlib"
|
30
|
+
require "securerandom"
|
31
|
+
require "socket"
|
32
32
|
module Scalyr
|
33
33
|
class ScalyrOut < Fluent::Plugin::Output
|
34
|
-
Fluent::Plugin.register_output(
|
34
|
+
Fluent::Plugin.register_output("scalyr", self)
|
35
35
|
helpers :compat_parameters
|
36
36
|
helpers :event_emitter
|
37
37
|
|
38
38
|
config_param :api_write_token, :string
|
39
|
-
config_param :server_attributes, :hash, :
|
40
|
-
config_param :
|
41
|
-
config_param :
|
42
|
-
config_param :
|
43
|
-
config_param :
|
44
|
-
config_param :
|
45
|
-
config_param :
|
46
|
-
config_param :
|
47
|
-
config_param :
|
48
|
-
config_param :
|
49
|
-
config_param :
|
50
|
-
config_param :
|
39
|
+
config_param :server_attributes, :hash, default: nil
|
40
|
+
config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
|
41
|
+
config_param :use_hostname_for_serverhost, :bool, default: true
|
42
|
+
config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
|
43
|
+
config_param :ssl_ca_bundle_path, :string, default: nil
|
44
|
+
config_param :ssl_verify_peer, :bool, default: true
|
45
|
+
config_param :ssl_verify_depth, :integer, default: 5
|
46
|
+
config_param :message_field, :string, default: "message"
|
47
|
+
config_param :max_request_buffer, :integer, default: 5_500_000
|
48
|
+
config_param :force_message_encoding, :string, default: nil
|
49
|
+
config_param :replace_invalid_utf8, :bool, default: false
|
50
|
+
config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
|
51
|
+
config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
|
51
52
|
|
52
53
|
config_section :buffer do
|
53
|
-
config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
|
54
|
-
config_set_default :retry_max_interval,
|
55
|
-
config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
|
56
|
-
config_set_default :flush_interval, 5 #default flush interval of 5 seconds
|
57
|
-
config_set_default :chunk_limit_size,
|
58
|
-
config_set_default :queue_limit_length, 1024 #default queue size of 1024
|
54
|
+
config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
|
55
|
+
config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
|
56
|
+
config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
|
57
|
+
config_set_default :flush_interval, 5 # default flush interval of 5 seconds
|
58
|
+
config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
|
59
|
+
config_set_default :queue_limit_length, 1024 # default queue size of 1024
|
59
60
|
end
|
60
61
|
|
61
62
|
# support for version 0.14.0:
|
@@ -71,46 +72,45 @@ module Scalyr
|
|
71
72
|
true
|
72
73
|
end
|
73
74
|
|
74
|
-
def configure(
|
75
|
-
|
76
|
-
|
77
|
-
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file"
|
75
|
+
def configure(conf)
|
76
|
+
if conf.elements("buffer").empty?
|
77
|
+
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
78
78
|
end
|
79
79
|
|
80
|
-
compat_parameters_buffer(
|
80
|
+
compat_parameters_buffer(conf, default_chunk_key: "")
|
81
81
|
|
82
82
|
super
|
83
83
|
|
84
|
-
if @buffer.chunk_limit_size >
|
85
|
-
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
|
84
|
+
if @buffer.chunk_limit_size > 6_000_000
|
85
|
+
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
86
86
|
end
|
87
87
|
|
88
|
-
if @max_request_buffer >
|
89
|
-
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
|
88
|
+
if @max_request_buffer > 6_000_000
|
89
|
+
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
90
90
|
end
|
91
91
|
|
92
92
|
@message_encoding = nil
|
93
|
-
if @force_message_encoding.to_s !=
|
93
|
+
if @force_message_encoding.to_s != ""
|
94
94
|
begin
|
95
|
-
@message_encoding = Encoding.find(
|
95
|
+
@message_encoding = Encoding.find(@force_message_encoding)
|
96
96
|
$log.debug "Forcing message encoding to '#{@force_message_encoding}'"
|
97
97
|
rescue ArgumentError
|
98
98
|
$log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
|
99
99
|
end
|
100
100
|
end
|
101
101
|
|
102
|
-
#evaluate any statements in string value of the server_attributes object
|
102
|
+
# evaluate any statements in string value of the server_attributes object
|
103
103
|
if @server_attributes
|
104
104
|
new_attributes = {}
|
105
105
|
@server_attributes.each do |key, value|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
106
|
+
next unless value.is_a?(String)
|
107
|
+
|
108
|
+
m = /^\#{(.*)}$/.match(value)
|
109
|
+
new_attributes[key] = if m
|
110
|
+
eval(m[1]) # rubocop:disable Security/Eval
|
111
|
+
else
|
112
|
+
value
|
113
|
+
end
|
114
114
|
end
|
115
115
|
@server_attributes = new_attributes
|
116
116
|
end
|
@@ -119,143 +119,131 @@ module Scalyr
|
|
119
119
|
if @use_hostname_for_serverhost
|
120
120
|
|
121
121
|
# ensure server_attributes is not nil
|
122
|
-
if @server_attributes.nil?
|
123
|
-
@server_attributes = {}
|
124
|
-
end
|
122
|
+
@server_attributes = {} if @server_attributes.nil?
|
125
123
|
|
126
124
|
# only set serverHost if it doesn't currently exist in server_attributes
|
127
125
|
# Note: Use strings rather than symbols for the key, because keys coming
|
128
126
|
# from the config file will be strings
|
129
|
-
|
130
|
-
@server_attributes[
|
127
|
+
unless @server_attributes.key? "serverHost"
|
128
|
+
@server_attributes["serverHost"] = Socket.gethostname
|
131
129
|
end
|
132
130
|
end
|
133
131
|
|
134
|
-
@scalyr_server <<
|
132
|
+
@scalyr_server << "/" unless @scalyr_server.end_with?("/")
|
135
133
|
|
136
134
|
@add_events_uri = URI @scalyr_server + "addEvents"
|
137
135
|
|
138
136
|
num_threads = @buffer_config.flush_thread_count
|
139
137
|
|
140
|
-
#forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
141
|
-
|
138
|
+
# forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
139
|
+
if num_threads > 1
|
140
|
+
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
|
141
|
+
end
|
142
142
|
end
|
143
143
|
|
144
144
|
def start
|
145
145
|
super
|
146
|
-
#Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
146
|
+
# Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
147
147
|
@session = SecureRandom.uuid
|
148
148
|
|
149
|
-
$log.info "Scalyr Fluentd Plugin ID id=#{
|
150
|
-
|
149
|
+
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
151
150
|
end
|
152
151
|
|
153
|
-
def format(
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
sec = components[0].to_i
|
166
|
-
nsec = (components[1] * 10**9).to_i
|
167
|
-
time = Fluent::EventTime.new( sec, nsec )
|
168
|
-
end
|
152
|
+
def format(tag, time, record)
|
153
|
+
time = Fluent::Engine.now if time.nil?
|
154
|
+
|
155
|
+
# handle timestamps that are not EventTime types
|
156
|
+
if time.is_a?(Integer)
|
157
|
+
time = Fluent::EventTime.new(time)
|
158
|
+
elsif time.is_a?(Float)
|
159
|
+
components = time.divmod 1 # get integer and decimal components
|
160
|
+
sec = components[0].to_i
|
161
|
+
nsec = (components[1] * 10**9).to_i
|
162
|
+
time = Fluent::EventTime.new(sec, nsec)
|
163
|
+
end
|
169
164
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
end
|
175
|
-
record["message"] = record[@message_field]
|
176
|
-
record.delete( @message_field )
|
165
|
+
if @message_field != "message"
|
166
|
+
if record.key? @message_field
|
167
|
+
if record.key? "message"
|
168
|
+
$log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
177
169
|
end
|
170
|
+
record["message"] = record[@message_field]
|
171
|
+
record.delete(@message_field)
|
178
172
|
end
|
173
|
+
end
|
179
174
|
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
end
|
175
|
+
if @message_encoding && record.key?("message") && record["message"]
|
176
|
+
if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
|
177
|
+
record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
|
+
else
|
179
|
+
record["message"].force_encoding(@message_encoding)
|
186
180
|
end
|
187
|
-
[tag, time.sec, time.nsec, record].to_msgpack
|
188
|
-
|
189
|
-
rescue JSON::GeneratorError
|
190
|
-
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
|
191
|
-
raise
|
192
181
|
end
|
182
|
+
[tag, time.sec, time.nsec, record].to_msgpack
|
183
|
+
rescue JSON::GeneratorError
|
184
|
+
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
|
185
|
+
raise
|
193
186
|
end
|
194
187
|
|
195
|
-
#called by fluentd when a chunk of log messages is ready
|
196
|
-
def write(
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
$log.debug "Chunk split into #{requests.size} request(s)."
|
201
|
-
|
202
|
-
requests.each_with_index { |request, index|
|
203
|
-
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
204
|
-
begin
|
205
|
-
response = self.post_request( @add_events_uri, request[:body] )
|
206
|
-
self.handle_response( response )
|
207
|
-
rescue OpenSSL::SSL::SSLError => e
|
208
|
-
if e.message.include? "certificate verify failed"
|
209
|
-
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
|
210
|
-
end
|
211
|
-
$log.warn e.message
|
212
|
-
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
213
|
-
rescue Scalyr::Client4xxError => e
|
214
|
-
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
|
188
|
+
# called by fluentd when a chunk of log messages is ready
|
189
|
+
def write(chunk)
|
190
|
+
$log.debug "Size of chunk is: #{chunk.size}"
|
191
|
+
requests = build_add_events_body(chunk)
|
192
|
+
$log.debug "Chunk split into #{requests.size} request(s)."
|
215
193
|
|
194
|
+
requests.each_with_index {|request, index|
|
195
|
+
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
196
|
+
begin
|
197
|
+
response = post_request(@add_events_uri, request[:body])
|
198
|
+
handle_response(response)
|
199
|
+
rescue OpenSSL::SSL::SSLError => e
|
200
|
+
if e.message.include? "certificate verify failed"
|
201
|
+
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
216
202
|
end
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
203
|
+
$log.warn e.message
|
204
|
+
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
205
|
+
rescue Scalyr::Client4xxError => e
|
206
|
+
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
|
207
|
+
end
|
208
|
+
}
|
209
|
+
rescue JSON::GeneratorError
|
210
|
+
$log.warn "Unable to format message due to JSON::GeneratorError."
|
211
|
+
raise
|
223
212
|
end
|
224
213
|
|
225
|
-
|
226
|
-
#
|
227
|
-
|
228
|
-
def to_nanos( seconds, nsec )
|
214
|
+
# explicit function to convert to nanoseconds
|
215
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
216
|
+
def to_nanos(seconds, nsec)
|
229
217
|
(seconds * 10**9) + nsec
|
230
218
|
end
|
231
219
|
|
232
|
-
#explicit function to convert to milliseconds
|
233
|
-
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
234
|
-
def to_millis(
|
220
|
+
# explicit function to convert to milliseconds
|
221
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
222
|
+
def to_millis(timestamp)
|
235
223
|
(timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
|
236
224
|
end
|
237
225
|
|
238
|
-
def post_request(
|
239
|
-
|
240
|
-
https = Net::HTTP.new( uri.host, uri.port )
|
226
|
+
def post_request(uri, body)
|
227
|
+
https = Net::HTTP.new(uri.host, uri.port)
|
241
228
|
https.use_ssl = true
|
242
229
|
|
243
|
-
#verify peers to prevent potential MITM attacks
|
230
|
+
# verify peers to prevent potential MITM attacks
|
244
231
|
if @ssl_verify_peer
|
245
|
-
https.ca_file = @ssl_ca_bundle_path
|
232
|
+
https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
|
233
|
+
https.ssl_version = :TLSv1_2
|
246
234
|
https.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
247
235
|
https.verify_depth = @ssl_verify_depth
|
248
236
|
end
|
249
237
|
|
250
|
-
#use compression if enabled
|
238
|
+
# use compression if enabled
|
251
239
|
encoding = nil
|
252
240
|
|
253
241
|
if @compression_type
|
254
|
-
if @compression_type ==
|
255
|
-
encoding =
|
242
|
+
if @compression_type == "deflate"
|
243
|
+
encoding = "deflate"
|
256
244
|
body = Zlib::Deflate.deflate(body, @compression_level)
|
257
|
-
elsif @compression_type ==
|
258
|
-
encoding =
|
245
|
+
elsif @compression_type == "bz2"
|
246
|
+
encoding = "bz2"
|
259
247
|
io = StringIO.new
|
260
248
|
bz2 = RBzip2.default_adapter::Compressor.new io
|
261
249
|
bz2.write body
|
@@ -265,164 +253,185 @@ module Scalyr
|
|
265
253
|
end
|
266
254
|
|
267
255
|
post = Net::HTTP::Post.new uri.path
|
268
|
-
post.add_field(
|
256
|
+
post.add_field("Content-Type", "application/json")
|
269
257
|
|
270
|
-
if @compression_type
|
271
|
-
post.add_field( 'Content-Encoding', encoding )
|
272
|
-
end
|
258
|
+
post.add_field("Content-Encoding", encoding) if @compression_type
|
273
259
|
|
274
260
|
post.body = body
|
275
261
|
|
276
|
-
https.request(
|
277
|
-
|
262
|
+
https.request(post)
|
278
263
|
end
|
279
264
|
|
280
|
-
def handle_response(
|
265
|
+
def handle_response(response)
|
281
266
|
$log.debug "Response Code: #{response.code}"
|
282
267
|
$log.debug "Response Body: #{response.body}"
|
283
268
|
|
284
|
-
response_hash =
|
269
|
+
response_hash = {}
|
285
270
|
|
286
271
|
begin
|
287
|
-
response_hash = JSON.parse(
|
288
|
-
rescue
|
272
|
+
response_hash = JSON.parse(response.body)
|
273
|
+
rescue StandardError
|
289
274
|
response_hash["status"] = "Invalid JSON response from server"
|
290
275
|
end
|
291
276
|
|
292
|
-
#make sure the JSON reponse has a "status" field
|
293
|
-
|
277
|
+
# make sure the JSON reponse has a "status" field
|
278
|
+
unless response_hash.key? "status"
|
294
279
|
$log.debug "JSON response does not contain status message"
|
295
280
|
raise Scalyr::ServerError.new "JSON response does not contain status message"
|
296
281
|
end
|
297
282
|
|
298
283
|
status = response_hash["status"]
|
299
284
|
|
300
|
-
#4xx codes are handled separately
|
285
|
+
# 4xx codes are handled separately
|
301
286
|
if response.code =~ /^4\d\d/
|
302
287
|
raise Scalyr::Client4xxError.new status
|
303
288
|
else
|
304
|
-
if status != "success"
|
289
|
+
if status != "success" # rubocop:disable Style/IfInsideElse
|
305
290
|
if status =~ /discardBuffer/
|
306
291
|
$log.warn "Received 'discardBuffer' message from server. Buffer dropped."
|
307
|
-
elsif status =~ %r
|
292
|
+
elsif status =~ %r{/client/}i
|
308
293
|
raise Scalyr::ClientError.new status
|
309
|
-
else #don't check specifically for server, we assume all non-client errors are server errors
|
294
|
+
else # don't check specifically for server, we assume all non-client errors are server errors
|
310
295
|
raise Scalyr::ServerError.new status
|
311
296
|
end
|
312
|
-
elsif !response.code.include? "200" #response code is a string not an int
|
297
|
+
elsif !response.code.include? "200" # response code is a string not an int
|
313
298
|
raise Scalyr::ServerError
|
314
299
|
end
|
315
300
|
end
|
316
|
-
|
317
301
|
end
|
318
302
|
|
319
|
-
def build_add_events_body(
|
303
|
+
def build_add_events_body(chunk)
|
304
|
+
# requests
|
305
|
+
requests = []
|
320
306
|
|
321
|
-
#
|
322
|
-
|
307
|
+
# set of unique scalyr threads for this chunk
|
308
|
+
current_threads = {}
|
323
309
|
|
324
|
-
#
|
325
|
-
current_threads = Hash.new
|
326
|
-
|
327
|
-
#byte count
|
310
|
+
# byte count
|
328
311
|
total_bytes = 0
|
329
312
|
|
330
|
-
#create a Scalyr event object for each record in the chunk
|
331
|
-
events =
|
332
|
-
chunk.msgpack_each {|(tag, sec, nsec, record)|
|
333
|
-
|
334
|
-
timestamp = self.to_nanos( sec, nsec )
|
313
|
+
# create a Scalyr event object for each record in the chunk
|
314
|
+
events = []
|
315
|
+
chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
|
316
|
+
timestamp = to_nanos(sec, nsec)
|
335
317
|
|
336
318
|
thread_id = tag
|
337
319
|
|
338
|
-
#then update the map of threads for this chunk
|
320
|
+
# then update the map of threads for this chunk
|
339
321
|
current_threads[tag] = thread_id
|
340
322
|
|
341
|
-
#add a logfile field if one doesn't exist
|
342
|
-
|
343
|
-
record["logfile"] = "/fluentd/#{tag}"
|
344
|
-
end
|
323
|
+
# add a logfile field if one doesn't exist
|
324
|
+
record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
|
345
325
|
|
346
|
-
#
|
347
|
-
|
348
|
-
:ts => timestamp,
|
349
|
-
:attrs => record
|
350
|
-
}
|
326
|
+
# set per-event parser if it is configured
|
327
|
+
record["parser"] = @parser unless @parser.nil?
|
351
328
|
|
352
|
-
#
|
329
|
+
# append to list of events
|
330
|
+
event = {thread: thread_id.to_s,
|
331
|
+
ts: timestamp,
|
332
|
+
attrs: record}
|
333
|
+
|
334
|
+
# get json string of event to keep track of how many bytes we are sending
|
353
335
|
|
354
336
|
begin
|
355
337
|
event_json = event.to_json
|
356
338
|
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
357
|
-
$log.warn "#{e.class}: #{e.message}"
|
339
|
+
$log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
|
358
340
|
|
359
341
|
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
360
|
-
time = Fluent::EventTime.new(
|
342
|
+
time = Fluent::EventTime.new(sec, nsec)
|
361
343
|
router.emit_error_event(tag, time, record, e)
|
362
344
|
|
345
|
+
# Print attribute values for debugging / troubleshooting purposes
|
346
|
+
$log.debug "Event attributes:"
|
347
|
+
|
363
348
|
event[:attrs].each do |key, value|
|
364
|
-
|
365
|
-
|
349
|
+
# NOTE: value doesn't always value.encoding attribute so we use .class which is always available
|
350
|
+
$log.debug "\t#{key} (#{value.class}): '#{value}'"
|
366
351
|
end
|
352
|
+
|
353
|
+
# Recursively re-encode and sanitize potentially bad string values
|
354
|
+
event[:attrs] = sanitize_and_reencode_value(event[:attrs])
|
367
355
|
event_json = event.to_json
|
368
356
|
end
|
369
357
|
|
370
|
-
#generate new request if json size of events in the array exceed maximum request buffer size
|
358
|
+
# generate new request if json size of events in the array exceed maximum request buffer size
|
371
359
|
append_event = true
|
372
360
|
if total_bytes + event_json.bytesize > @max_request_buffer
|
373
|
-
#
|
374
|
-
if events.
|
375
|
-
|
361
|
+
# the case where a single event causes us to exceed the @max_request_buffer
|
362
|
+
if events.empty?
|
363
|
+
# if we are able to truncate the content (and append an ellipsis)
|
364
|
+
# inside the @message_field we do so here
|
365
|
+
if record.key?(@message_field) &&
|
366
|
+
record[@message_field].is_a?(String) &&
|
367
|
+
record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
|
368
|
+
record[@message_field].bytesize >= 3
|
369
|
+
|
370
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
371
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
372
|
+
"is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
|
373
|
+
max_msg_size = @max_request_buffer - event_json.bytesize - 3
|
374
|
+
truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
|
375
|
+
event[:attrs][@message_field] = truncated_msg
|
376
|
+
events << event
|
377
|
+
|
378
|
+
# otherwise we drop the event and save ourselves hitting a 4XX response from the server
|
379
|
+
else
|
380
|
+
@log.warn "Received a record that cannot fit within max_request_buffer "\
|
381
|
+
"(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
|
382
|
+
"is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
|
383
|
+
"dropping event."
|
384
|
+
end
|
376
385
|
append_event = false
|
377
386
|
end
|
378
|
-
|
379
|
-
|
387
|
+
|
388
|
+
unless events.empty?
|
389
|
+
request = create_request(events, current_threads)
|
390
|
+
requests << request
|
391
|
+
end
|
380
392
|
|
381
393
|
total_bytes = 0
|
382
|
-
current_threads =
|
383
|
-
events =
|
394
|
+
current_threads = {}
|
395
|
+
events = []
|
384
396
|
end
|
385
397
|
|
386
|
-
#if we haven't consumed the current event already
|
387
|
-
#add it to the end of our array and keep track of the json bytesize
|
398
|
+
# if we haven't consumed the current event already
|
399
|
+
# add it to the end of our array and keep track of the json bytesize
|
388
400
|
if append_event
|
389
401
|
events << event
|
390
402
|
total_bytes += event_json.bytesize
|
391
403
|
end
|
392
|
-
|
393
404
|
}
|
394
405
|
|
395
|
-
#create a final request with any left over events
|
396
|
-
|
397
|
-
|
406
|
+
# create a final request with any left over events
|
407
|
+
unless events.empty?
|
408
|
+
request = create_request(events, current_threads)
|
409
|
+
requests << request
|
410
|
+
end
|
398
411
|
|
412
|
+
requests
|
399
413
|
end
|
400
414
|
|
401
|
-
def create_request(
|
402
|
-
#build the scalyr thread objects
|
403
|
-
threads =
|
415
|
+
def create_request(events, current_threads)
|
416
|
+
# build the scalyr thread objects
|
417
|
+
threads = []
|
404
418
|
current_threads.each do |tag, id|
|
405
|
-
threads << {
|
406
|
-
|
407
|
-
}
|
419
|
+
threads << {id: id.to_s,
|
420
|
+
name: "Fluentd: #{tag}"}
|
408
421
|
end
|
409
422
|
|
410
|
-
current_time =
|
423
|
+
current_time = to_millis(Fluent::Engine.now)
|
411
424
|
|
412
|
-
body = {
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
}
|
425
|
+
body = {token: @api_write_token,
|
426
|
+
client_timestamp: current_time.to_s,
|
427
|
+
session: @session,
|
428
|
+
events: events,
|
429
|
+
threads: threads}
|
418
430
|
|
419
|
-
#add server_attributes hash if it exists
|
420
|
-
if @server_attributes
|
421
|
-
body[:sessionInfo] = @server_attributes
|
422
|
-
end
|
431
|
+
# add server_attributes hash if it exists
|
432
|
+
body[:sessionInfo] = @server_attributes if @server_attributes
|
423
433
|
|
424
|
-
{
|
434
|
+
{body: body.to_json, record_count: events.size}
|
425
435
|
end
|
426
|
-
|
427
436
|
end
|
428
437
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,8 +17,6 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
|
20
20
|
module Scalyr
|
21
21
|
class ClientError < StandardError; end
|
22
22
|
class Client4xxError < StandardError; end
|