fluent-plugin-scalyr 0.8.9 → 0.8.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +2 -0
- data/README.md +59 -22
- data/Rakefile +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +11 -8
- data/fluent.conf.sample +1 -1
- data/lib/fluent/plugin/out_scalyr.rb +202 -237
- data/lib/fluent/plugin/{scalyr-exceptions.rb → scalyr_exceptions.rb} +2 -2
- data/lib/fluent/plugin/scalyr_utils.rb +65 -0
- data/test/helper.rb +12 -6
- data/test/test_config.rb +24 -30
- data/test/test_events.rb +254 -142
- data/test/test_handle_response.rb +34 -35
- data/test/test_ssl_verify.rb +101 -10
- data/test/test_utils.rb +100 -0
- metadata +50 -34
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,47 +17,46 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
require
|
25
|
-
require
|
26
|
-
require
|
27
|
-
require
|
28
|
-
require
|
29
|
-
require
|
30
|
-
require 'thread'
|
31
|
-
|
20
|
+
require "fluent/plugin/output"
|
21
|
+
require "fluent/plugin/scalyr_exceptions"
|
22
|
+
require "fluent/plugin/scalyr_utils"
|
23
|
+
require "fluent/plugin_helper/compat_parameters"
|
24
|
+
require "json"
|
25
|
+
require "net/http"
|
26
|
+
require "net/https"
|
27
|
+
require "rbzip2"
|
28
|
+
require "stringio"
|
29
|
+
require "zlib"
|
30
|
+
require "securerandom"
|
31
|
+
require "socket"
|
32
32
|
module Scalyr
|
33
33
|
class ScalyrOut < Fluent::Plugin::Output
|
34
|
-
Fluent::Plugin.register_output(
|
34
|
+
Fluent::Plugin.register_output("scalyr", self)
|
35
35
|
helpers :compat_parameters
|
36
36
|
helpers :event_emitter
|
37
37
|
|
38
38
|
config_param :api_write_token, :string
|
39
|
-
config_param :server_attributes, :hash, :
|
40
|
-
config_param :
|
41
|
-
config_param :
|
42
|
-
config_param :
|
43
|
-
config_param :
|
44
|
-
config_param :
|
45
|
-
config_param :
|
46
|
-
config_param :
|
47
|
-
config_param :
|
48
|
-
config_param :
|
49
|
-
config_param :
|
50
|
-
config_param :
|
39
|
+
config_param :server_attributes, :hash, default: nil
|
40
|
+
config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
|
41
|
+
config_param :use_hostname_for_serverhost, :bool, default: true
|
42
|
+
config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
|
43
|
+
config_param :ssl_ca_bundle_path, :string, default: nil
|
44
|
+
config_param :ssl_verify_peer, :bool, default: true
|
45
|
+
config_param :ssl_verify_depth, :integer, default: 5
|
46
|
+
config_param :message_field, :string, default: "message"
|
47
|
+
config_param :max_request_buffer, :integer, default: 5_500_000
|
48
|
+
config_param :force_message_encoding, :string, default: nil
|
49
|
+
config_param :replace_invalid_utf8, :bool, default: false
|
50
|
+
config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
|
51
|
+
config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
|
51
52
|
|
52
53
|
config_section :buffer do
|
53
|
-
config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
|
54
|
-
config_set_default :retry_max_interval,
|
55
|
-
config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
|
56
|
-
config_set_default :flush_interval, 5 #default flush interval of 5 seconds
|
57
|
-
config_set_default :chunk_limit_size,
|
58
|
-
config_set_default :queue_limit_length, 1024 #default queue size of 1024
|
54
|
+
config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
|
55
|
+
config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
|
56
|
+
config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
|
57
|
+
config_set_default :flush_interval, 5 # default flush interval of 5 seconds
|
58
|
+
config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
|
59
|
+
config_set_default :queue_limit_length, 1024 # default queue size of 1024
|
59
60
|
end
|
60
61
|
|
61
62
|
# support for version 0.14.0:
|
@@ -67,46 +68,49 @@ module Scalyr
|
|
67
68
|
true
|
68
69
|
end
|
69
70
|
|
70
|
-
def
|
71
|
+
def multi_workers_ready?
|
72
|
+
true
|
73
|
+
end
|
71
74
|
|
72
|
-
|
73
|
-
|
75
|
+
def configure(conf)
|
76
|
+
if conf.elements("buffer").empty?
|
77
|
+
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
74
78
|
end
|
75
79
|
|
76
|
-
compat_parameters_buffer(
|
80
|
+
compat_parameters_buffer(conf, default_chunk_key: "")
|
77
81
|
|
78
82
|
super
|
79
83
|
|
80
|
-
if @buffer.chunk_limit_size >
|
81
|
-
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
|
84
|
+
if @buffer.chunk_limit_size > 6_000_000
|
85
|
+
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
82
86
|
end
|
83
87
|
|
84
|
-
if @max_request_buffer >
|
85
|
-
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
|
88
|
+
if @max_request_buffer > 6_000_000
|
89
|
+
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
86
90
|
end
|
87
91
|
|
88
92
|
@message_encoding = nil
|
89
|
-
if @force_message_encoding.to_s !=
|
93
|
+
if @force_message_encoding.to_s != ""
|
90
94
|
begin
|
91
|
-
@message_encoding = Encoding.find(
|
95
|
+
@message_encoding = Encoding.find(@force_message_encoding)
|
92
96
|
$log.debug "Forcing message encoding to '#{@force_message_encoding}'"
|
93
97
|
rescue ArgumentError
|
94
98
|
$log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
|
95
99
|
end
|
96
100
|
end
|
97
101
|
|
98
|
-
#evaluate any statements in string value of the server_attributes object
|
102
|
+
# evaluate any statements in string value of the server_attributes object
|
99
103
|
if @server_attributes
|
100
104
|
new_attributes = {}
|
101
105
|
@server_attributes.each do |key, value|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
106
|
+
next unless value.is_a?(String)
|
107
|
+
|
108
|
+
m = /^\#{(.*)}$/.match(value)
|
109
|
+
new_attributes[key] = if m
|
110
|
+
eval(m[1]) # rubocop:disable Security/Eval
|
111
|
+
else
|
112
|
+
value
|
113
|
+
end
|
110
114
|
end
|
111
115
|
@server_attributes = new_attributes
|
112
116
|
end
|
@@ -115,148 +119,131 @@ module Scalyr
|
|
115
119
|
if @use_hostname_for_serverhost
|
116
120
|
|
117
121
|
# ensure server_attributes is not nil
|
118
|
-
if @server_attributes.nil?
|
119
|
-
@server_attributes = {}
|
120
|
-
end
|
122
|
+
@server_attributes = {} if @server_attributes.nil?
|
121
123
|
|
122
124
|
# only set serverHost if it doesn't currently exist in server_attributes
|
123
125
|
# Note: Use strings rather than symbols for the key, because keys coming
|
124
126
|
# from the config file will be strings
|
125
|
-
|
126
|
-
@server_attributes[
|
127
|
+
unless @server_attributes.key? "serverHost"
|
128
|
+
@server_attributes["serverHost"] = Socket.gethostname
|
127
129
|
end
|
128
130
|
end
|
129
131
|
|
130
|
-
@scalyr_server <<
|
132
|
+
@scalyr_server << "/" unless @scalyr_server.end_with?("/")
|
131
133
|
|
132
134
|
@add_events_uri = URI @scalyr_server + "addEvents"
|
133
135
|
|
134
136
|
num_threads = @buffer_config.flush_thread_count
|
135
137
|
|
136
|
-
#forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
137
|
-
|
138
|
+
# forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
139
|
+
if num_threads > 1
|
140
|
+
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
|
141
|
+
end
|
138
142
|
end
|
139
143
|
|
140
144
|
def start
|
141
145
|
super
|
142
|
-
|
143
|
-
#Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
146
|
+
# Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
144
147
|
@session = SecureRandom.uuid
|
145
148
|
|
146
|
-
@
|
147
|
-
#the following variables are all under the control of the above mutex
|
148
|
-
@thread_ids = Hash.new #hash of tags -> id
|
149
|
-
@next_id = 1 #incrementing thread id for the session
|
150
|
-
@last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
|
151
|
-
|
149
|
+
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
152
150
|
end
|
153
151
|
|
154
|
-
def format(
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
sec = components[0].to_i
|
167
|
-
nsec = (components[1] * 10**9).to_i
|
168
|
-
time = Fluent::EventTime.new( sec, nsec )
|
169
|
-
end
|
152
|
+
def format(tag, time, record)
|
153
|
+
time = Fluent::Engine.now if time.nil?
|
154
|
+
|
155
|
+
# handle timestamps that are not EventTime types
|
156
|
+
if time.is_a?(Integer)
|
157
|
+
time = Fluent::EventTime.new(time)
|
158
|
+
elsif time.is_a?(Float)
|
159
|
+
components = time.divmod 1 # get integer and decimal components
|
160
|
+
sec = components[0].to_i
|
161
|
+
nsec = (components[1] * 10**9).to_i
|
162
|
+
time = Fluent::EventTime.new(sec, nsec)
|
163
|
+
end
|
170
164
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
end
|
176
|
-
record["message"] = record[@message_field]
|
177
|
-
record.delete( @message_field )
|
165
|
+
if @message_field != "message"
|
166
|
+
if record.key? @message_field
|
167
|
+
if record.key? "message"
|
168
|
+
$log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
169
|
end
|
170
|
+
record["message"] = record[@message_field]
|
171
|
+
record.delete(@message_field)
|
179
172
|
end
|
173
|
+
end
|
180
174
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
end
|
175
|
+
if @message_encoding && record.key?("message") && record["message"]
|
176
|
+
if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
|
177
|
+
record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
|
+
else
|
179
|
+
record["message"].force_encoding(@message_encoding)
|
187
180
|
end
|
188
|
-
[tag, time.sec, time.nsec, record].to_msgpack
|
189
|
-
|
190
|
-
rescue JSON::GeneratorError
|
191
|
-
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
|
192
|
-
raise
|
193
181
|
end
|
182
|
+
[tag, time.sec, time.nsec, record].to_msgpack
|
183
|
+
rescue JSON::GeneratorError
|
184
|
+
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
|
185
|
+
raise
|
194
186
|
end
|
195
187
|
|
196
|
-
#called by fluentd when a chunk of log messages is ready
|
197
|
-
def write(
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
$log.debug "Chunk split into #{requests.size} request(s)."
|
202
|
-
|
203
|
-
requests.each_with_index { |request, index|
|
204
|
-
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
205
|
-
begin
|
206
|
-
response = self.post_request( @add_events_uri, request[:body] )
|
207
|
-
self.handle_response( response )
|
208
|
-
rescue OpenSSL::SSL::SSLError => e
|
209
|
-
if e.message.include? "certificate verify failed"
|
210
|
-
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
|
211
|
-
end
|
212
|
-
$log.warn e.message
|
213
|
-
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
214
|
-
rescue Scalyr::Client4xxError => e
|
215
|
-
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
|
188
|
+
# called by fluentd when a chunk of log messages is ready
|
189
|
+
def write(chunk)
|
190
|
+
$log.debug "Size of chunk is: #{chunk.size}"
|
191
|
+
requests = build_add_events_body(chunk)
|
192
|
+
$log.debug "Chunk split into #{requests.size} request(s)."
|
216
193
|
|
194
|
+
requests.each_with_index {|request, index|
|
195
|
+
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
196
|
+
begin
|
197
|
+
response = post_request(@add_events_uri, request[:body])
|
198
|
+
handle_response(response)
|
199
|
+
rescue OpenSSL::SSL::SSLError => e
|
200
|
+
if e.message.include? "certificate verify failed"
|
201
|
+
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
217
202
|
end
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
203
|
+
$log.warn e.message
|
204
|
+
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
205
|
+
rescue Scalyr::Client4xxError => e
|
206
|
+
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
|
207
|
+
end
|
208
|
+
}
|
209
|
+
rescue JSON::GeneratorError
|
210
|
+
$log.warn "Unable to format message due to JSON::GeneratorError."
|
211
|
+
raise
|
224
212
|
end
|
225
213
|
|
226
|
-
|
227
|
-
#
|
228
|
-
|
229
|
-
def to_nanos( seconds, nsec )
|
214
|
+
# explicit function to convert to nanoseconds
|
215
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
216
|
+
def to_nanos(seconds, nsec)
|
230
217
|
(seconds * 10**9) + nsec
|
231
218
|
end
|
232
219
|
|
233
|
-
#explicit function to convert to milliseconds
|
234
|
-
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
235
|
-
def to_millis(
|
220
|
+
# explicit function to convert to milliseconds
|
221
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
222
|
+
def to_millis(timestamp)
|
236
223
|
(timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
|
237
224
|
end
|
238
225
|
|
239
|
-
def post_request(
|
240
|
-
|
241
|
-
https = Net::HTTP.new( uri.host, uri.port )
|
226
|
+
def post_request(uri, body)
|
227
|
+
https = Net::HTTP.new(uri.host, uri.port)
|
242
228
|
https.use_ssl = true
|
243
229
|
|
244
|
-
#verify peers to prevent potential MITM attacks
|
230
|
+
# verify peers to prevent potential MITM attacks
|
245
231
|
if @ssl_verify_peer
|
246
|
-
https.ca_file = @ssl_ca_bundle_path
|
232
|
+
https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
|
233
|
+
https.ssl_version = :TLSv1_2
|
247
234
|
https.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
248
235
|
https.verify_depth = @ssl_verify_depth
|
249
236
|
end
|
250
237
|
|
251
|
-
#use compression if enabled
|
238
|
+
# use compression if enabled
|
252
239
|
encoding = nil
|
253
240
|
|
254
241
|
if @compression_type
|
255
|
-
if @compression_type ==
|
256
|
-
encoding =
|
242
|
+
if @compression_type == "deflate"
|
243
|
+
encoding = "deflate"
|
257
244
|
body = Zlib::Deflate.deflate(body, @compression_level)
|
258
|
-
elsif @compression_type ==
|
259
|
-
encoding =
|
245
|
+
elsif @compression_type == "bz2"
|
246
|
+
encoding = "bz2"
|
260
247
|
io = StringIO.new
|
261
248
|
bz2 = RBzip2.default_adapter::Compressor.new io
|
262
249
|
bz2.write body
|
@@ -266,179 +253,157 @@ module Scalyr
|
|
266
253
|
end
|
267
254
|
|
268
255
|
post = Net::HTTP::Post.new uri.path
|
269
|
-
post.add_field(
|
256
|
+
post.add_field("Content-Type", "application/json")
|
270
257
|
|
271
|
-
if @compression_type
|
272
|
-
post.add_field( 'Content-Encoding', encoding )
|
273
|
-
end
|
258
|
+
post.add_field("Content-Encoding", encoding) if @compression_type
|
274
259
|
|
275
260
|
post.body = body
|
276
261
|
|
277
|
-
https.request(
|
278
|
-
|
262
|
+
https.request(post)
|
279
263
|
end
|
280
264
|
|
281
|
-
def handle_response(
|
265
|
+
def handle_response(response)
|
282
266
|
$log.debug "Response Code: #{response.code}"
|
283
267
|
$log.debug "Response Body: #{response.body}"
|
284
268
|
|
285
|
-
response_hash =
|
269
|
+
response_hash = {}
|
286
270
|
|
287
271
|
begin
|
288
|
-
response_hash = JSON.parse(
|
289
|
-
rescue
|
272
|
+
response_hash = JSON.parse(response.body)
|
273
|
+
rescue StandardError
|
290
274
|
response_hash["status"] = "Invalid JSON response from server"
|
291
275
|
end
|
292
276
|
|
293
|
-
#make sure the JSON reponse has a "status" field
|
294
|
-
|
277
|
+
# make sure the JSON reponse has a "status" field
|
278
|
+
unless response_hash.key? "status"
|
295
279
|
$log.debug "JSON response does not contain status message"
|
296
280
|
raise Scalyr::ServerError.new "JSON response does not contain status message"
|
297
281
|
end
|
298
282
|
|
299
283
|
status = response_hash["status"]
|
300
284
|
|
301
|
-
#4xx codes are handled separately
|
285
|
+
# 4xx codes are handled separately
|
302
286
|
if response.code =~ /^4\d\d/
|
303
287
|
raise Scalyr::Client4xxError.new status
|
304
288
|
else
|
305
|
-
if status != "success"
|
289
|
+
if status != "success" # rubocop:disable Style/IfInsideElse
|
306
290
|
if status =~ /discardBuffer/
|
307
291
|
$log.warn "Received 'discardBuffer' message from server. Buffer dropped."
|
308
|
-
elsif status =~ %r
|
292
|
+
elsif status =~ %r{/client/}i
|
309
293
|
raise Scalyr::ClientError.new status
|
310
|
-
else #don't check specifically for server, we assume all non-client errors are server errors
|
294
|
+
else # don't check specifically for server, we assume all non-client errors are server errors
|
311
295
|
raise Scalyr::ServerError.new status
|
312
296
|
end
|
313
|
-
elsif !response.code.include? "200" #response code is a string not an int
|
297
|
+
elsif !response.code.include? "200" # response code is a string not an int
|
314
298
|
raise Scalyr::ServerError
|
315
299
|
end
|
316
300
|
end
|
317
|
-
|
318
301
|
end
|
319
302
|
|
320
|
-
def build_add_events_body(
|
321
|
-
|
322
|
-
|
323
|
-
requests = Array.new
|
303
|
+
def build_add_events_body(chunk)
|
304
|
+
# requests
|
305
|
+
requests = []
|
324
306
|
|
325
|
-
#set of unique scalyr threads for this chunk
|
326
|
-
current_threads =
|
307
|
+
# set of unique scalyr threads for this chunk
|
308
|
+
current_threads = {}
|
327
309
|
|
328
|
-
#byte count
|
310
|
+
# byte count
|
329
311
|
total_bytes = 0
|
330
312
|
|
331
|
-
#create a Scalyr event object for each record in the chunk
|
332
|
-
events =
|
333
|
-
chunk.msgpack_each {|(tag, sec, nsec, record)|
|
334
|
-
|
335
|
-
timestamp = self.to_nanos( sec, nsec )
|
336
|
-
|
337
|
-
thread_id = 0
|
313
|
+
# create a Scalyr event object for each record in the chunk
|
314
|
+
events = []
|
315
|
+
chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
|
316
|
+
timestamp = to_nanos(sec, nsec)
|
338
317
|
|
339
|
-
|
340
|
-
#ensure timestamp is at least 1 nanosecond greater than the last one
|
341
|
-
timestamp = [timestamp, @last_timestamp + 1].max
|
342
|
-
@last_timestamp = timestamp
|
318
|
+
thread_id = tag
|
343
319
|
|
344
|
-
|
345
|
-
if @thread_ids.key? tag
|
346
|
-
thread_id = @thread_ids[tag]
|
347
|
-
else
|
348
|
-
thread_id = @next_id
|
349
|
-
@thread_ids[tag] = thread_id
|
350
|
-
@next_id += 1
|
351
|
-
end
|
352
|
-
}
|
353
|
-
|
354
|
-
#then update the map of threads for this chunk
|
320
|
+
# then update the map of threads for this chunk
|
355
321
|
current_threads[tag] = thread_id
|
356
322
|
|
357
|
-
#add a logfile field if one doesn't exist
|
358
|
-
|
359
|
-
record["logfile"] = "/fluentd/#{tag}"
|
360
|
-
end
|
323
|
+
# add a logfile field if one doesn't exist
|
324
|
+
record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
|
361
325
|
|
362
|
-
#
|
363
|
-
|
364
|
-
:ts => timestamp,
|
365
|
-
:attrs => record
|
366
|
-
}
|
326
|
+
# set per-event parser if it is configured
|
327
|
+
record["parser"] = @parser unless @parser.nil?
|
367
328
|
|
368
|
-
#
|
329
|
+
# append to list of events
|
330
|
+
event = {thread: thread_id.to_s,
|
331
|
+
ts: timestamp,
|
332
|
+
attrs: record}
|
333
|
+
|
334
|
+
# get json string of event to keep track of how many bytes we are sending
|
369
335
|
|
370
336
|
begin
|
371
337
|
event_json = event.to_json
|
372
338
|
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
373
|
-
$log.warn "#{e.class}: #{e.message}"
|
339
|
+
$log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
|
374
340
|
|
375
341
|
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
376
|
-
time = Fluent::EventTime.new(
|
342
|
+
time = Fluent::EventTime.new(sec, nsec)
|
377
343
|
router.emit_error_event(tag, time, record, e)
|
378
344
|
|
345
|
+
# Print attribute values for debugging / troubleshooting purposes
|
346
|
+
$log.debug "Event attributes:"
|
347
|
+
|
379
348
|
event[:attrs].each do |key, value|
|
380
|
-
|
381
|
-
|
349
|
+
# NOTE: value doesn't always value.encoding attribute so we use .class which is always available
|
350
|
+
$log.debug "\t#{key} (#{value.class}): '#{value}'"
|
382
351
|
end
|
352
|
+
|
353
|
+
# Recursively re-encode and sanitize potentially bad string values
|
354
|
+
event[:attrs] = sanitize_and_reencode_value(event[:attrs])
|
383
355
|
event_json = event.to_json
|
384
356
|
end
|
385
357
|
|
386
|
-
#generate new request if json size of events in the array exceed maximum request buffer size
|
358
|
+
# generate new request if json size of events in the array exceed maximum request buffer size
|
387
359
|
append_event = true
|
388
360
|
if total_bytes + event_json.bytesize > @max_request_buffer
|
389
|
-
#make sure we always have at least one event
|
390
|
-
if events.
|
361
|
+
# make sure we always have at least one event
|
362
|
+
if events.empty?
|
391
363
|
events << event
|
392
364
|
append_event = false
|
393
365
|
end
|
394
|
-
request =
|
366
|
+
request = create_request(events, current_threads)
|
395
367
|
requests << request
|
396
368
|
|
397
369
|
total_bytes = 0
|
398
|
-
current_threads =
|
399
|
-
events =
|
370
|
+
current_threads = {}
|
371
|
+
events = []
|
400
372
|
end
|
401
373
|
|
402
|
-
#if we haven't consumed the current event already
|
403
|
-
#add it to the end of our array and keep track of the json bytesize
|
374
|
+
# if we haven't consumed the current event already
|
375
|
+
# add it to the end of our array and keep track of the json bytesize
|
404
376
|
if append_event
|
405
377
|
events << event
|
406
378
|
total_bytes += event_json.bytesize
|
407
379
|
end
|
408
|
-
|
409
380
|
}
|
410
381
|
|
411
|
-
#create a final request with any left over events
|
412
|
-
request =
|
382
|
+
# create a final request with any left over events
|
383
|
+
request = create_request(events, current_threads)
|
413
384
|
requests << request
|
414
|
-
|
415
385
|
end
|
416
386
|
|
417
|
-
def create_request(
|
418
|
-
#build the scalyr thread objects
|
419
|
-
threads =
|
387
|
+
def create_request(events, current_threads)
|
388
|
+
# build the scalyr thread objects
|
389
|
+
threads = []
|
420
390
|
current_threads.each do |tag, id|
|
421
|
-
threads << {
|
422
|
-
|
423
|
-
}
|
391
|
+
threads << {id: id.to_s,
|
392
|
+
name: "Fluentd: #{tag}"}
|
424
393
|
end
|
425
394
|
|
426
|
-
current_time =
|
395
|
+
current_time = to_millis(Fluent::Engine.now)
|
427
396
|
|
428
|
-
body = {
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
}
|
397
|
+
body = {token: @api_write_token,
|
398
|
+
client_timestamp: current_time.to_s,
|
399
|
+
session: @session,
|
400
|
+
events: events,
|
401
|
+
threads: threads}
|
434
402
|
|
435
|
-
#add server_attributes hash if it exists
|
436
|
-
if @server_attributes
|
437
|
-
body[:sessionInfo] = @server_attributes
|
438
|
-
end
|
403
|
+
# add server_attributes hash if it exists
|
404
|
+
body[:sessionInfo] = @server_attributes if @server_attributes
|
439
405
|
|
440
|
-
{
|
406
|
+
{body: body.to_json, record_count: events.size}
|
441
407
|
end
|
442
|
-
|
443
408
|
end
|
444
409
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,8 +17,6 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
|
20
20
|
module Scalyr
|
21
21
|
class ClientError < StandardError; end
|
22
22
|
class Client4xxError < StandardError; end
|