fluent-plugin-scalyr 0.8.9 → 0.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +2 -0
- data/README.md +59 -22
- data/Rakefile +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-scalyr.gemspec +11 -8
- data/fluent.conf.sample +1 -1
- data/lib/fluent/plugin/out_scalyr.rb +202 -237
- data/lib/fluent/plugin/{scalyr-exceptions.rb → scalyr_exceptions.rb} +2 -2
- data/lib/fluent/plugin/scalyr_utils.rb +65 -0
- data/test/helper.rb +12 -6
- data/test/test_config.rb +24 -30
- data/test/test_events.rb +254 -142
- data/test/test_handle_response.rb +34 -35
- data/test/test_ssl_verify.rb +101 -10
- data/test/test_utils.rb +100 -0
- metadata +50 -34
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,47 +17,46 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
require
|
25
|
-
require
|
26
|
-
require
|
27
|
-
require
|
28
|
-
require
|
29
|
-
require
|
30
|
-
require 'thread'
|
31
|
-
|
20
|
+
require "fluent/plugin/output"
|
21
|
+
require "fluent/plugin/scalyr_exceptions"
|
22
|
+
require "fluent/plugin/scalyr_utils"
|
23
|
+
require "fluent/plugin_helper/compat_parameters"
|
24
|
+
require "json"
|
25
|
+
require "net/http"
|
26
|
+
require "net/https"
|
27
|
+
require "rbzip2"
|
28
|
+
require "stringio"
|
29
|
+
require "zlib"
|
30
|
+
require "securerandom"
|
31
|
+
require "socket"
|
32
32
|
module Scalyr
|
33
33
|
class ScalyrOut < Fluent::Plugin::Output
|
34
|
-
Fluent::Plugin.register_output(
|
34
|
+
Fluent::Plugin.register_output("scalyr", self)
|
35
35
|
helpers :compat_parameters
|
36
36
|
helpers :event_emitter
|
37
37
|
|
38
38
|
config_param :api_write_token, :string
|
39
|
-
config_param :server_attributes, :hash, :
|
40
|
-
config_param :
|
41
|
-
config_param :
|
42
|
-
config_param :
|
43
|
-
config_param :
|
44
|
-
config_param :
|
45
|
-
config_param :
|
46
|
-
config_param :
|
47
|
-
config_param :
|
48
|
-
config_param :
|
49
|
-
config_param :
|
50
|
-
config_param :
|
39
|
+
config_param :server_attributes, :hash, default: nil
|
40
|
+
config_param :parser, :string, default: nil # Set the "parser" field to this, per event.
|
41
|
+
config_param :use_hostname_for_serverhost, :bool, default: true
|
42
|
+
config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
|
43
|
+
config_param :ssl_ca_bundle_path, :string, default: nil
|
44
|
+
config_param :ssl_verify_peer, :bool, default: true
|
45
|
+
config_param :ssl_verify_depth, :integer, default: 5
|
46
|
+
config_param :message_field, :string, default: "message"
|
47
|
+
config_param :max_request_buffer, :integer, default: 5_500_000
|
48
|
+
config_param :force_message_encoding, :string, default: nil
|
49
|
+
config_param :replace_invalid_utf8, :bool, default: false
|
50
|
+
config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
|
51
|
+
config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
|
51
52
|
|
52
53
|
config_section :buffer do
|
53
|
-
config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
|
54
|
-
config_set_default :retry_max_interval,
|
55
|
-
config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
|
56
|
-
config_set_default :flush_interval, 5 #default flush interval of 5 seconds
|
57
|
-
config_set_default :chunk_limit_size,
|
58
|
-
config_set_default :queue_limit_length, 1024 #default queue size of 1024
|
54
|
+
config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
|
55
|
+
config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
|
56
|
+
config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
|
57
|
+
config_set_default :flush_interval, 5 # default flush interval of 5 seconds
|
58
|
+
config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
|
59
|
+
config_set_default :queue_limit_length, 1024 # default queue size of 1024
|
59
60
|
end
|
60
61
|
|
61
62
|
# support for version 0.14.0:
|
@@ -67,46 +68,49 @@ module Scalyr
|
|
67
68
|
true
|
68
69
|
end
|
69
70
|
|
70
|
-
def
|
71
|
+
def multi_workers_ready?
|
72
|
+
true
|
73
|
+
end
|
71
74
|
|
72
|
-
|
73
|
-
|
75
|
+
def configure(conf)
|
76
|
+
if conf.elements("buffer").empty?
|
77
|
+
$log.warn "Pre 0.14.0 configuration file detected. Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
74
78
|
end
|
75
79
|
|
76
|
-
compat_parameters_buffer(
|
80
|
+
compat_parameters_buffer(conf, default_chunk_key: "")
|
77
81
|
|
78
82
|
super
|
79
83
|
|
80
|
-
if @buffer.chunk_limit_size >
|
81
|
-
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr"
|
84
|
+
if @buffer.chunk_limit_size > 6_000_000
|
85
|
+
$log.warn "Buffer chunk size is greater than 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
82
86
|
end
|
83
87
|
|
84
|
-
if @max_request_buffer >
|
85
|
-
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr"
|
88
|
+
if @max_request_buffer > 6_000_000
|
89
|
+
$log.warn "Maximum request buffer > 6Mb. This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
86
90
|
end
|
87
91
|
|
88
92
|
@message_encoding = nil
|
89
|
-
if @force_message_encoding.to_s !=
|
93
|
+
if @force_message_encoding.to_s != ""
|
90
94
|
begin
|
91
|
-
@message_encoding = Encoding.find(
|
95
|
+
@message_encoding = Encoding.find(@force_message_encoding)
|
92
96
|
$log.debug "Forcing message encoding to '#{@force_message_encoding}'"
|
93
97
|
rescue ArgumentError
|
94
98
|
$log.warn "No encoding '#{@force_message_encoding}' found. Ignoring"
|
95
99
|
end
|
96
100
|
end
|
97
101
|
|
98
|
-
#evaluate any statements in string value of the server_attributes object
|
102
|
+
# evaluate any statements in string value of the server_attributes object
|
99
103
|
if @server_attributes
|
100
104
|
new_attributes = {}
|
101
105
|
@server_attributes.each do |key, value|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
106
|
+
next unless value.is_a?(String)
|
107
|
+
|
108
|
+
m = /^\#{(.*)}$/.match(value)
|
109
|
+
new_attributes[key] = if m
|
110
|
+
eval(m[1]) # rubocop:disable Security/Eval
|
111
|
+
else
|
112
|
+
value
|
113
|
+
end
|
110
114
|
end
|
111
115
|
@server_attributes = new_attributes
|
112
116
|
end
|
@@ -115,148 +119,131 @@ module Scalyr
|
|
115
119
|
if @use_hostname_for_serverhost
|
116
120
|
|
117
121
|
# ensure server_attributes is not nil
|
118
|
-
if @server_attributes.nil?
|
119
|
-
@server_attributes = {}
|
120
|
-
end
|
122
|
+
@server_attributes = {} if @server_attributes.nil?
|
121
123
|
|
122
124
|
# only set serverHost if it doesn't currently exist in server_attributes
|
123
125
|
# Note: Use strings rather than symbols for the key, because keys coming
|
124
126
|
# from the config file will be strings
|
125
|
-
|
126
|
-
@server_attributes[
|
127
|
+
unless @server_attributes.key? "serverHost"
|
128
|
+
@server_attributes["serverHost"] = Socket.gethostname
|
127
129
|
end
|
128
130
|
end
|
129
131
|
|
130
|
-
@scalyr_server <<
|
132
|
+
@scalyr_server << "/" unless @scalyr_server.end_with?("/")
|
131
133
|
|
132
134
|
@add_events_uri = URI @scalyr_server + "addEvents"
|
133
135
|
|
134
136
|
num_threads = @buffer_config.flush_thread_count
|
135
137
|
|
136
|
-
#forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
137
|
-
|
138
|
+
# forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
|
139
|
+
if num_threads > 1
|
140
|
+
raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
|
141
|
+
end
|
138
142
|
end
|
139
143
|
|
140
144
|
def start
|
141
145
|
super
|
142
|
-
|
143
|
-
#Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
146
|
+
# Generate a session id. This will be called once for each <match> in fluent.conf that uses scalyr
|
144
147
|
@session = SecureRandom.uuid
|
145
148
|
|
146
|
-
@
|
147
|
-
#the following variables are all under the control of the above mutex
|
148
|
-
@thread_ids = Hash.new #hash of tags -> id
|
149
|
-
@next_id = 1 #incrementing thread id for the session
|
150
|
-
@last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
|
151
|
-
|
149
|
+
$log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
152
150
|
end
|
153
151
|
|
154
|
-
def format(
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
sec = components[0].to_i
|
167
|
-
nsec = (components[1] * 10**9).to_i
|
168
|
-
time = Fluent::EventTime.new( sec, nsec )
|
169
|
-
end
|
152
|
+
def format(tag, time, record)
|
153
|
+
time = Fluent::Engine.now if time.nil?
|
154
|
+
|
155
|
+
# handle timestamps that are not EventTime types
|
156
|
+
if time.is_a?(Integer)
|
157
|
+
time = Fluent::EventTime.new(time)
|
158
|
+
elsif time.is_a?(Float)
|
159
|
+
components = time.divmod 1 # get integer and decimal components
|
160
|
+
sec = components[0].to_i
|
161
|
+
nsec = (components[1] * 10**9).to_i
|
162
|
+
time = Fluent::EventTime.new(sec, nsec)
|
163
|
+
end
|
170
164
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
end
|
176
|
-
record["message"] = record[@message_field]
|
177
|
-
record.delete( @message_field )
|
165
|
+
if @message_field != "message"
|
166
|
+
if record.key? @message_field
|
167
|
+
if record.key? "message"
|
168
|
+
$log.warn "Overwriting log record field 'message'. You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
169
|
end
|
170
|
+
record["message"] = record[@message_field]
|
171
|
+
record.delete(@message_field)
|
179
172
|
end
|
173
|
+
end
|
180
174
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
end
|
175
|
+
if @message_encoding && record.key?("message") && record["message"]
|
176
|
+
if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
|
177
|
+
record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
178
|
+
else
|
179
|
+
record["message"].force_encoding(@message_encoding)
|
187
180
|
end
|
188
|
-
[tag, time.sec, time.nsec, record].to_msgpack
|
189
|
-
|
190
|
-
rescue JSON::GeneratorError
|
191
|
-
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record.to_s}"
|
192
|
-
raise
|
193
181
|
end
|
182
|
+
[tag, time.sec, time.nsec, record].to_msgpack
|
183
|
+
rescue JSON::GeneratorError
|
184
|
+
$log.warn "Unable to format message due to JSON::GeneratorError. Record is:\n\t#{record}"
|
185
|
+
raise
|
194
186
|
end
|
195
187
|
|
196
|
-
#called by fluentd when a chunk of log messages is ready
|
197
|
-
def write(
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
$log.debug "Chunk split into #{requests.size} request(s)."
|
202
|
-
|
203
|
-
requests.each_with_index { |request, index|
|
204
|
-
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
205
|
-
begin
|
206
|
-
response = self.post_request( @add_events_uri, request[:body] )
|
207
|
-
self.handle_response( response )
|
208
|
-
rescue OpenSSL::SSL::SSLError => e
|
209
|
-
if e.message.include? "certificate verify failed"
|
210
|
-
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
|
211
|
-
end
|
212
|
-
$log.warn e.message
|
213
|
-
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
214
|
-
rescue Scalyr::Client4xxError => e
|
215
|
-
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
|
188
|
+
# called by fluentd when a chunk of log messages is ready
|
189
|
+
def write(chunk)
|
190
|
+
$log.debug "Size of chunk is: #{chunk.size}"
|
191
|
+
requests = build_add_events_body(chunk)
|
192
|
+
$log.debug "Chunk split into #{requests.size} request(s)."
|
216
193
|
|
194
|
+
requests.each_with_index {|request, index|
|
195
|
+
$log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
|
196
|
+
begin
|
197
|
+
response = post_request(@add_events_uri, request[:body])
|
198
|
+
handle_response(response)
|
199
|
+
rescue OpenSSL::SSL::SSLError => e
|
200
|
+
if e.message.include? "certificate verify failed"
|
201
|
+
$log.warn "SSL certificate verification failed. Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
217
202
|
end
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
203
|
+
$log.warn e.message
|
204
|
+
$log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
|
205
|
+
rescue Scalyr::Client4xxError => e
|
206
|
+
$log.warn "4XX status code received for request #{index + 1}/#{requests.size}. Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
|
207
|
+
end
|
208
|
+
}
|
209
|
+
rescue JSON::GeneratorError
|
210
|
+
$log.warn "Unable to format message due to JSON::GeneratorError."
|
211
|
+
raise
|
224
212
|
end
|
225
213
|
|
226
|
-
|
227
|
-
#
|
228
|
-
|
229
|
-
def to_nanos( seconds, nsec )
|
214
|
+
# explicit function to convert to nanoseconds
|
215
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
216
|
+
def to_nanos(seconds, nsec)
|
230
217
|
(seconds * 10**9) + nsec
|
231
218
|
end
|
232
219
|
|
233
|
-
#explicit function to convert to milliseconds
|
234
|
-
#will make things easier to maintain if/when fluentd supports higher than second resolutions
|
235
|
-
def to_millis(
|
220
|
+
# explicit function to convert to milliseconds
|
221
|
+
# will make things easier to maintain if/when fluentd supports higher than second resolutions
|
222
|
+
def to_millis(timestamp)
|
236
223
|
(timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
|
237
224
|
end
|
238
225
|
|
239
|
-
def post_request(
|
240
|
-
|
241
|
-
https = Net::HTTP.new( uri.host, uri.port )
|
226
|
+
def post_request(uri, body)
|
227
|
+
https = Net::HTTP.new(uri.host, uri.port)
|
242
228
|
https.use_ssl = true
|
243
229
|
|
244
|
-
#verify peers to prevent potential MITM attacks
|
230
|
+
# verify peers to prevent potential MITM attacks
|
245
231
|
if @ssl_verify_peer
|
246
|
-
https.ca_file = @ssl_ca_bundle_path
|
232
|
+
https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
|
233
|
+
https.ssl_version = :TLSv1_2
|
247
234
|
https.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
248
235
|
https.verify_depth = @ssl_verify_depth
|
249
236
|
end
|
250
237
|
|
251
|
-
#use compression if enabled
|
238
|
+
# use compression if enabled
|
252
239
|
encoding = nil
|
253
240
|
|
254
241
|
if @compression_type
|
255
|
-
if @compression_type ==
|
256
|
-
encoding =
|
242
|
+
if @compression_type == "deflate"
|
243
|
+
encoding = "deflate"
|
257
244
|
body = Zlib::Deflate.deflate(body, @compression_level)
|
258
|
-
elsif @compression_type ==
|
259
|
-
encoding =
|
245
|
+
elsif @compression_type == "bz2"
|
246
|
+
encoding = "bz2"
|
260
247
|
io = StringIO.new
|
261
248
|
bz2 = RBzip2.default_adapter::Compressor.new io
|
262
249
|
bz2.write body
|
@@ -266,179 +253,157 @@ module Scalyr
|
|
266
253
|
end
|
267
254
|
|
268
255
|
post = Net::HTTP::Post.new uri.path
|
269
|
-
post.add_field(
|
256
|
+
post.add_field("Content-Type", "application/json")
|
270
257
|
|
271
|
-
if @compression_type
|
272
|
-
post.add_field( 'Content-Encoding', encoding )
|
273
|
-
end
|
258
|
+
post.add_field("Content-Encoding", encoding) if @compression_type
|
274
259
|
|
275
260
|
post.body = body
|
276
261
|
|
277
|
-
https.request(
|
278
|
-
|
262
|
+
https.request(post)
|
279
263
|
end
|
280
264
|
|
281
|
-
def handle_response(
|
265
|
+
def handle_response(response)
|
282
266
|
$log.debug "Response Code: #{response.code}"
|
283
267
|
$log.debug "Response Body: #{response.body}"
|
284
268
|
|
285
|
-
response_hash =
|
269
|
+
response_hash = {}
|
286
270
|
|
287
271
|
begin
|
288
|
-
response_hash = JSON.parse(
|
289
|
-
rescue
|
272
|
+
response_hash = JSON.parse(response.body)
|
273
|
+
rescue StandardError
|
290
274
|
response_hash["status"] = "Invalid JSON response from server"
|
291
275
|
end
|
292
276
|
|
293
|
-
#make sure the JSON reponse has a "status" field
|
294
|
-
|
277
|
+
# make sure the JSON reponse has a "status" field
|
278
|
+
unless response_hash.key? "status"
|
295
279
|
$log.debug "JSON response does not contain status message"
|
296
280
|
raise Scalyr::ServerError.new "JSON response does not contain status message"
|
297
281
|
end
|
298
282
|
|
299
283
|
status = response_hash["status"]
|
300
284
|
|
301
|
-
#4xx codes are handled separately
|
285
|
+
# 4xx codes are handled separately
|
302
286
|
if response.code =~ /^4\d\d/
|
303
287
|
raise Scalyr::Client4xxError.new status
|
304
288
|
else
|
305
|
-
if status != "success"
|
289
|
+
if status != "success" # rubocop:disable Style/IfInsideElse
|
306
290
|
if status =~ /discardBuffer/
|
307
291
|
$log.warn "Received 'discardBuffer' message from server. Buffer dropped."
|
308
|
-
elsif status =~ %r
|
292
|
+
elsif status =~ %r{/client/}i
|
309
293
|
raise Scalyr::ClientError.new status
|
310
|
-
else #don't check specifically for server, we assume all non-client errors are server errors
|
294
|
+
else # don't check specifically for server, we assume all non-client errors are server errors
|
311
295
|
raise Scalyr::ServerError.new status
|
312
296
|
end
|
313
|
-
elsif !response.code.include? "200" #response code is a string not an int
|
297
|
+
elsif !response.code.include? "200" # response code is a string not an int
|
314
298
|
raise Scalyr::ServerError
|
315
299
|
end
|
316
300
|
end
|
317
|
-
|
318
301
|
end
|
319
302
|
|
320
|
-
def build_add_events_body(
|
321
|
-
|
322
|
-
|
323
|
-
requests = Array.new
|
303
|
+
def build_add_events_body(chunk)
|
304
|
+
# requests
|
305
|
+
requests = []
|
324
306
|
|
325
|
-
#set of unique scalyr threads for this chunk
|
326
|
-
current_threads =
|
307
|
+
# set of unique scalyr threads for this chunk
|
308
|
+
current_threads = {}
|
327
309
|
|
328
|
-
#byte count
|
310
|
+
# byte count
|
329
311
|
total_bytes = 0
|
330
312
|
|
331
|
-
#create a Scalyr event object for each record in the chunk
|
332
|
-
events =
|
333
|
-
chunk.msgpack_each {|(tag, sec, nsec, record)|
|
334
|
-
|
335
|
-
timestamp = self.to_nanos( sec, nsec )
|
336
|
-
|
337
|
-
thread_id = 0
|
313
|
+
# create a Scalyr event object for each record in the chunk
|
314
|
+
events = []
|
315
|
+
chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
|
316
|
+
timestamp = to_nanos(sec, nsec)
|
338
317
|
|
339
|
-
|
340
|
-
#ensure timestamp is at least 1 nanosecond greater than the last one
|
341
|
-
timestamp = [timestamp, @last_timestamp + 1].max
|
342
|
-
@last_timestamp = timestamp
|
318
|
+
thread_id = tag
|
343
319
|
|
344
|
-
|
345
|
-
if @thread_ids.key? tag
|
346
|
-
thread_id = @thread_ids[tag]
|
347
|
-
else
|
348
|
-
thread_id = @next_id
|
349
|
-
@thread_ids[tag] = thread_id
|
350
|
-
@next_id += 1
|
351
|
-
end
|
352
|
-
}
|
353
|
-
|
354
|
-
#then update the map of threads for this chunk
|
320
|
+
# then update the map of threads for this chunk
|
355
321
|
current_threads[tag] = thread_id
|
356
322
|
|
357
|
-
#add a logfile field if one doesn't exist
|
358
|
-
|
359
|
-
record["logfile"] = "/fluentd/#{tag}"
|
360
|
-
end
|
323
|
+
# add a logfile field if one doesn't exist
|
324
|
+
record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
|
361
325
|
|
362
|
-
#
|
363
|
-
|
364
|
-
:ts => timestamp,
|
365
|
-
:attrs => record
|
366
|
-
}
|
326
|
+
# set per-event parser if it is configured
|
327
|
+
record["parser"] = @parser unless @parser.nil?
|
367
328
|
|
368
|
-
#
|
329
|
+
# append to list of events
|
330
|
+
event = {thread: thread_id.to_s,
|
331
|
+
ts: timestamp,
|
332
|
+
attrs: record}
|
333
|
+
|
334
|
+
# get json string of event to keep track of how many bytes we are sending
|
369
335
|
|
370
336
|
begin
|
371
337
|
event_json = event.to_json
|
372
338
|
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
373
|
-
$log.warn "#{e.class}: #{e.message}"
|
339
|
+
$log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
|
374
340
|
|
375
341
|
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
376
|
-
time = Fluent::EventTime.new(
|
342
|
+
time = Fluent::EventTime.new(sec, nsec)
|
377
343
|
router.emit_error_event(tag, time, record, e)
|
378
344
|
|
345
|
+
# Print attribute values for debugging / troubleshooting purposes
|
346
|
+
$log.debug "Event attributes:"
|
347
|
+
|
379
348
|
event[:attrs].each do |key, value|
|
380
|
-
|
381
|
-
|
349
|
+
# NOTE: value doesn't always value.encoding attribute so we use .class which is always available
|
350
|
+
$log.debug "\t#{key} (#{value.class}): '#{value}'"
|
382
351
|
end
|
352
|
+
|
353
|
+
# Recursively re-encode and sanitize potentially bad string values
|
354
|
+
event[:attrs] = sanitize_and_reencode_value(event[:attrs])
|
383
355
|
event_json = event.to_json
|
384
356
|
end
|
385
357
|
|
386
|
-
#generate new request if json size of events in the array exceed maximum request buffer size
|
358
|
+
# generate new request if json size of events in the array exceed maximum request buffer size
|
387
359
|
append_event = true
|
388
360
|
if total_bytes + event_json.bytesize > @max_request_buffer
|
389
|
-
#make sure we always have at least one event
|
390
|
-
if events.
|
361
|
+
# make sure we always have at least one event
|
362
|
+
if events.empty?
|
391
363
|
events << event
|
392
364
|
append_event = false
|
393
365
|
end
|
394
|
-
request =
|
366
|
+
request = create_request(events, current_threads)
|
395
367
|
requests << request
|
396
368
|
|
397
369
|
total_bytes = 0
|
398
|
-
current_threads =
|
399
|
-
events =
|
370
|
+
current_threads = {}
|
371
|
+
events = []
|
400
372
|
end
|
401
373
|
|
402
|
-
#if we haven't consumed the current event already
|
403
|
-
#add it to the end of our array and keep track of the json bytesize
|
374
|
+
# if we haven't consumed the current event already
|
375
|
+
# add it to the end of our array and keep track of the json bytesize
|
404
376
|
if append_event
|
405
377
|
events << event
|
406
378
|
total_bytes += event_json.bytesize
|
407
379
|
end
|
408
|
-
|
409
380
|
}
|
410
381
|
|
411
|
-
#create a final request with any left over events
|
412
|
-
request =
|
382
|
+
# create a final request with any left over events
|
383
|
+
request = create_request(events, current_threads)
|
413
384
|
requests << request
|
414
|
-
|
415
385
|
end
|
416
386
|
|
417
|
-
def create_request(
|
418
|
-
#build the scalyr thread objects
|
419
|
-
threads =
|
387
|
+
def create_request(events, current_threads)
|
388
|
+
# build the scalyr thread objects
|
389
|
+
threads = []
|
420
390
|
current_threads.each do |tag, id|
|
421
|
-
threads << {
|
422
|
-
|
423
|
-
}
|
391
|
+
threads << {id: id.to_s,
|
392
|
+
name: "Fluentd: #{tag}"}
|
424
393
|
end
|
425
394
|
|
426
|
-
current_time =
|
395
|
+
current_time = to_millis(Fluent::Engine.now)
|
427
396
|
|
428
|
-
body = {
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
}
|
397
|
+
body = {token: @api_write_token,
|
398
|
+
client_timestamp: current_time.to_s,
|
399
|
+
session: @session,
|
400
|
+
events: events,
|
401
|
+
threads: threads}
|
434
402
|
|
435
|
-
#add server_attributes hash if it exists
|
436
|
-
if @server_attributes
|
437
|
-
body[:sessionInfo] = @server_attributes
|
438
|
-
end
|
403
|
+
# add server_attributes hash if it exists
|
404
|
+
body[:sessionInfo] = @server_attributes if @server_attributes
|
439
405
|
|
440
|
-
{
|
406
|
+
{body: body.to_json, record_count: events.size}
|
441
407
|
end
|
442
|
-
|
443
408
|
end
|
444
409
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Scalyr Output Plugin for Fluentd
|
3
5
|
#
|
@@ -15,8 +17,6 @@
|
|
15
17
|
# See the License for the specific language governing permissions and
|
16
18
|
# limitations under the License.
|
17
19
|
|
18
|
-
|
19
|
-
|
20
20
|
module Scalyr
|
21
21
|
class ClientError < StandardError; end
|
22
22
|
class Client4xxError < StandardError; end
|