fluent-plugin-scalyr 0.8.11 → 0.8.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +1 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_scalyr.rb +10 -3
- data/lib/fluent/plugin/scalyr_utils.rb +65 -0
- data/test/test_events.rb +82 -0
- data/test/test_utils.rb +100 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1e738c523b1cea2d7aa0976064cb4fa85379b63795f1230a965da934033bc619
|
4
|
+
data.tar.gz: '09241b0eabc17074c5b4effb5f53ba47d183bb97ad724a064d6e9cfa9d5e8641'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 238b972ec9340a013d473911c05955e0a39d03f706d215167f2b3837bf46610917d2741680fd698c5d1075d1f7a307debd3c7cf4111ebba07280110bff2239dc
|
7
|
+
data.tar.gz: 1c42dfccbfa8963652198b3b2c0da102feda2b1052370b3296c26c781f5ab1fcd0126b513f87139e055c437c82347c597fe930f3739b6f61a4f8ac7d1764996a
|
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.8.
|
1
|
+
0.8.12
|
@@ -19,6 +19,7 @@
|
|
19
19
|
|
20
20
|
require "fluent/plugin/output"
|
21
21
|
require "fluent/plugin/scalyr_exceptions"
|
22
|
+
require "fluent/plugin/scalyr_utils"
|
22
23
|
require "fluent/plugin_helper/compat_parameters"
|
23
24
|
require "json"
|
24
25
|
require "net/http"
|
@@ -331,16 +332,22 @@ module Scalyr
|
|
331
332
|
begin
|
332
333
|
event_json = event.to_json
|
333
334
|
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
334
|
-
$log.warn "#{e.class}: #{e.message}"
|
335
|
+
$log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
|
335
336
|
|
336
337
|
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
337
338
|
time = Fluent::EventTime.new(sec, nsec)
|
338
339
|
router.emit_error_event(tag, time, record, e)
|
339
340
|
|
341
|
+
# Print attribute values for debugging / troubleshooting purposes
|
342
|
+
$log.debug "Event attributes:"
|
343
|
+
|
340
344
|
event[:attrs].each do |key, value|
|
341
|
-
|
342
|
-
|
345
|
+
# NOTE: value doesn't always value.encoding attribute so we use .class which is always available
|
346
|
+
$log.debug "\t#{key} (#{value.class}): '#{value}'"
|
343
347
|
end
|
348
|
+
|
349
|
+
# Recursively re-encode and sanitize potentially bad string values
|
350
|
+
event[:attrs] = sanitize_and_reencode_value(event[:attrs])
|
344
351
|
event_json = event.to_json
|
345
352
|
end
|
346
353
|
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Scalyr Output Plugin for Fluentd
|
5
|
+
#
|
6
|
+
# Copyright (C) 2020 Scalyr, Inc.
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
|
20
|
+
module Scalyr
|
21
|
+
def sanitize_and_reencode_value(value)
|
22
|
+
# Method which recursively sanitizes the provided value and tries to re-encode all the strings as
|
23
|
+
# UTF-8 ignoring any bad unicode sequences
|
24
|
+
case value
|
25
|
+
when Hash
|
26
|
+
return sanitize_and_reencode_hash(value)
|
27
|
+
when Array
|
28
|
+
return sanitize_and_reencode_array(value)
|
29
|
+
when String
|
30
|
+
value = sanitize_and_reencode_string(value)
|
31
|
+
return value
|
32
|
+
end
|
33
|
+
|
34
|
+
# We only need to re-encode strings, for other value types (ints, nils,
|
35
|
+
# etc. no reencoding is needed)
|
36
|
+
value
|
37
|
+
end
|
38
|
+
|
39
|
+
def sanitize_and_reencode_array(array)
|
40
|
+
array.each_with_index do |value, index|
|
41
|
+
value = sanitize_and_reencode_value(value)
|
42
|
+
array[index] = value
|
43
|
+
end
|
44
|
+
|
45
|
+
array
|
46
|
+
end
|
47
|
+
|
48
|
+
def sanitize_and_reencode_hash(hash)
|
49
|
+
hash.each do |key, value|
|
50
|
+
hash[key] = sanitize_and_reencode_value(value)
|
51
|
+
end
|
52
|
+
|
53
|
+
hash
|
54
|
+
end
|
55
|
+
|
56
|
+
def sanitize_and_reencode_string(value)
|
57
|
+
# Function which sanitized the provided string value and tries to re-encode it as UTF-8
|
58
|
+
# ignoring any encoding error which could arise due to bad or partial unicode sequence
|
59
|
+
begin # rubocop:disable Style/RedundantBegin
|
60
|
+
value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
|
61
|
+
rescue # rubocop:disable Style/RescueStandardError
|
62
|
+
"failed-to-reencode-as-utf8"
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/test/test_events.rb
CHANGED
@@ -170,6 +170,88 @@ class EventsTest < Scalyr::ScalyrOutTest
|
|
170
170
|
assert_equal(mock_called, true, "mock method was never called!")
|
171
171
|
end
|
172
172
|
|
173
|
+
def test_build_add_events_body_non_json_serializable_value
|
174
|
+
d = create_driver
|
175
|
+
|
176
|
+
time = event_time("2015-04-01 10:00:00 UTC")
|
177
|
+
attrs = {"a" => 1}
|
178
|
+
attrs["int1"] = 1_601_923_119
|
179
|
+
attrs["int2"] = Integer(1_601_923_119)
|
180
|
+
attrs["int3"] = Integer(9_223_372_036_854_775_807)
|
181
|
+
attrs["int4"] = Integer(-1)
|
182
|
+
attrs["nil"] = nil
|
183
|
+
attrs["array"] = [1, 2, "a", "b", nil]
|
184
|
+
attrs["hash"] = {
|
185
|
+
"a" => "1",
|
186
|
+
"b" => "c"
|
187
|
+
}
|
188
|
+
attrs["logfile"] = "/some/log/file"
|
189
|
+
|
190
|
+
# This partial unicode sequence will fail encoding so we make sure it doesn't break the plugin
|
191
|
+
# and we correctly cast it to a value which we can send to the API
|
192
|
+
attrs["partial_unicode_sequence"] = "\xC2"
|
193
|
+
attrs["array_with_partial_unicode_sequence"] = [1, 2, "a", "b", nil, "7", "\xC2"]
|
194
|
+
attrs["nested_array_with_partial_unicode_sequence"] = [1, 2, "a", "b", nil, "7",
|
195
|
+
[8, 9, [10, "\xC2"]],
|
196
|
+
{"a" => 1, "b" => "\xC2"}]
|
197
|
+
attrs["hash_with_partial_unicode_sequence"] = {
|
198
|
+
"a" => "1",
|
199
|
+
"b" => "\xC2",
|
200
|
+
"c" => nil
|
201
|
+
}
|
202
|
+
attrs["nested_hash_with_partial_unicode_sequence"] = {
|
203
|
+
"a" => "1",
|
204
|
+
"b" => {
|
205
|
+
"c" => "\xC2",
|
206
|
+
"d" => "e",
|
207
|
+
"f" => nil,
|
208
|
+
"g" => {
|
209
|
+
"h" => "\xC2",
|
210
|
+
"b" => 3
|
211
|
+
}
|
212
|
+
}
|
213
|
+
}
|
214
|
+
|
215
|
+
response = flexmock(Net::HTTPResponse, code: "200", body: '{ "status":"success" }')
|
216
|
+
mock = flexmock(d.instance)
|
217
|
+
|
218
|
+
mock_called = false
|
219
|
+
|
220
|
+
# NOTE: We need to perform a deep clone / copy
|
221
|
+
expected_attrs = Marshal.load(Marshal.dump(attrs))
|
222
|
+
|
223
|
+
expected_attrs["partial_unicode_sequence"] = "<?>"
|
224
|
+
expected_attrs["array_with_partial_unicode_sequence"][-1] = "<?>"
|
225
|
+
expected_attrs["nested_array_with_partial_unicode_sequence"][-2][-1][-1] = "<?>"
|
226
|
+
expected_attrs["nested_array_with_partial_unicode_sequence"][-1]["b"] = "<?>"
|
227
|
+
expected_attrs["hash_with_partial_unicode_sequence"]["b"] = "<?>"
|
228
|
+
expected_attrs["nested_hash_with_partial_unicode_sequence"]["b"]["c"] = "<?>"
|
229
|
+
expected_attrs["nested_hash_with_partial_unicode_sequence"]["b"]["g"]["h"] = "<?>"
|
230
|
+
|
231
|
+
# Verify that clone / copy was correct and the original object wasn't modified
|
232
|
+
assert_not_equal(expected_attrs, attrs, "Objects are the same but should be different")
|
233
|
+
assert_not_equal(Marshal.load(Marshal.dump(attrs)), Marshal.load(Marshal.dump(expected_attrs)))
|
234
|
+
assert_equal(attrs["partial_unicode_sequence"], "\xC2")
|
235
|
+
assert_equal(attrs["array_with_partial_unicode_sequence"][-1], "\xC2")
|
236
|
+
assert_equal(attrs["nested_hash_with_partial_unicode_sequence"]["b"]["g"]["h"], "\xC2")
|
237
|
+
|
238
|
+
mock.should_receive(:post_request).with(
|
239
|
+
URI,
|
240
|
+
on {|request_body|
|
241
|
+
body = JSON.parse(request_body)
|
242
|
+
assert_equal(expected_attrs, body["events"][0]["attrs"], "Value of attrs differs from log")
|
243
|
+
mock_called = true
|
244
|
+
true
|
245
|
+
}
|
246
|
+
).once.and_return(response)
|
247
|
+
|
248
|
+
d.run(default_tag: "test") do
|
249
|
+
d.feed(time, attrs)
|
250
|
+
end
|
251
|
+
|
252
|
+
assert_equal(mock_called, true, "mock method was never called!")
|
253
|
+
end
|
254
|
+
|
173
255
|
def test_default_message_field
|
174
256
|
d = create_driver CONFIG
|
175
257
|
|
data/test/test_utils.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Scalyr Output Plugin for Fluentd
|
5
|
+
#
|
6
|
+
# Copyright (C) 2015 Scalyr, Inc.
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
|
20
|
+
require "helper"
|
21
|
+
require "flexmock/test_unit"
|
22
|
+
|
23
|
+
require "fluent/plugin/scalyr_utils"
|
24
|
+
include Scalyr # rubocop:disable Style/MixinUsage
|
25
|
+
|
26
|
+
module Scalyr
|
27
|
+
class UtilsTest < Test::Unit::TestCase
|
28
|
+
def test_sanitize_and_reencode_value_simple_types
|
29
|
+
# Simple value - string
|
30
|
+
result = sanitize_and_reencode_value("test foo")
|
31
|
+
assert_equal("test foo", result)
|
32
|
+
|
33
|
+
# Simple value - string with bad unicode sequences
|
34
|
+
result = sanitize_and_reencode_value("test \xC2 foo \xC2 bar")
|
35
|
+
assert_equal("test <?> foo <?> bar", result)
|
36
|
+
|
37
|
+
# Simple value - int
|
38
|
+
result = sanitize_and_reencode_value(100)
|
39
|
+
assert_equal(100, result)
|
40
|
+
|
41
|
+
# Simple value - nill
|
42
|
+
result = sanitize_and_reencode_value(nil)
|
43
|
+
assert_equal(nil, result)
|
44
|
+
|
45
|
+
# Simple value - bool
|
46
|
+
result = sanitize_and_reencode_value(true)
|
47
|
+
assert_equal(true, result)
|
48
|
+
|
49
|
+
result = sanitize_and_reencode_value(false)
|
50
|
+
assert_equal(false, result)
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_sanitize_and_reencode_value_complex_nested_types
|
54
|
+
actual = [1, 2, "a", "b", nil, "7", "\xC2"]
|
55
|
+
expected = [1, 2, "a", "b", nil, "7", "<?>"]
|
56
|
+
|
57
|
+
result = sanitize_and_reencode_value(actual)
|
58
|
+
assert_equal(expected, result)
|
59
|
+
|
60
|
+
actual = [1, 2, "a", "b", nil, "7",
|
61
|
+
[8, 9, [10, "\xC2"]],
|
62
|
+
{"a" => 1, "b" => "\xC2"}]
|
63
|
+
expected = [1, 2, "a", "b", nil, "7",
|
64
|
+
[8, 9, [10, "<?>"]],
|
65
|
+
{"a" => 1, "b" => "<?>"}]
|
66
|
+
|
67
|
+
result = sanitize_and_reencode_value(actual)
|
68
|
+
assert_equal(expected, result)
|
69
|
+
|
70
|
+
actual = {
|
71
|
+
"a" => "1",
|
72
|
+
"b" => {
|
73
|
+
"c" => "\xC2",
|
74
|
+
"d" => "e",
|
75
|
+
"f" => nil,
|
76
|
+
"g" => {
|
77
|
+
"h" => "bar \xC2",
|
78
|
+
"b" => 3,
|
79
|
+
"l" => [1, 2, "foo\xC2", 3, 4, 5]
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
expected = {
|
84
|
+
"a" => "1",
|
85
|
+
"b" => {
|
86
|
+
"c" => "<?>",
|
87
|
+
"d" => "e",
|
88
|
+
"f" => nil,
|
89
|
+
"g" => {
|
90
|
+
"h" => "bar <?>",
|
91
|
+
"b" => 3,
|
92
|
+
"l" => [1, 2, "foo<?>", 3, 4, 5]
|
93
|
+
}
|
94
|
+
}
|
95
|
+
}
|
96
|
+
result = sanitize_and_reencode_value(actual)
|
97
|
+
assert_equal(expected, result)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Imron Alston
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -158,11 +158,13 @@ files:
|
|
158
158
|
- fluent.conf.sample
|
159
159
|
- lib/fluent/plugin/out_scalyr.rb
|
160
160
|
- lib/fluent/plugin/scalyr_exceptions.rb
|
161
|
+
- lib/fluent/plugin/scalyr_utils.rb
|
161
162
|
- test/helper.rb
|
162
163
|
- test/test_config.rb
|
163
164
|
- test/test_events.rb
|
164
165
|
- test/test_handle_response.rb
|
165
166
|
- test/test_ssl_verify.rb
|
167
|
+
- test/test_utils.rb
|
166
168
|
homepage: https://github.com/scalyr/scalyr-fluentd
|
167
169
|
licenses:
|
168
170
|
- Apache-2.0
|
@@ -191,4 +193,5 @@ test_files:
|
|
191
193
|
- test/test_events.rb
|
192
194
|
- test/test_handle_response.rb
|
193
195
|
- test/test_config.rb
|
196
|
+
- test/test_utils.rb
|
194
197
|
- test/test_ssl_verify.rb
|