logstash-output-scalyr 0.2.7.beta → 0.2.9.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -0
- data/Gemfile +5 -2
- data/README.md +2 -4
- data/lib/logstash/outputs/scalyr.rb +192 -48
- data/lib/scalyr/common/client.rb +40 -4
- data/lib/scalyr/common/util.rb +5 -5
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +2 -2
- metadata +3 -23
- data/spec/benchmarks/bignum_fixing.rb +0 -87
- data/spec/benchmarks/flattening_and_serialization.rb +0 -100
- data/spec/benchmarks/json_serialization.rb +0 -85
- data/spec/benchmarks/metrics_overhead.rb +0 -48
- data/spec/benchmarks/set_session_level_serverhost_on_events.rb +0 -107
- data/spec/benchmarks/util.rb +0 -24
- data/spec/logstash/outputs/fixtures/example_com.pem +0 -41
- data/spec/logstash/outputs/scalyr_integration_spec.rb +0 -318
- data/spec/logstash/outputs/scalyr_spec.rb +0 -1262
- data/spec/scalyr/common/util_spec.rb +0 -543
@@ -1,318 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/outputs/scalyr"
|
4
|
-
require "logstash/codecs/plain"
|
5
|
-
require "logstash/event"
|
6
|
-
require "json"
|
7
|
-
require 'webmock/rspec'
|
8
|
-
|
9
|
-
# Require the specific version of `json` used in logstash
|
10
|
-
gem 'json', '1.8.6'
|
11
|
-
require 'json'
|
12
|
-
|
13
|
-
EXAMPLE_COME_CA_CERTS_PATH = File.expand_path(File.join(File.dirname(__FILE__), + "/fixtures/example_com.pem"))
|
14
|
-
|
15
|
-
WebMock.allow_net_connect!
|
16
|
-
|
17
|
-
RSpec.configure do |rspec|
|
18
|
-
rspec.expect_with :rspec do |c|
|
19
|
-
c.max_formatted_output_length = nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
describe LogStash::Outputs::Scalyr do
|
24
|
-
let(:sample_events) {
|
25
|
-
events = []
|
26
|
-
for i in 1..3 do
|
27
|
-
e = LogStash::Event.new
|
28
|
-
e.set('source_host', "my host #{i}")
|
29
|
-
e.set('source_file', "my file #{i}")
|
30
|
-
e.set('seq', i)
|
31
|
-
e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
|
32
|
-
e.set('tags', ['t1', 't2', 't3'])
|
33
|
-
events.push(e)
|
34
|
-
end
|
35
|
-
events
|
36
|
-
}
|
37
|
-
|
38
|
-
describe "#ssl_tests" do
|
39
|
-
context "with default SSL configuration" do
|
40
|
-
it "throws a ServerError due to fake api key" do
|
41
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
42
|
-
'api_write_token' => '1234',
|
43
|
-
'perform_connectivity_check' => false,
|
44
|
-
'max_retries' => 2,
|
45
|
-
'retry_max_interval' => 2,
|
46
|
-
'retry_initial_interval' => 0.2,
|
47
|
-
})
|
48
|
-
plugin.register
|
49
|
-
plugin.instance_variable_set(:@running, false)
|
50
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
51
|
-
plugin.multi_receive(sample_events)
|
52
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
|
53
|
-
{
|
54
|
-
:error_class=>"Scalyr::Common::Client::ServerError",
|
55
|
-
:batch_num=>1,
|
56
|
-
:code=>401,
|
57
|
-
:message=>"error/client/badParam",
|
58
|
-
:payload_size=>737,
|
59
|
-
:record_count=>3,
|
60
|
-
:total_batches=>1,
|
61
|
-
:url=>"https://agent.scalyr.com/addEvents",
|
62
|
-
:will_retry_in_seconds=>0.4,
|
63
|
-
:body=>"{\n \"message\": \"Couldn't decode API token ...234.\",\n \"status\": \"error/client/badParam\"\n}"
|
64
|
-
}
|
65
|
-
)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
context "when pointing at an invalid location (doesnt exist) without any valid certs" do
|
70
|
-
it "throws an Errno::ENOENT error" do
|
71
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
72
|
-
'api_write_token' => '1234',
|
73
|
-
'perform_connectivity_check' => false,
|
74
|
-
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
75
|
-
'max_retries' => 2,
|
76
|
-
'retry_max_interval' => 2,
|
77
|
-
'retry_initial_interval' => 0.2,
|
78
|
-
})
|
79
|
-
|
80
|
-
expect {
|
81
|
-
plugin.register
|
82
|
-
}.to raise_error(Errno::ENOENT, /Invalid path for ssl_ca_bundle_path config option - file doesn't exist or is not readable/)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
context "when pointing to an empty certs file" do
|
87
|
-
it "throws an SSLError" do
|
88
|
-
temp_file = file = Tempfile.new('emot_certs_file')
|
89
|
-
|
90
|
-
begin
|
91
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
92
|
-
'api_write_token' => '1234',
|
93
|
-
'perform_connectivity_check' => false,
|
94
|
-
'max_retries' => 2,
|
95
|
-
'retry_max_interval' => 2,
|
96
|
-
'retry_initial_interval' => 0.2,
|
97
|
-
'ssl_ca_bundle_path' => temp_file.path
|
98
|
-
})
|
99
|
-
plugin.register
|
100
|
-
plugin.instance_variable_set(:@running, false)
|
101
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
102
|
-
plugin.multi_receive(sample_events)
|
103
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
|
104
|
-
{
|
105
|
-
:error_class=>"Manticore::UnknownException",
|
106
|
-
:batch_num=>1,
|
107
|
-
:message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
|
108
|
-
#:message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
|
109
|
-
:payload_size=>737,
|
110
|
-
:record_count=>3,
|
111
|
-
:total_batches=>1,
|
112
|
-
:url=>"https://agent.scalyr.com/addEvents",
|
113
|
-
:will_retry_in_seconds=>0.4
|
114
|
-
}
|
115
|
-
)
|
116
|
-
end
|
117
|
-
ensure
|
118
|
-
temp_file.unlink
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
context "when server hostname doesn't match the cert" do
|
123
|
-
it "throws an SSLError" do
|
124
|
-
agent_scalyr_com_ip = `dig +short agent.scalyr.com 2> /dev/null | tail -n 1 | tr -d "\n"`
|
125
|
-
if agent_scalyr_com_ip.empty?
|
126
|
-
agent_scalyr_com_ip = `getent hosts agent.scalyr.com \
|
127
|
-
| awk '{ print $1 }' | tail -n 1 | tr -d "\n"`
|
128
|
-
end
|
129
|
-
mock_host = "invalid.mitm.should.fail.test.agent.scalyr.com"
|
130
|
-
etc_hosts_entry = "#{agent_scalyr_com_ip} #{mock_host}"
|
131
|
-
hosts_bkp = `sudo cat /etc/hosts`
|
132
|
-
hosts_bkp = hosts_bkp.chomp
|
133
|
-
# Add mock /etc/hosts entry and config scalyr_server entry
|
134
|
-
`echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
|
135
|
-
|
136
|
-
begin
|
137
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
138
|
-
'api_write_token' => '1234',
|
139
|
-
'perform_connectivity_check' => false,
|
140
|
-
'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
|
141
|
-
'max_retries' => 2,
|
142
|
-
'retry_max_interval' => 2,
|
143
|
-
'retry_initial_interval' => 0.2,
|
144
|
-
})
|
145
|
-
plugin.register
|
146
|
-
plugin.instance_variable_set(:@running, false)
|
147
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
148
|
-
plugin.multi_receive(sample_events)
|
149
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
|
150
|
-
{
|
151
|
-
:error_class=>"Manticore::UnknownException",
|
152
|
-
:batch_num=>1,
|
153
|
-
:message=>"Certificate for <invalid.mitm.should.fail.test.agent.scalyr.com> doesn't match any of the subject alternative names: [*.scalyr.com, scalyr.com]",
|
154
|
-
:payload_size=>737,
|
155
|
-
:record_count=>3,
|
156
|
-
:total_batches=>1,
|
157
|
-
:url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
|
158
|
-
:will_retry_in_seconds=>0.4
|
159
|
-
}
|
160
|
-
)
|
161
|
-
ensure
|
162
|
-
# Clean up the hosts file
|
163
|
-
`sudo truncate -s 0 /etc/hosts`
|
164
|
-
`echo "#{hosts_bkp}" | sudo tee -a /etc/hosts`
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
context "when an error occurs with retries at 15 and invalid example_com cert" do
|
170
|
-
it "exits after 15 retries and emits a log" do
|
171
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
172
|
-
'api_write_token' => '1234',
|
173
|
-
'perform_connectivity_check' => false,
|
174
|
-
'ssl_ca_bundle_path' => EXAMPLE_COME_CA_CERTS_PATH,
|
175
|
-
'max_retries' => 15,
|
176
|
-
'retry_max_interval' => 0.2,
|
177
|
-
'retry_initial_interval' => 0.1,
|
178
|
-
})
|
179
|
-
plugin.register
|
180
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
181
|
-
plugin.multi_receive(sample_events)
|
182
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 15 tries.", anything
|
183
|
-
)
|
184
|
-
end
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
describe "response_handling_tests" do
|
189
|
-
context "when receiving a 503 response" do
|
190
|
-
it "don't throw an error but do log one to debug" do
|
191
|
-
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
192
|
-
to_return(status: 503, body: "stubbed response", headers: {})
|
193
|
-
|
194
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
195
|
-
'api_write_token' => '1234',
|
196
|
-
'perform_connectivity_check' => false,
|
197
|
-
'ssl_ca_bundle_path' => EXAMPLE_COME_CA_CERTS_PATH,
|
198
|
-
'max_retries' => 2,
|
199
|
-
'retry_max_interval' => 0.2,
|
200
|
-
'retry_initial_interval' => 0.1,
|
201
|
-
})
|
202
|
-
plugin.register
|
203
|
-
plugin.instance_variable_set(:@running, false)
|
204
|
-
|
205
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:debug)
|
206
|
-
plugin.multi_receive(sample_events)
|
207
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
|
208
|
-
{
|
209
|
-
:error_class=>"Scalyr::Common::Client::ServerError",
|
210
|
-
:batch_num=>1,
|
211
|
-
:code=>503,
|
212
|
-
:message=>"Invalid JSON response from server",
|
213
|
-
:payload_size=>737,
|
214
|
-
:record_count=>3,
|
215
|
-
:total_batches=>1,
|
216
|
-
:url=>"https://agent.scalyr.com/addEvents",
|
217
|
-
:will_retry_in_seconds=>0.2,
|
218
|
-
:body=>"stubbed response"
|
219
|
-
}
|
220
|
-
)
|
221
|
-
end
|
222
|
-
end
|
223
|
-
|
224
|
-
context "when receiving a 500 response" do
|
225
|
-
it "don't throw an error but do log one to error" do
|
226
|
-
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
227
|
-
to_return(status: 500, body: "stubbed response", headers: {})
|
228
|
-
|
229
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
230
|
-
'api_write_token' => '1234',
|
231
|
-
'perform_connectivity_check' => false,
|
232
|
-
'ssl_ca_bundle_path' => EXAMPLE_COME_CA_CERTS_PATH,
|
233
|
-
'max_retries' => 2,
|
234
|
-
'retry_max_interval' => 0.2,
|
235
|
-
'retry_initial_interval' => 0.1,
|
236
|
-
})
|
237
|
-
plugin.register
|
238
|
-
plugin.instance_variable_set(:@running, false)
|
239
|
-
|
240
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
241
|
-
plugin.multi_receive(sample_events)
|
242
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
|
243
|
-
{
|
244
|
-
:error_class=>"Scalyr::Common::Client::ServerError",
|
245
|
-
:batch_num=>1,
|
246
|
-
:code=>500,
|
247
|
-
:message=>"Invalid JSON response from server",
|
248
|
-
:payload_size=>737,
|
249
|
-
:record_count=>3,
|
250
|
-
:total_batches=>1,
|
251
|
-
:url=>"https://agent.scalyr.com/addEvents",
|
252
|
-
:will_retry_in_seconds=>0.2,
|
253
|
-
:body=>"stubbed response"
|
254
|
-
}
|
255
|
-
)
|
256
|
-
end
|
257
|
-
end
|
258
|
-
|
259
|
-
context "when receiving a long non-json response" do
|
260
|
-
it "don't throw an error but do log one to error" do
|
261
|
-
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
262
|
-
to_return(status: 500, body: "0123456789" * 52, headers: {})
|
263
|
-
|
264
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
265
|
-
'api_write_token' => '1234',
|
266
|
-
'perform_connectivity_check' => false,
|
267
|
-
'ssl_ca_bundle_path' => EXAMPLE_COME_CA_CERTS_PATH,
|
268
|
-
'max_retries' => 2,
|
269
|
-
'retry_max_interval' => 0.2,
|
270
|
-
'retry_initial_interval' => 0.1,
|
271
|
-
})
|
272
|
-
plugin.register
|
273
|
-
plugin.instance_variable_set(:@running, false)
|
274
|
-
|
275
|
-
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
276
|
-
plugin.multi_receive(sample_events)
|
277
|
-
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
|
278
|
-
{
|
279
|
-
:error_class=>"Scalyr::Common::Client::ServerError",
|
280
|
-
:batch_num=>1,
|
281
|
-
:code=>500,
|
282
|
-
:message=>"Invalid JSON response from server",
|
283
|
-
:payload_size=>737,
|
284
|
-
:record_count=>3,
|
285
|
-
:total_batches=>1,
|
286
|
-
:url=>"https://agent.scalyr.com/addEvents",
|
287
|
-
:will_retry_in_seconds=>0.2,
|
288
|
-
:body=>("0123456789" * 50) + "012345678..."
|
289
|
-
}
|
290
|
-
)
|
291
|
-
end
|
292
|
-
end
|
293
|
-
|
294
|
-
context 'when DLQ is enabled' do
|
295
|
-
let(:dlq_writer) { double('DLQ writer') }
|
296
|
-
it 'should send the event to the DLQ' do
|
297
|
-
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
298
|
-
to_return(status: 500, body: "stubbed response", headers: {})
|
299
|
-
|
300
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
301
|
-
'api_write_token' => '1234',
|
302
|
-
'perform_connectivity_check' => false,
|
303
|
-
'ssl_ca_bundle_path' => EXAMPLE_COME_CA_CERTS_PATH,
|
304
|
-
'max_retries' => 2,
|
305
|
-
'retry_max_interval' => 0.2,
|
306
|
-
'retry_initial_interval' => 0.1,
|
307
|
-
})
|
308
|
-
plugin.register
|
309
|
-
plugin.instance_variable_set(:@running, false)
|
310
|
-
plugin.instance_variable_set('@dlq_writer', dlq_writer)
|
311
|
-
|
312
|
-
expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
|
313
|
-
plugin.multi_receive(sample_events)
|
314
|
-
end
|
315
|
-
end
|
316
|
-
end
|
317
|
-
|
318
|
-
end
|