logstash-output-scalyr 0.2.0 → 0.2.1.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +15 -0
- data/lib/logstash/outputs/scalyr.rb +96 -9
- data/lib/scalyr/common/client.rb +8 -0
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_integration_spec.rb +51 -9
- data/spec/logstash/outputs/scalyr_spec.rb +89 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a42ad86b12842e9e684f6e155a5a18f72e976a116af35b3bd7a15b38f4ab4fe8
|
4
|
+
data.tar.gz: 7b89fcda801a9d8b2a0df34c95288d4d1599c4c08795f600aba78206fe45407c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2da035c07ee249d16c5f1041f6dcc72fbe3ae39edbc161ddc602ea2af3cac3823584563d5ccb03bc0e099dc256bc0396898d26562558dcf03777a5eccc690920
|
7
|
+
data.tar.gz: d9491cd2bb02bc2f5d5e612b69ca9ba3923b773d21c37e8b2bc22d270c467552fe1569a85e0755961857f360d8a59841134156e390b4a5c536ede5299f196b89
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.2.1.beta
|
4
|
+
|
5
|
+
- Update plugin to fail fast on register and throw more user-friendly error on invalid URL for
|
6
|
+
``scalyr_server`` configuration option value and other fatal server errors (e.g. invalid
|
7
|
+
hostname).
|
8
|
+
- On plugin init / register we now perform connectivity check and verify that we can talk to
|
9
|
+
Scalyr API and validate that the API key is valid. This ensures that the plugin doesn't start and
|
10
|
+
start consuming events until we can successfully perform a connectivity check. This means
|
11
|
+
we can't end up in situation when we could potentially drop some events to the ground in case of
|
12
|
+
an invalid API key or similar when reaching retry limit and DLQ disabled. If you want to disable
|
13
|
+
this check on register, you can set ``perform_connectivity_check`` config option to ``false``.
|
14
|
+
|
3
15
|
## 0.2.0.beta, 0.2.0
|
4
16
|
|
5
17
|
- Fix a bug and correctly handle ``serverHost`` event level attribute. Now if an event contains
|
data/README.md
CHANGED
@@ -439,6 +439,21 @@ bundle exec rspec
|
|
439
439
|
bundle exec rake publish_gem
|
440
440
|
```
|
441
441
|
|
442
|
+
Or as an alternative if ``rake publish_gem`` task doesn't appear to work for whatever reason
|
443
|
+
(``publish_gem`` logstash Rake task silently swallows all the errors):
|
444
|
+
|
445
|
+
```
|
446
|
+
rm -rf vendor/
|
447
|
+
bundle check --path vendor/bundle || bundle install --deployment
|
448
|
+
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
449
|
+
chmod 0600 ~/.gem/credentials
|
450
|
+
bundle exec rake vendor
|
451
|
+
bundle exec rspec
|
452
|
+
rvm use jruby
|
453
|
+
bundle exec gem build logstash-output-scalyr.gemspec
|
454
|
+
bundle exec gem push logstash-output-scalyr-<version>.gem
|
455
|
+
```
|
456
|
+
|
442
457
|
`RUBY_USER` and `RUBY_PASSWORD` should be replaced with the username and password to the RubyGems.org account you wish to release to,
|
443
458
|
these credentials should be found in Keeper.
|
444
459
|
|
@@ -35,6 +35,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
35
35
|
# If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
|
36
36
|
config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
|
37
37
|
|
38
|
+
# True to perform connectivity check with Scalyr on plugin start up / register phase. This
|
39
|
+
# ensures an exception is thrown if we can't communicate with Scalyr and we don't start
|
40
|
+
# consuming events until plugin is correctly configured.
|
41
|
+
config :perform_connectivity_check, :validate => :boolean, :default => true
|
42
|
+
|
38
43
|
# server_attributes is a dictionary of key value pairs that represents/identifies the logstash aggregator server
|
39
44
|
# (where this plugin is running). Keys are arbitrary except for the 'serverHost' key which holds special meaning to
|
40
45
|
# Scalyr and is given special treatment in the Scalyr UI. All of these attributes are optional (not required for logs
|
@@ -130,6 +135,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
130
135
|
# minutes.
|
131
136
|
config :status_report_interval, :validate => :number, :default => 300
|
132
137
|
|
138
|
+
# Status will be reported regardless if the plugin receives empty batch if the plugin is
|
139
|
+
# "active" (received and processed some events during the plugin life time) and more than this
|
140
|
+
# amount of time has passed since the last reporting.
|
141
|
+
config :idle_status_report_deadline, :validate => :number, :default => 300
|
142
|
+
|
133
143
|
# True to also call send_status when multi_receive() is called with no events.
|
134
144
|
# In some situations (e.g. when logstash is configured with multiple scalyr
|
135
145
|
# plugins conditionally where most are idle) you may want to set this to false
|
@@ -287,6 +297,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
287
297
|
|
288
298
|
@scalyr_server << '/' unless @scalyr_server.end_with?('/')
|
289
299
|
|
300
|
+
# Validate the URL
|
301
|
+
uri = URI.parse(@scalyr_server)
|
302
|
+
|
303
|
+
if not uri.kind_of?(URI::HTTP) and not uri.kind_of?(URI::HTTPS)
|
304
|
+
raise LogStash::ConfigurationError, "scalyr_server configuration option value is not a valid URL. " \
|
305
|
+
"This value needs contain a full URL with the protocol. e.g. " \
|
306
|
+
"https://agent.scalyr.com for US and https://eu.scalyr.com for EU"
|
307
|
+
end
|
308
|
+
|
290
309
|
@add_events_uri = URI(@scalyr_server) + "addEvents"
|
291
310
|
|
292
311
|
@logger.info "Scalyr LogStash Plugin ID - #{self.id}"
|
@@ -317,12 +336,23 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
317
336
|
@http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
|
318
337
|
)
|
319
338
|
|
339
|
+
# We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
|
340
|
+
# Here priming just means setting up the client parameters without opening any connections.
|
341
|
+
# Since client writes certs to a temporary file there could be a race in case we don't do that
|
342
|
+
# here since multi_receive() is multi threaded. An alternative would be to put a look around
|
343
|
+
# client init method (aka client_config())
|
344
|
+
@client_session.client
|
345
|
+
|
346
|
+
# Send a ping to verify that the configuration API key is correct and that we can establish
|
347
|
+
# connection with Scalyr API
|
348
|
+
connectivity_check
|
349
|
+
|
320
350
|
@logger.info(sprintf("Started Scalyr LogStash output plugin %s (compression_type=%s,compression_level=%s,json_library=%s)." %
|
321
351
|
[PLUGIN_VERSION, @compression_type, @compression_type, @json_library]), :class => self.class.name)
|
322
352
|
|
323
353
|
# Finally, send a status line to Scalyr
|
324
354
|
# We use a special separate short lived client session for sending the initial client status.
|
325
|
-
# This is done to avoid the overhead in case single logstash instance has many scalyr output
|
355
|
+
# This is done to avoid the overhead in case single logstash instance has many scalyr output
|
326
356
|
# plugins configured with conditionals and majority of them are inactive (aka receive no data).
|
327
357
|
# This way we don't need to keep idle long running connection open.
|
328
358
|
initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
|
@@ -331,18 +361,52 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
331
361
|
@record_stats_for_status, @flush_quantile_estimates_on_status_send,
|
332
362
|
@http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
|
333
363
|
)
|
364
|
+
|
365
|
+
# We propagate errors on intial request to better handle errors related to invalid hostname
|
366
|
+
# or similar
|
334
367
|
send_status(initial_send_status_client_session)
|
335
368
|
initial_send_status_client_session.close
|
336
369
|
|
337
|
-
# We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
|
338
|
-
# Here priming just means setting up the client parameters without opening any connections.
|
339
|
-
# Since client writes certs to a temporary file there could be a race in case we don't do that
|
340
|
-
# here since multi_receive() is multi threaded. An alternative would be to put a look around
|
341
|
-
# client init method (aka client_config())
|
342
|
-
@client_session.client
|
343
|
-
|
344
370
|
end # def register
|
345
371
|
|
372
|
+
# Method which performs connectivity check with Scalyr API, verifies that wt can talk to the API
|
373
|
+
# and that the API token is valid.
|
374
|
+
def connectivity_check()
|
375
|
+
if not @perform_connectivity_check
|
376
|
+
return
|
377
|
+
end
|
378
|
+
|
379
|
+
@logger.debug("Performing connectivity check against the Scalyr API")
|
380
|
+
|
381
|
+
body = create_multi_event_request([], nil, nil, nil)[:body]
|
382
|
+
|
383
|
+
begin
|
384
|
+
@client_session.send_ping(body)
|
385
|
+
rescue Scalyr::Common::Client::ClientError, Manticore::ResolutionFailure => e
|
386
|
+
if not e.message.nil? and (e.message.include?("nodename nor servname provided") or
|
387
|
+
e.message.downcase.include?("name or service not know"))
|
388
|
+
raise LogStash::ConfigurationError,
|
389
|
+
format("Received error when trying to communicate with Scalyr API. This likely means that " \
|
390
|
+
"the configured value for 'scalyr_server' config option is invalid. Original error: %s",
|
391
|
+
e.message)
|
392
|
+
end
|
393
|
+
|
394
|
+
# For now, we consider rest of the errors non fatal and just log them and don't propagate
|
395
|
+
# them and fail register
|
396
|
+
@logger.warn("Received error when trying to send connectivity check request to Scalyr",
|
397
|
+
:error => e.message)
|
398
|
+
rescue Scalyr::Common::Client::ServerError => e
|
399
|
+
if e.code == 401
|
400
|
+
raise LogStash::ConfigurationError,
|
401
|
+
format("Received 401 from Scalyr API during connectivity check which indicates " \
|
402
|
+
"an invalid API key. Server Response: %s", e.body)
|
403
|
+
end
|
404
|
+
|
405
|
+
rescue => e
|
406
|
+
@logger.warn("Received non-fatal error during connectivity check", :error => e.message)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
346
410
|
# Convenience method to create a fresh quantile estimator
|
347
411
|
def get_new_metrics
|
348
412
|
return {
|
@@ -499,7 +563,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
499
563
|
end
|
500
564
|
end
|
501
565
|
|
502
|
-
if @report_status_for_empty_batches or records_count > 0
|
566
|
+
if @report_status_for_empty_batches or records_count > 0 or should_call_send_status_for_empty_batch()
|
503
567
|
send_status
|
504
568
|
end
|
505
569
|
|
@@ -516,6 +580,28 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
516
580
|
end
|
517
581
|
end # def multi_receive
|
518
582
|
|
583
|
+
# Return true if "send_status()" should be called even for an empty batch.
|
584
|
+
# NOTE: We always report status even if record count is 0 if more than X minutes have passed
|
585
|
+
# from the last status reporting for active plugins. Logstash sends empty batches for flush
|
586
|
+
# purposes quite often, but we only want to report status if we haven't reported it for a
|
587
|
+
# longer period (aka there hasn't been any data flowing in for a while). If we reported it
|
588
|
+
# for every flush event (which happens very often), this would skew metrics for regular
|
589
|
+
# events and output plugins which receive no data at all.
|
590
|
+
# We perform check against total events processed so we don't send status for output plugins
|
591
|
+
# which receive no data at all.
|
592
|
+
def should_call_send_status_for_empty_batch()
|
593
|
+
if @multi_receive_statistics[:total_events_processed] < 100
|
594
|
+
# Ignore idle plugins with no data
|
595
|
+
return false
|
596
|
+
end
|
597
|
+
|
598
|
+
if @last_status_transmit_time.nil?
|
599
|
+
# If this value is not set it means plugin is idle and hasn't received any data at all
|
600
|
+
return false
|
601
|
+
end
|
602
|
+
|
603
|
+
return (Time.now.to_i - @last_status_transmit_time.to_i > @idle_status_report_deadline)
|
604
|
+
end
|
519
605
|
|
520
606
|
def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
|
521
607
|
@stats_lock.synchronize do
|
@@ -1030,6 +1116,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1030
1116
|
:error_class => e.class.name
|
1031
1117
|
)
|
1032
1118
|
end
|
1119
|
+
|
1033
1120
|
return
|
1034
1121
|
end
|
1035
1122
|
@last_status_transmit_time = Time.now()
|
data/lib/scalyr/common/client.rb
CHANGED
@@ -219,7 +219,15 @@ class ClientSession
|
|
219
219
|
end
|
220
220
|
end
|
221
221
|
|
222
|
+
# Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
|
223
|
+
# and verify that the API key is valid.
|
224
|
+
def send_ping(body)
|
225
|
+
post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
|
226
|
+
response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
|
227
|
+
handle_response(response)
|
222
228
|
|
229
|
+
response
|
230
|
+
end
|
223
231
|
|
224
232
|
# Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
|
225
233
|
def post_add_events(body, is_status, body_serialization_duration = 0)
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.2.
|
3
|
+
s.version = '0.2.1.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -36,7 +36,10 @@ describe LogStash::Outputs::Scalyr do
|
|
36
36
|
describe "#ssl_tests" do
|
37
37
|
context "with default SSL configuration" do
|
38
38
|
it "throws a ServerError due to fake api key" do
|
39
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
39
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
40
|
+
'api_write_token' => '1234',
|
41
|
+
'perform_connectivity_check' => false,
|
42
|
+
})
|
40
43
|
plugin.register
|
41
44
|
plugin.instance_variable_set(:@running, false)
|
42
45
|
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
@@ -60,7 +63,12 @@ describe LogStash::Outputs::Scalyr do
|
|
60
63
|
|
61
64
|
context "when pointing at a location without any valid certs and not using builtin" do
|
62
65
|
it "throws an SSLError" do
|
63
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
66
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
67
|
+
'api_write_token' => '1234',
|
68
|
+
'perform_connectivity_check' => false,
|
69
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
70
|
+
'append_builtin_cert' => false,
|
71
|
+
})
|
64
72
|
plugin.register
|
65
73
|
plugin.instance_variable_set(:@running, false)
|
66
74
|
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
@@ -86,7 +94,11 @@ describe LogStash::Outputs::Scalyr do
|
|
86
94
|
`sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
|
87
95
|
|
88
96
|
begin
|
89
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
97
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
98
|
+
'api_write_token' => '1234',
|
99
|
+
'perform_connectivity_check' => false,
|
100
|
+
'append_builtin_cert' => false,
|
101
|
+
})
|
90
102
|
plugin.register
|
91
103
|
plugin.instance_variable_set(:@running, false)
|
92
104
|
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
@@ -125,7 +137,11 @@ describe LogStash::Outputs::Scalyr do
|
|
125
137
|
`echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
|
126
138
|
|
127
139
|
begin
|
128
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
140
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
141
|
+
'api_write_token' => '1234',
|
142
|
+
'perform_connectivity_check' => false,
|
143
|
+
'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
|
144
|
+
})
|
129
145
|
plugin.register
|
130
146
|
plugin.instance_variable_set(:@running, false)
|
131
147
|
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
@@ -152,7 +168,13 @@ describe LogStash::Outputs::Scalyr do
|
|
152
168
|
|
153
169
|
context "when an error occurs with retries at 5" do
|
154
170
|
it "exits after 5 retries and emits a log" do
|
155
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
171
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
172
|
+
'api_write_token' => '1234',
|
173
|
+
'perform_connectivity_check' => false,
|
174
|
+
'retry_initial_interval' => 0.1,
|
175
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
176
|
+
'append_builtin_cert' => false
|
177
|
+
})
|
156
178
|
plugin.register
|
157
179
|
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
158
180
|
plugin.multi_receive(sample_events)
|
@@ -168,7 +190,12 @@ describe LogStash::Outputs::Scalyr do
|
|
168
190
|
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
169
191
|
to_return(status: 503, body: "stubbed response", headers: {})
|
170
192
|
|
171
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
193
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
194
|
+
'api_write_token' => '1234',
|
195
|
+
'perform_connectivity_check' => false,
|
196
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
197
|
+
'append_builtin_cert' => false
|
198
|
+
})
|
172
199
|
plugin.register
|
173
200
|
plugin.instance_variable_set(:@running, false)
|
174
201
|
|
@@ -196,7 +223,12 @@ describe LogStash::Outputs::Scalyr do
|
|
196
223
|
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
197
224
|
to_return(status: 500, body: "stubbed response", headers: {})
|
198
225
|
|
199
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
226
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
227
|
+
'api_write_token' => '1234',
|
228
|
+
'perform_connectivity_check' => false,
|
229
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
230
|
+
'append_builtin_cert' => false
|
231
|
+
})
|
200
232
|
plugin.register
|
201
233
|
plugin.instance_variable_set(:@running, false)
|
202
234
|
|
@@ -224,7 +256,12 @@ describe LogStash::Outputs::Scalyr do
|
|
224
256
|
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
225
257
|
to_return(status: 500, body: "0123456789" * 52, headers: {})
|
226
258
|
|
227
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
259
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
260
|
+
'api_write_token' => '1234',
|
261
|
+
'perform_connectivity_check' => false,
|
262
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
263
|
+
'append_builtin_cert' => false
|
264
|
+
})
|
228
265
|
plugin.register
|
229
266
|
plugin.instance_variable_set(:@running, false)
|
230
267
|
|
@@ -253,7 +290,12 @@ describe LogStash::Outputs::Scalyr do
|
|
253
290
|
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
254
291
|
to_return(status: 500, body: "stubbed response", headers: {})
|
255
292
|
|
256
|
-
plugin = LogStash::Outputs::Scalyr.new({
|
293
|
+
plugin = LogStash::Outputs::Scalyr.new({
|
294
|
+
'api_write_token' => '1234',
|
295
|
+
'perform_connectivity_check' => false,
|
296
|
+
'ssl_ca_bundle_path' => '/fakepath/nocerts',
|
297
|
+
'append_builtin_cert' => false
|
298
|
+
})
|
257
299
|
plugin.register
|
258
300
|
plugin.instance_variable_set(:@running, false)
|
259
301
|
plugin.instance_variable_set('@dlq_writer', dlq_writer)
|
@@ -61,6 +61,7 @@ describe LogStash::Outputs::Scalyr do
|
|
61
61
|
context "test get_stats and send_status" do
|
62
62
|
plugin = LogStash::Outputs::Scalyr.new({
|
63
63
|
'api_write_token' => '1234',
|
64
|
+
'perform_connectivity_check' => false,
|
64
65
|
'serverhost_field' => 'source_host',
|
65
66
|
'log_constants' => ['tags'],
|
66
67
|
'flatten_nested_values' => true,
|
@@ -77,6 +78,7 @@ describe LogStash::Outputs::Scalyr do
|
|
77
78
|
it "it doesnt include flatten metrics if flattening is disabled" do
|
78
79
|
plugin1 = LogStash::Outputs::Scalyr.new({
|
79
80
|
'api_write_token' => '1234',
|
81
|
+
'perform_connectivity_check' => false,
|
80
82
|
'serverhost_field' => 'source_host',
|
81
83
|
'log_constants' => ['tags'],
|
82
84
|
'flatten_nested_values' => false,
|
@@ -151,6 +153,7 @@ describe LogStash::Outputs::Scalyr do
|
|
151
153
|
it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
|
152
154
|
plugin2 = LogStash::Outputs::Scalyr.new({
|
153
155
|
'api_write_token' => '1234',
|
156
|
+
'perform_connectivity_check' => false,
|
154
157
|
'serverhost_field' => 'source_host',
|
155
158
|
'log_constants' => ['tags'],
|
156
159
|
'flatten_nested_values' => false,
|
@@ -206,6 +209,7 @@ describe LogStash::Outputs::Scalyr do
|
|
206
209
|
it "creates logfile from serverHost" do
|
207
210
|
plugin = LogStash::Outputs::Scalyr.new({
|
208
211
|
'api_write_token' => '1234',
|
212
|
+
'perform_connectivity_check' => false,
|
209
213
|
'serverhost_field' => 'source_host',
|
210
214
|
'log_constants' => ['tags'],
|
211
215
|
})
|
@@ -223,7 +227,7 @@ describe LogStash::Outputs::Scalyr do
|
|
223
227
|
|
224
228
|
context "when serverhost_field is missing" do
|
225
229
|
it "does not contain log file" do
|
226
|
-
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
|
230
|
+
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'perform_connectivity_check' => false})
|
227
231
|
allow(plugin).to receive(:send_status).and_return(nil)
|
228
232
|
plugin.register
|
229
233
|
result = plugin.build_multi_event_request_array(sample_events)
|
@@ -237,6 +241,7 @@ describe LogStash::Outputs::Scalyr do
|
|
237
241
|
it "creates logfile from serverHost" do
|
238
242
|
plugin = LogStash::Outputs::Scalyr.new({
|
239
243
|
'api_write_token' => '1234',
|
244
|
+
'perform_connectivity_check' => false,
|
240
245
|
'serverhost_field' => 'source_host',
|
241
246
|
})
|
242
247
|
allow(plugin).to receive(:send_status).and_return(nil)
|
@@ -254,6 +259,7 @@ describe LogStash::Outputs::Scalyr do
|
|
254
259
|
it "does not contain log file" do
|
255
260
|
plugin = LogStash::Outputs::Scalyr.new({
|
256
261
|
'api_write_token' => '1234',
|
262
|
+
'perform_connectivity_check' => false,
|
257
263
|
'serverhost_field' => 'source_host',
|
258
264
|
'logfile_field' => 'source_file',
|
259
265
|
})
|
@@ -271,6 +277,7 @@ describe LogStash::Outputs::Scalyr do
|
|
271
277
|
context "when configured to flatten values with custom delimiter" do
|
272
278
|
config = {
|
273
279
|
'api_write_token' => '1234',
|
280
|
+
'perform_connectivity_check' => false,
|
274
281
|
'flatten_tags' => true,
|
275
282
|
'flat_tag_value' => 'true',
|
276
283
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -303,6 +310,7 @@ describe LogStash::Outputs::Scalyr do
|
|
303
310
|
context "when configured to flatten values with custom delimiter and deep delimiter fix" do
|
304
311
|
config = {
|
305
312
|
'api_write_token' => '1234',
|
313
|
+
'perform_connectivity_check' => false,
|
306
314
|
'flatten_tags' => true,
|
307
315
|
'flat_tag_value' => 'true',
|
308
316
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -336,6 +344,7 @@ describe LogStash::Outputs::Scalyr do
|
|
336
344
|
context "when configured to flatten values with custom delimiter, no array flattening" do
|
337
345
|
config = {
|
338
346
|
'api_write_token' => '1234',
|
347
|
+
'perform_connectivity_check' => false,
|
339
348
|
'flatten_tags' => true,
|
340
349
|
'flat_tag_value' => 'true',
|
341
350
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -367,6 +376,7 @@ describe LogStash::Outputs::Scalyr do
|
|
367
376
|
context "when configured to flatten values and tags" do
|
368
377
|
config = {
|
369
378
|
'api_write_token' => '1234',
|
379
|
+
'perform_connectivity_check' => false,
|
370
380
|
'flatten_tags' => true,
|
371
381
|
'flat_tag_value' => 'true',
|
372
382
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -399,6 +409,7 @@ describe LogStash::Outputs::Scalyr do
|
|
399
409
|
it "estimate_each_event_size is true explicit (default) batch split into 3 scalyr requests" do
|
400
410
|
config = {
|
401
411
|
'api_write_token' => '1234',
|
412
|
+
'perform_connectivity_check' => false,
|
402
413
|
'flatten_tags' => true,
|
403
414
|
'flat_tag_value' => 'true',
|
404
415
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -439,6 +450,7 @@ describe LogStash::Outputs::Scalyr do
|
|
439
450
|
it "estimate_each_event_size is true implicit (default) batch split into 3 scalyr requests" do
|
440
451
|
config = {
|
441
452
|
'api_write_token' => '1234',
|
453
|
+
'perform_connectivity_check' => false,
|
442
454
|
'flatten_tags' => true,
|
443
455
|
'flat_tag_value' => 'true',
|
444
456
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -478,6 +490,7 @@ describe LogStash::Outputs::Scalyr do
|
|
478
490
|
it "estimate_each_event_size is false batch not split into multiple scalyr requests" do
|
479
491
|
config = {
|
480
492
|
'api_write_token' => '1234',
|
493
|
+
'perform_connectivity_check' => false,
|
481
494
|
'flatten_tags' => true,
|
482
495
|
'flat_tag_value' => 'true',
|
483
496
|
'flat_tag_prefix' => 'tag_prefix_',
|
@@ -513,6 +526,7 @@ describe LogStash::Outputs::Scalyr do
|
|
513
526
|
context "when not configured to flatten values and tags" do
|
514
527
|
config = {
|
515
528
|
'api_write_token' => '1234',
|
529
|
+
'perform_connectivity_check' => false,
|
516
530
|
}
|
517
531
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
518
532
|
it "does not flatten" do
|
@@ -535,6 +549,7 @@ describe LogStash::Outputs::Scalyr do
|
|
535
549
|
context "when configured to flatten with max keys configured to 3" do
|
536
550
|
config = {
|
537
551
|
'api_write_token' => '1234',
|
552
|
+
'perform_connectivity_check' => false,
|
538
553
|
'flatten_nested_values' => true, # this converts into string 'true'
|
539
554
|
'flattening_max_key_count' => 3,
|
540
555
|
}
|
@@ -567,6 +582,7 @@ describe LogStash::Outputs::Scalyr do
|
|
567
582
|
it "no serverHost defined in server_attributes, no serverHost defined on event level - should use node hostname as the default session level value" do
|
568
583
|
config = {
|
569
584
|
'api_write_token' => '1234',
|
585
|
+
'perform_connectivity_check' => false,
|
570
586
|
}
|
571
587
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
572
588
|
|
@@ -585,6 +601,7 @@ describe LogStash::Outputs::Scalyr do
|
|
585
601
|
it "serverHost defined in server_attributes, nothing defined on event level - server_attributes value should be used" do
|
586
602
|
config = {
|
587
603
|
'api_write_token' => '1234',
|
604
|
+
'perform_connectivity_check' => false,
|
588
605
|
'server_attributes' => {'serverHost' => 'fooHost'}
|
589
606
|
}
|
590
607
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -604,6 +621,7 @@ describe LogStash::Outputs::Scalyr do
|
|
604
621
|
it "serverHost defined in server_attributes (explicitly defined), event level serverHost defined - event level value should be used" do
|
605
622
|
config = {
|
606
623
|
'api_write_token' => '1234',
|
624
|
+
'perform_connectivity_check' => false,
|
607
625
|
'server_attributes' => {'serverHost' => 'fooHost', 'attr1' => 'val1'}
|
608
626
|
}
|
609
627
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -660,6 +678,7 @@ describe LogStash::Outputs::Scalyr do
|
|
660
678
|
it "serverHost defined in server_attributes (defined via node hostname), event level serverHost defined - event level value should be used" do
|
661
679
|
config = {
|
662
680
|
'api_write_token' => '1234',
|
681
|
+
'perform_connectivity_check' => false,
|
663
682
|
'server_attributes' => {'attr1' => 'val1'}
|
664
683
|
}
|
665
684
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -719,6 +738,7 @@ describe LogStash::Outputs::Scalyr do
|
|
719
738
|
it "serverHost defined in server_attributes (explicitly defined), event level serverHost defined - event level value should be used and server level one for events without server host" do
|
720
739
|
config = {
|
721
740
|
'api_write_token' => '1234',
|
741
|
+
'perform_connectivity_check' => false,
|
722
742
|
'server_attributes' => {'serverHost' => 'top-level-session-host', 'attr1' => 'val1'}
|
723
743
|
}
|
724
744
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -771,6 +791,7 @@ describe LogStash::Outputs::Scalyr do
|
|
771
791
|
it "no serverHost defined, event level serverHost defined via non-default serverhost_field - event level value should be used" do
|
772
792
|
config = {
|
773
793
|
'api_write_token' => '1234',
|
794
|
+
'perform_connectivity_check' => false,
|
774
795
|
'server_attributes' => {'attr1' => 'val1'},
|
775
796
|
'use_hostname_for_serverhost' => false,
|
776
797
|
'serverhost_field' => 'custom_server_host',
|
@@ -827,6 +848,7 @@ describe LogStash::Outputs::Scalyr do
|
|
827
848
|
it "no serverHost defined, event level serverHost defined - event level value should be used" do
|
828
849
|
config = {
|
829
850
|
'api_write_token' => '1234',
|
851
|
+
'perform_connectivity_check' => false,
|
830
852
|
'server_attributes' => {'attr1' => 'val1'},
|
831
853
|
'use_hostname_for_serverhost' => false
|
832
854
|
}
|
@@ -882,6 +904,7 @@ describe LogStash::Outputs::Scalyr do
|
|
882
904
|
context "when receiving an event with Bignums" do
|
883
905
|
config = {
|
884
906
|
'api_write_token' => '1234',
|
907
|
+
'perform_connectivity_check' => false,
|
885
908
|
}
|
886
909
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
887
910
|
it "doesn't throw an error" do
|
@@ -902,6 +925,7 @@ describe LogStash::Outputs::Scalyr do
|
|
902
925
|
it "host attribute removed by default" do
|
903
926
|
config = {
|
904
927
|
'api_write_token' => '1234',
|
928
|
+
'perform_connectivity_check' => false,
|
905
929
|
}
|
906
930
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
907
931
|
|
@@ -927,6 +951,7 @@ describe LogStash::Outputs::Scalyr do
|
|
927
951
|
it "host attribute not removed if config option set" do
|
928
952
|
config = {
|
929
953
|
'api_write_token' => '1234',
|
954
|
+
'perform_connectivity_check' => false,
|
930
955
|
'remove_host_attribute_from_events' => false,
|
931
956
|
}
|
932
957
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -955,6 +980,7 @@ describe LogStash::Outputs::Scalyr do
|
|
955
980
|
it "stdlib (implicit)" do
|
956
981
|
config = {
|
957
982
|
'api_write_token' => '1234',
|
983
|
+
'perform_connectivity_check' => false,
|
958
984
|
}
|
959
985
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
960
986
|
|
@@ -971,6 +997,7 @@ describe LogStash::Outputs::Scalyr do
|
|
971
997
|
it "stdlib (explicit)" do
|
972
998
|
config = {
|
973
999
|
'api_write_token' => '1234',
|
1000
|
+
'perform_connectivity_check' => false,
|
974
1001
|
'json_library' => 'stdlib'
|
975
1002
|
}
|
976
1003
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -988,6 +1015,7 @@ describe LogStash::Outputs::Scalyr do
|
|
988
1015
|
it "jrjackson" do
|
989
1016
|
config = {
|
990
1017
|
'api_write_token' => '1234',
|
1018
|
+
'perform_connectivity_check' => false,
|
991
1019
|
'json_library' => 'jrjackson'
|
992
1020
|
}
|
993
1021
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
@@ -1002,5 +1030,65 @@ describe LogStash::Outputs::Scalyr do
|
|
1002
1030
|
expect(body['events'].size).to eq(1)
|
1003
1031
|
end
|
1004
1032
|
end
|
1033
|
+
|
1034
|
+
context "scalyr_server config option handling and connectivity check" do
|
1035
|
+
it "doesn't throw an error on valid url" do
|
1036
|
+
config = {
|
1037
|
+
'api_write_token' => '1234',
|
1038
|
+
'perform_connectivity_check' => false,
|
1039
|
+
'scalyr_server' => 'https://agent.scalyr.com'
|
1040
|
+
}
|
1041
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1042
|
+
plugin.register
|
1043
|
+
|
1044
|
+
config = {
|
1045
|
+
'api_write_token' => '1234',
|
1046
|
+
'perform_connectivity_check' => false,
|
1047
|
+
'scalyr_server' => 'https://eu.scalyr.com'
|
1048
|
+
}
|
1049
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1050
|
+
plugin.register
|
1051
|
+
end
|
1052
|
+
|
1053
|
+
it "throws on invalid URL" do
|
1054
|
+
config = {
|
1055
|
+
'api_write_token' => '1234',
|
1056
|
+
'perform_connectivity_check' => false,
|
1057
|
+
'scalyr_server' => 'agent.scalyr.com'
|
1058
|
+
}
|
1059
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1060
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /is not a valid URL/)
|
1061
|
+
|
1062
|
+
config = {
|
1063
|
+
'api_write_token' => '1234',
|
1064
|
+
'perform_connectivity_check' => false,
|
1065
|
+
'scalyr_server' => 'eu.scalyr.com'
|
1066
|
+
}
|
1067
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1068
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /is not a valid URL/)
|
1069
|
+
end
|
1070
|
+
|
1071
|
+
it "throws on invalid hostname" do
|
1072
|
+
config = {
|
1073
|
+
'api_write_token' => '1234',
|
1074
|
+
'perform_connectivity_check' => false,
|
1075
|
+
'scalyr_server' => 'https://agent.invalid.foo.scalyr.com',
|
1076
|
+
'perform_connectivity_check' => true
|
1077
|
+
}
|
1078
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1079
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Received error when trying to communicate/)
|
1080
|
+
end
|
1081
|
+
|
1082
|
+
it "throws on invalid api key" do
|
1083
|
+
config = {
|
1084
|
+
'api_write_token' => '1234',
|
1085
|
+
'perform_connectivity_check' => false,
|
1086
|
+
'scalyr_server' => 'https://agent.scalyr.com',
|
1087
|
+
'perform_connectivity_check' => true
|
1088
|
+
}
|
1089
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
1090
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Received 401 from Scalyr API during/)
|
1091
|
+
end
|
1092
|
+
end
|
1005
1093
|
end
|
1006
1094
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -4149,9 +4149,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
4149
4149
|
version: '0'
|
4150
4150
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
4151
4151
|
requirements:
|
4152
|
-
- - "
|
4152
|
+
- - ">"
|
4153
4153
|
- !ruby/object:Gem::Version
|
4154
|
-
version:
|
4154
|
+
version: 1.3.1
|
4155
4155
|
requirements: []
|
4156
4156
|
rubyforge_project:
|
4157
4157
|
rubygems_version: 2.7.10
|