logstash-output-scalyr 0.1.9 → 0.1.10.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/Gemfile +1 -0
- data/lib/logstash/outputs/scalyr.rb +98 -72
- data/lib/scalyr/common/client.rb +74 -50
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_integration_spec.rb +3 -3
- data/spec/logstash/outputs/scalyr_spec.rb +3 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 290310d36108a3311f55a47efbb3a73e01f19ccdf025a15590f86493459963da
|
4
|
+
data.tar.gz: a5bf9f872c496fa65670cab36e0955f1417541db5f151e11f0d914abe5fc9920
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 35d60d700b6bb6ccc43aea5151d07c306b29c5524ac8d80145ae792263ffe5e7264d22a59de3a21de9aa549c5963b4cd3c9b230b845f67c6ff9618faca863cad
|
7
|
+
data.tar.gz: 41cf50b968a9a11efff957cda768cf9d05e1ccf3c36e4f4d8f12c3d1aca9fb9da049647432bf047d51c2b6fe33f7b711484632d108dcd390ca852bc45aa7a00d
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.10.beta
|
4
|
+
|
5
|
+
- Switch to shared concurrency to allow the use of multiple worker threads for increased
|
6
|
+
throughput.
|
7
|
+
- Switch HTTP client library to `manticore` to work better with new shared concurrency.
|
8
|
+
|
3
9
|
## 0.1.9
|
4
10
|
|
5
11
|
- Add support for logging status messages with metrics to stdout in addition to sending this
|
data/Gemfile
CHANGED
@@ -8,9 +8,7 @@ require "thread" # for safe queueing
|
|
8
8
|
require "uri" # for escaping user input
|
9
9
|
require 'json' # for converting event object to JSON for upload
|
10
10
|
|
11
|
-
require '
|
12
|
-
require 'net/http/persistent'
|
13
|
-
require 'net/https'
|
11
|
+
require 'manticore'
|
14
12
|
require 'rbzip2'
|
15
13
|
require 'zlib'
|
16
14
|
require 'stringio'
|
@@ -27,9 +25,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
27
25
|
|
28
26
|
config_name "scalyr"
|
29
27
|
|
30
|
-
|
31
|
-
# anyway but we should be explicit.
|
32
|
-
concurrency :single
|
28
|
+
concurrency :shared
|
33
29
|
|
34
30
|
# The Scalyr API write token, these are available at https://www.scalyr.com/keys. This is the only compulsory configuration field required for proper upload
|
35
31
|
config :api_write_token, :validate => :string, :required => true
|
@@ -37,12 +33,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
37
33
|
# If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
|
38
34
|
config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
|
39
35
|
|
40
|
-
# Path to SSL bundle file.
|
41
|
-
config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-bundle.crt"
|
42
|
-
|
43
|
-
# If we should append our built-in Scalyr cert to the one we find at `ssl_ca_bundle_path`.
|
44
|
-
config :append_builtin_cert, :validate => :boolean, :default => true
|
45
|
-
|
46
36
|
# server_attributes is a dictionary of key value pairs that represents/identifies the logstash aggregator server
|
47
37
|
# (where this plugin is running). Keys are arbitrary except for the 'serverHost' key which holds special meaning to
|
48
38
|
# Scalyr and is given special treatment in the Scalyr UI. All of these attributes are optional (not required for logs
|
@@ -91,9 +81,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
91
81
|
# Set max interval in seconds between bulk retries.
|
92
82
|
config :retry_max_interval, :validate => :number, :default => 64
|
93
83
|
|
94
|
-
#
|
84
|
+
# Whether or not to verify the connection to Scalyr, only set to false for debugging.
|
95
85
|
config :ssl_verify_peer, :validate => :boolean, :default => true
|
96
|
-
|
86
|
+
|
87
|
+
# Path to SSL bundle file.
|
88
|
+
config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-bundle.crt"
|
89
|
+
|
90
|
+
# If we should append our built-in Scalyr cert to the one we find at `ssl_ca_bundle_path`.
|
91
|
+
config :append_builtin_cert, :validate => :boolean, :default => true
|
97
92
|
|
98
93
|
config :max_request_buffer, :validate => :number, :default => 5500000 # echee TODO: eliminate?
|
99
94
|
config :force_message_encoding, :validate => :string, :default => nil
|
@@ -132,6 +127,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
132
127
|
# Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
|
133
128
|
# these stas this might be wanted or not.
|
134
129
|
config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
|
130
|
+
|
131
|
+
# Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
|
132
|
+
# after no work being done.
|
133
|
+
config :noop_mode, :validate => :boolean, :default => false
|
134
|
+
|
135
|
+
# Manticore related options
|
136
|
+
config :http_connect_timeout, :validate => :number, :default => 10
|
137
|
+
config :http_socket_timeout, :validate => :number, :default => 10
|
138
|
+
config :http_request_timeout, :validate => :number, :default => 60
|
139
|
+
config :http_pool_max, :validate => :number, :default => 50
|
140
|
+
config :http_pool_max_per_route, :validate => :number, :default => 25
|
141
|
+
|
142
|
+
def initialize(*params)
|
143
|
+
super
|
144
|
+
# Request statistics are accumulated across multiple threads and must be accessed through a mutex
|
145
|
+
@stats_lock = Mutex.new
|
146
|
+
@send_stats = Mutex.new
|
147
|
+
end
|
135
148
|
|
136
149
|
def close
|
137
150
|
@running = false
|
@@ -140,6 +153,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
140
153
|
|
141
154
|
public
|
142
155
|
def register
|
156
|
+
# This prng is used exclusively to determine when to sample statistics and no security related purpose, for this
|
157
|
+
# reason we do not ensure thread safety for it.
|
143
158
|
@prng = Random.new
|
144
159
|
|
145
160
|
if @event_metrics_sample_rate < 0 or @event_metrics_sample_rate > 1
|
@@ -223,9 +238,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
223
238
|
@running = true
|
224
239
|
@client_session = Scalyr::Common::Client::ClientSession.new(
|
225
240
|
@logger, @add_events_uri,
|
226
|
-
@compression_type, @compression_level,
|
227
|
-
@
|
228
|
-
@
|
241
|
+
@compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
|
242
|
+
@record_stats_for_status, @flush_quantile_estimates_on_status_send,
|
243
|
+
@http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
|
229
244
|
)
|
230
245
|
|
231
246
|
@logger.info("Started Scalyr output plugin", :class => self.class.name)
|
@@ -257,6 +272,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
257
272
|
#
|
258
273
|
public
|
259
274
|
def multi_receive(events)
|
275
|
+
# Just return and pretend we did something if running in noop mode
|
276
|
+
return events if @noop_mode
|
277
|
+
|
260
278
|
start_time = Time.now.to_f
|
261
279
|
|
262
280
|
multi_event_request_array = build_multi_event_request_array(events)
|
@@ -276,6 +294,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
276
294
|
# ignore these.
|
277
295
|
if !multi_event_request.nil?
|
278
296
|
@client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
|
297
|
+
|
279
298
|
sleep_interval = 0
|
280
299
|
result.push(multi_event_request)
|
281
300
|
end
|
@@ -306,7 +325,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
306
325
|
exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
|
307
326
|
if e.is_commonly_retried?
|
308
327
|
# well-known retriable errors should be debug
|
309
|
-
@logger.
|
328
|
+
@logger.error(message, exc_data)
|
310
329
|
else
|
311
330
|
# all other failed uploads should be errors
|
312
331
|
@logger.error(message, exc_data)
|
@@ -328,9 +347,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
328
347
|
end
|
329
348
|
|
330
349
|
if records_count > 0
|
331
|
-
@
|
332
|
-
|
333
|
-
|
350
|
+
@stats_lock.synchronize do
|
351
|
+
@multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
|
352
|
+
@plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
|
353
|
+
@plugin_metrics[:multi_receive_event_count].observe(records_count)
|
354
|
+
end
|
334
355
|
end
|
335
356
|
|
336
357
|
send_status
|
@@ -501,10 +522,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
501
522
|
end
|
502
523
|
|
503
524
|
if should_sample_event_metrics
|
504
|
-
@
|
525
|
+
@stats_lock.synchronize do
|
526
|
+
@plugin_metrics[:event_attributes_count].observe(record.count)
|
505
527
|
|
506
|
-
|
507
|
-
|
528
|
+
if @flatten_nested_values
|
529
|
+
@plugin_metrics[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
|
530
|
+
end
|
508
531
|
end
|
509
532
|
end
|
510
533
|
|
@@ -598,7 +621,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
598
621
|
def create_multi_event_request(scalyr_events, current_threads, current_logs)
|
599
622
|
|
600
623
|
body = {
|
601
|
-
:session => @session_id,
|
624
|
+
:session => @session_id + Thread.current.object_id.to_s,
|
602
625
|
:token => @api_write_token,
|
603
626
|
:events => scalyr_events,
|
604
627
|
}
|
@@ -638,33 +661,35 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
638
661
|
|
639
662
|
# Retrieve batch and other event level metric values
|
640
663
|
def get_stats
|
641
|
-
|
664
|
+
@stats_lock.synchronize do
|
665
|
+
current_stats = @multi_receive_statistics.clone
|
642
666
|
|
643
|
-
|
644
|
-
|
645
|
-
|
667
|
+
current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
|
668
|
+
current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
|
669
|
+
current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)
|
646
670
|
|
647
|
-
|
648
|
-
|
649
|
-
|
671
|
+
current_stats[:multi_receive_event_count_p50] = @plugin_metrics[:multi_receive_event_count].query(0.5)
|
672
|
+
current_stats[:multi_receive_event_count_p90] = @plugin_metrics[:multi_receive_event_count].query(0.9)
|
673
|
+
current_stats[:multi_receive_event_count_p99] = @plugin_metrics[:multi_receive_event_count].query(0.99)
|
650
674
|
|
651
|
-
|
652
|
-
|
653
|
-
|
675
|
+
current_stats[:event_attributes_count_p50] = @plugin_metrics[:event_attributes_count].query(0.5)
|
676
|
+
current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
|
677
|
+
current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
|
654
678
|
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
679
|
+
if @flatten_nested_values
|
680
|
+
# We only return those metrics in case flattening is enabled
|
681
|
+
current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
|
682
|
+
current_stats[:flatten_values_duration_secs_p90] = @plugin_metrics[:flatten_values_duration_secs].query(0.9)
|
683
|
+
current_stats[:flatten_values_duration_secs_p99] = @plugin_metrics[:flatten_values_duration_secs].query(0.99)
|
684
|
+
end
|
661
685
|
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
686
|
+
if @flush_quantile_estimates_on_status_send
|
687
|
+
@logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
|
688
|
+
@plugin_metrics = get_new_metrics
|
689
|
+
end
|
666
690
|
|
667
|
-
|
691
|
+
current_stats
|
692
|
+
end
|
668
693
|
end
|
669
694
|
|
670
695
|
|
@@ -683,33 +708,34 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
683
708
|
'plugin_id' => self.id,
|
684
709
|
}
|
685
710
|
}
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
711
|
+
@send_stats.synchronize do
|
712
|
+
if !@last_status_transmit_time
|
713
|
+
status_event[:attrs]['message'] = "Started Scalyr LogStash output plugin."
|
714
|
+
status_event[:attrs]['serverHost'] = @node_hostname
|
715
|
+
else
|
716
|
+
cur_time = Time.now()
|
717
|
+
return if (cur_time.to_i - @last_status_transmit_time.to_i) < @status_report_interval
|
718
|
+
# echee TODO: get instance stats from session and create a status log line
|
719
|
+
msg = 'plugin_status: '
|
720
|
+
cnt = 0
|
721
|
+
@client_session.get_stats.each do |k, v|
|
722
|
+
val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
|
723
|
+
val = val.nil? ? 0 : val
|
724
|
+
msg << ' ' if cnt > 0
|
725
|
+
msg << "#{k.to_s}=#{val}"
|
726
|
+
cnt += 1
|
727
|
+
end
|
728
|
+
get_stats.each do |k, v|
|
729
|
+
val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
|
730
|
+
val = val.nil? ? 0 : val
|
731
|
+
msg << ' ' if cnt > 0
|
732
|
+
msg << "#{k.to_s}=#{val}"
|
733
|
+
cnt += 1
|
734
|
+
end
|
735
|
+
status_event[:attrs]['message'] = msg
|
736
|
+
status_event[:attrs]['serverHost'] = @node_hostname
|
737
|
+
status_event[:attrs]['parser'] = @status_parser
|
709
738
|
end
|
710
|
-
status_event[:attrs]['message'] = msg
|
711
|
-
status_event[:attrs]['serverHost'] = @node_hostname
|
712
|
-
status_event[:attrs]['parser'] = @status_parser
|
713
739
|
end
|
714
740
|
multi_event_request = create_multi_event_request([status_event], nil, nil)
|
715
741
|
@client_session.post_add_events(multi_event_request[:body], true, 0)
|
data/lib/scalyr/common/client.rb
CHANGED
@@ -52,8 +52,9 @@ end
|
|
52
52
|
class ClientSession
|
53
53
|
|
54
54
|
def initialize(logger, add_events_uri, compression_type, compression_level,
|
55
|
-
ssl_verify_peer, ssl_ca_bundle_path,
|
56
|
-
record_stats_for_status, flush_quantile_estimates_on_status_send
|
55
|
+
ssl_verify_peer, ssl_ca_bundle_path, append_builtin_cert,
|
56
|
+
record_stats_for_status, flush_quantile_estimates_on_status_send,
|
57
|
+
connect_timeout, socket_timeout, request_timeout, pool_max, pool_max_per_route)
|
57
58
|
@logger = logger
|
58
59
|
@add_events_uri = add_events_uri # typically /addEvents
|
59
60
|
@compression_type = compression_type
|
@@ -61,9 +62,13 @@ class ClientSession
|
|
61
62
|
@ssl_verify_peer = ssl_verify_peer
|
62
63
|
@ssl_ca_bundle_path = ssl_ca_bundle_path
|
63
64
|
@append_builtin_cert = append_builtin_cert
|
64
|
-
@ssl_verify_depth = ssl_verify_depth
|
65
65
|
@record_stats_for_status = record_stats_for_status
|
66
66
|
@flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
|
67
|
+
@connect_timeout = connect_timeout
|
68
|
+
@socket_timeout = socket_timeout
|
69
|
+
@request_timeout = request_timeout
|
70
|
+
@pool_max = pool_max
|
71
|
+
@pool_max_per_route = pool_max_per_route
|
67
72
|
|
68
73
|
# A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
|
69
74
|
@cert_string = "" \
|
@@ -126,11 +131,29 @@ class ClientSession
|
|
126
131
|
:compression_type => @compression_type,
|
127
132
|
:compression_level => @compression_level,
|
128
133
|
}
|
134
|
+
end # def initialize
|
129
135
|
|
130
|
-
|
136
|
+
def client_config
|
137
|
+
# TODO: Eventually expose some more of these as config options, though nothing here really needs tuning normally
|
138
|
+
# besides SSL
|
139
|
+
c = {
|
140
|
+
connect_timeout: @connect_timeout,
|
141
|
+
socket_timeout: @socket_timeout,
|
142
|
+
request_timeout: @request_timeout,
|
143
|
+
follow_redirects: true,
|
144
|
+
automatic_retries: 1,
|
145
|
+
retry_non_idempotent: false,
|
146
|
+
check_connection_timeout: 200,
|
147
|
+
pool_max: @pool_max,
|
148
|
+
pool_max_per_route: @pool_max_per_route,
|
149
|
+
cookies: true,
|
150
|
+
keepalive: true,
|
151
|
+
ssl: {}
|
152
|
+
}
|
131
153
|
|
132
154
|
# verify peers to prevent potential MITM attacks
|
133
155
|
if @ssl_verify_peer
|
156
|
+
c[:ssl][:verify] = :strict
|
134
157
|
@ca_cert = Tempfile.new("ca_cert")
|
135
158
|
if File.file?(@ssl_ca_bundle_path)
|
136
159
|
@ca_cert.write(File.read(@ssl_ca_bundle_path))
|
@@ -142,13 +165,17 @@ class ClientSession
|
|
142
165
|
end
|
143
166
|
end
|
144
167
|
@ca_cert.flush
|
145
|
-
|
146
|
-
@http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
147
|
-
@http.verify_depth = @ssl_verify_depth
|
168
|
+
c[:ssl][:ca_file] = @ca_cert.path
|
148
169
|
else
|
149
|
-
|
170
|
+
c[:ssl][:verify] = :disable
|
150
171
|
end
|
151
|
-
|
172
|
+
|
173
|
+
c
|
174
|
+
end
|
175
|
+
|
176
|
+
def client
|
177
|
+
@client ||= Manticore::Client.new(client_config)
|
178
|
+
end
|
152
179
|
|
153
180
|
# Convenience method to create a fresh quantile estimator
|
154
181
|
def get_new_latency_stats
|
@@ -163,60 +190,56 @@ class ClientSession
|
|
163
190
|
|
164
191
|
# Get a clone of current statistics hash and calculate percentiles
|
165
192
|
def get_stats
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
@
|
183
|
-
|
193
|
+
@stats_lock.synchronize do
|
194
|
+
current_stats = @stats.clone
|
195
|
+
|
196
|
+
current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
|
197
|
+
current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
|
198
|
+
current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
|
199
|
+
current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
|
200
|
+
current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
|
201
|
+
current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
|
202
|
+
current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
|
203
|
+
current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
|
204
|
+
current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
|
205
|
+
current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
|
206
|
+
current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
|
207
|
+
current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
|
208
|
+
|
209
|
+
if @flush_quantile_estimates_on_status_send
|
210
|
+
@logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
|
211
|
+
@latency_stats = get_new_latency_stats
|
212
|
+
end
|
213
|
+
current_stats
|
184
214
|
end
|
185
|
-
current_stats
|
186
215
|
end
|
187
216
|
|
188
217
|
|
189
218
|
|
190
219
|
# Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
|
191
220
|
def post_add_events(body, is_status, body_serialization_duration = 0)
|
192
|
-
|
221
|
+
post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
|
193
222
|
fail_count = 1 # putative assume failure
|
194
223
|
start_time = Time.now
|
195
224
|
uncompressed_bytes_sent = 0
|
196
225
|
compressed_bytes_sent = 0
|
197
226
|
bytes_received = 0
|
198
227
|
begin
|
199
|
-
response =
|
228
|
+
response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
|
200
229
|
handle_response(response)
|
201
230
|
|
202
231
|
fail_count -= 1 # success means we negate the putative failure
|
203
232
|
uncompressed_bytes_sent = (body.bytesize + @add_events_uri.path.bytesize)
|
204
|
-
compressed_bytes_sent = (
|
233
|
+
compressed_bytes_sent = (post_body.bytesize + @add_events_uri.path.bytesize)
|
205
234
|
bytes_received = response.body.bytesize # echee: double check
|
206
235
|
# echee TODO add more statistics
|
207
236
|
|
237
|
+
# TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
|
238
|
+
# way to detect that it is from SSL.
|
208
239
|
rescue OpenSSL::SSL::SSLError => e
|
209
|
-
|
210
|
-
@ca_cert = Tempfile.new("ca_cert")
|
211
|
-
@ca_cert.write(@cert_string)
|
212
|
-
@ca_cert.flush
|
213
|
-
@http.ca_file = @ca_cert.path
|
214
|
-
raise ClientError.new("Packaged certificate appears to have been deleted, writing a new one.", @add_events_uri)
|
215
|
-
else
|
216
|
-
raise e
|
217
|
-
end
|
240
|
+
raise e
|
218
241
|
|
219
|
-
rescue
|
242
|
+
rescue Manticore::ManticoreException => e
|
220
243
|
# The underlying persistent-connection library automatically retries when there are network-related errors.
|
221
244
|
# Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
|
222
245
|
raise ClientError.new(e.message, @add_events_uri)
|
@@ -245,7 +268,6 @@ class ClientSession
|
|
245
268
|
|
246
269
|
|
247
270
|
def close
|
248
|
-
@http.shutdown
|
249
271
|
end # def close
|
250
272
|
|
251
273
|
|
@@ -273,18 +295,20 @@ class ClientSession
|
|
273
295
|
compression_duration = end_time - start_time
|
274
296
|
end
|
275
297
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
298
|
+
version = 'output-logstash-scalyr 0.1.10.beta'
|
299
|
+
post_headers = {
|
300
|
+
'Content-Type': 'application/json',
|
301
|
+
'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM
|
302
|
+
}
|
280
303
|
|
304
|
+
post_body = nil
|
281
305
|
if not encoding.nil?
|
282
|
-
|
283
|
-
|
306
|
+
post_headers['Content-Encoding'] = encoding
|
307
|
+
post_body = compressed_body
|
284
308
|
else
|
285
|
-
|
309
|
+
post_body = body
|
286
310
|
end
|
287
|
-
return
|
311
|
+
return post_body, post_headers, compression_duration
|
288
312
|
end # def prepare_post_object
|
289
313
|
|
290
314
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.10.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -24,7 +24,7 @@ describe LogStash::Outputs::Scalyr do
|
|
24
24
|
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
|
25
25
|
plugin.register
|
26
26
|
plugin.multi_receive(sample_events)
|
27
|
-
}.to raise_error(
|
27
|
+
}.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
@@ -38,7 +38,7 @@ describe LogStash::Outputs::Scalyr do
|
|
38
38
|
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
|
39
39
|
plugin.register
|
40
40
|
plugin.multi_receive(sample_events)
|
41
|
-
}.to raise_error(
|
41
|
+
}.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
|
42
42
|
end
|
43
43
|
ensure
|
44
44
|
`sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
|
@@ -65,7 +65,7 @@ describe LogStash::Outputs::Scalyr do
|
|
65
65
|
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
|
66
66
|
plugin.register
|
67
67
|
plugin.multi_receive(sample_events)
|
68
|
-
}.to raise_error(
|
68
|
+
}.to raise_error(Scalyr::Common::Client::ClientError, "Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)")
|
69
69
|
ensure
|
70
70
|
# Clean up the hosts file
|
71
71
|
`sudo truncate -s 0 /etc/hosts`
|
@@ -78,6 +78,7 @@ describe LogStash::Outputs::Scalyr do
|
|
78
78
|
mock_client_session = MockClientSession.new
|
79
79
|
plugin1.instance_variable_set(:@last_status_transmit_time, 100)
|
80
80
|
plugin1.instance_variable_set(:@client_session, mock_client_session)
|
81
|
+
plugin1.instance_variable_set(:@session_id, "some_session_id")
|
81
82
|
plugin1.instance_variable_set(:@plugin_metrics, {
|
82
83
|
:multi_receive_duration_secs => Quantile::Estimator.new,
|
83
84
|
:multi_receive_event_count => Quantile::Estimator.new,
|
@@ -95,6 +96,7 @@ describe LogStash::Outputs::Scalyr do
|
|
95
96
|
# 1. Initial send
|
96
97
|
plugin.instance_variable_set(:@last_status_transmit_time, nil)
|
97
98
|
plugin.instance_variable_set(:@client_session, mock_client_session)
|
99
|
+
plugin.instance_variable_set(:@session_id, "some_session_id")
|
98
100
|
status_event = plugin.send_status
|
99
101
|
expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
|
100
102
|
|
@@ -138,6 +140,7 @@ describe LogStash::Outputs::Scalyr do
|
|
138
140
|
# 1. Initial send
|
139
141
|
plugin.instance_variable_set(:@last_status_transmit_time, nil)
|
140
142
|
plugin.instance_variable_set(:@client_session, mock_client_session)
|
143
|
+
plugin.instance_variable_set(:@session_id, "some_session_id")
|
141
144
|
expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything)
|
142
145
|
plugin.send_status
|
143
146
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-06-
|
11
|
+
date: 2021-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -3745,9 +3745,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
3745
3745
|
version: '0'
|
3746
3746
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
3747
3747
|
requirements:
|
3748
|
-
- - "
|
3748
|
+
- - ">"
|
3749
3749
|
- !ruby/object:Gem::Version
|
3750
|
-
version:
|
3750
|
+
version: 1.3.1
|
3751
3751
|
requirements: []
|
3752
3752
|
rubyforge_project:
|
3753
3753
|
rubygems_version: 2.7.10
|