logstash-output-scalyr 0.1.22.beta → 0.1.23.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/scalyr.rb +83 -65
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8e2757da74eb03ea09a88597ed713db01bc050e17c31c771558bdb16a80ca40e
|
4
|
+
data.tar.gz: 1bc8519d1b30aea135d66b8b9ecfb632f688fd62abd58cd3b91cc92c4ad8957f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 327a3e2ba020f3166b0c435700bdfedf4f96598a352f1f66f45fe9e671a053ee684aa812d686a4d58b5dd9f319e8b0f87e2ea8290d876e504b8416a8010795ed
|
7
|
+
data.tar.gz: 1ba44520daddda5c17da810221063ce815af082645ae3fd12e76b031247ff8a13cb3e7dab4fbf69e84e5a876605290702785cc17639de69435f0b780a74691f9
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.23.beta
|
4
|
+
- Add testing support for disabling estimation of serialized event size for each event in the batch.
|
5
|
+
|
3
6
|
## 0.1.22.beta
|
4
7
|
- Add new plugin metric for tracking the duration of ``build_multi_event_request_array`` method.
|
5
8
|
- Update internal dependencies (``manticore``) to latest stable version.
|
@@ -140,11 +140,17 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
140
140
|
# Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
|
141
141
|
# these stas this might be wanted or not.
|
142
142
|
config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
|
143
|
-
|
143
|
+
|
144
144
|
# Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
|
145
145
|
# after no work being done.
|
146
146
|
config :noop_mode, :validate => :boolean, :default => false
|
147
147
|
|
148
|
+
# Set to true to disable estimiating the size of each serialized event to make sure we don't go over the max request
|
149
|
+
# size (5.5) and split batch into multiple Scalyr requests, if needed. Since this estimation is not "free", especially
|
150
|
+
# for large batches, it may make sense to disable this option when logstash batch size is configured in a way that
|
151
|
+
# Scalyr single request limit won't be reached.
|
152
|
+
config :estimate_each_event_size, :validate => :boolean, :default => true
|
153
|
+
|
148
154
|
# Manticore related options
|
149
155
|
config :http_connect_timeout, :validate => :number, :default => 10
|
150
156
|
config :http_socket_timeout, :validate => :number, :default => 10
|
@@ -316,16 +322,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
316
322
|
begin
|
317
323
|
records_count = events.to_a.length
|
318
324
|
|
319
|
-
# We also time the duration of the build_multi_event_request_array method
|
325
|
+
# We also time the duration of the build_multi_event_request_array method. To avoid locking twice,
|
326
|
+
# we store the duration value here and record metric at the end.
|
320
327
|
start_time = Time.now.to_f
|
321
328
|
|
322
329
|
multi_event_request_array = build_multi_event_request_array(events)
|
323
|
-
|
324
|
-
if records_count > 0
|
325
|
-
@stats_lock.synchronize do
|
326
|
-
@plugin_metrics[:build_multi_duration_secs].observe(Time.now.to_f - start_time)
|
327
|
-
end
|
328
|
-
end
|
330
|
+
build_multi_duration_secs = Time.now.to_f - start_time
|
329
331
|
|
330
332
|
# Loop over all array of multi-event requests, sending each multi-event to Scalyr
|
331
333
|
sleep_interval = @retry_initial_interval
|
@@ -439,6 +441,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
439
441
|
if records_count > 0
|
440
442
|
@stats_lock.synchronize do
|
441
443
|
@multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
|
444
|
+
@plugin_metrics[:build_multi_duration_secs].observe(build_multi_duration_secs)
|
442
445
|
@plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
|
443
446
|
@plugin_metrics[:multi_receive_event_count].observe(records_count)
|
444
447
|
@plugin_metrics[:batches_per_multi_receive].observe(total_batches)
|
@@ -675,67 +678,74 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
675
678
|
scalyr_event[:log] = logs_ids[log_identifier]
|
676
679
|
end
|
677
680
|
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
@
|
681
|
+
if @estimate_each_event_size
|
682
|
+
# get json string of event to keep track of how many bytes we are sending
|
683
|
+
begin
|
684
|
+
event_json = scalyr_event.to_json
|
685
|
+
log_json = nil
|
686
|
+
if add_log
|
687
|
+
log_json = logs[log_identifier].to_json
|
688
|
+
end
|
689
|
+
rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
|
690
|
+
@logger.warn "#{e.class}: #{e.message}"
|
691
|
+
|
692
|
+
# Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
|
693
|
+
# TODO
|
694
|
+
# atime = Fluent::EventTime.new( sec, nsec )
|
695
|
+
# router.emit_error_event(serverHost, time, record, e)
|
696
|
+
|
697
|
+
scalyr_event[:attrs].each do |key, value|
|
698
|
+
@logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
|
699
|
+
scalyr_event[:attrs][key] = value.encode(
|
700
|
+
"UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
|
701
|
+
).force_encoding('UTF-8')
|
702
|
+
end
|
703
|
+
event_json = scalyr_event.to_json
|
704
|
+
rescue Java::JavaLang::ClassCastException => e
|
705
|
+
# Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
|
706
|
+
# Because of the version of jruby logstash works with we don't have the option to just update this away,
|
707
|
+
# so if we run into it we convert bignums into strings so we can get the data in at least.
|
708
|
+
# This is fixed in JRuby 9.2.7, which includes json 2.2.0
|
709
|
+
@logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
|
710
|
+
@stats_lock.synchronize do
|
711
|
+
@multi_receive_statistics[:total_java_class_cast_errors] += 1
|
712
|
+
end
|
713
|
+
Scalyr::Common::Util.convert_bignums(scalyr_event)
|
714
|
+
event_json = scalyr_event.to_json
|
715
|
+
log_json = nil
|
716
|
+
if add_log
|
717
|
+
log_json = logs[log_identifier].to_json
|
718
|
+
end
|
708
719
|
end
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
720
|
+
|
721
|
+
# generate new request if json size of events in the array exceed maximum request buffer size
|
722
|
+
append_event = true
|
723
|
+
add_bytes = event_json.bytesize
|
724
|
+
if log_json
|
725
|
+
add_bytes = add_bytes + log_json.bytesize
|
714
726
|
end
|
715
|
-
end
|
716
727
|
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
728
|
+
if total_bytes + add_bytes > @max_request_buffer
|
729
|
+
# make sure we always have at least one event
|
730
|
+
if scalyr_events.size == 0
|
731
|
+
scalyr_events << scalyr_event
|
732
|
+
l_events << l_event
|
733
|
+
append_event = false
|
734
|
+
end
|
735
|
+
|
736
|
+
multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
|
737
|
+
multi_event_request_array << multi_event_request
|
738
|
+
|
739
|
+
total_bytes = 0
|
740
|
+
current_threads = Hash.new
|
741
|
+
logs = Hash.new
|
742
|
+
logs_ids = Hash.new
|
743
|
+
scalyr_events = Array.new
|
744
|
+
l_events = Array.new
|
729
745
|
end
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
total_bytes = 0
|
734
|
-
current_threads = Hash.new
|
735
|
-
logs = Hash.new
|
736
|
-
logs_ids = Hash.new
|
737
|
-
scalyr_events = Array.new
|
738
|
-
l_events = Array.new
|
746
|
+
else
|
747
|
+
# If size estimation is disabled we simply append the event and handle splitting later on (if needed)
|
748
|
+
append_event = true
|
739
749
|
end
|
740
750
|
|
741
751
|
# if we haven't consumed the current event already
|
@@ -812,6 +822,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
812
822
|
end
|
813
823
|
end_time = Time.now.to_f
|
814
824
|
serialization_duration = end_time - start_time
|
825
|
+
|
826
|
+
serialized_request_size = serialized_body.bytesize
|
827
|
+
|
828
|
+
if serialized_request_size >= @max_request_buffer
|
829
|
+
# TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
|
830
|
+
@logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
|
831
|
+
end
|
832
|
+
|
815
833
|
{
|
816
834
|
:body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
|
817
835
|
:logstash_events => logstash_events
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.23.beta"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.23.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.23.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|