logstash-output-scalyr 0.2.7.beta → 0.2.8.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +2 -4
- data/lib/logstash/outputs/scalyr.rb +29 -4
- data/lib/scalyr/common/client.rb +16 -2
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +2 -2
- metadata +3 -23
- data/spec/benchmarks/bignum_fixing.rb +0 -87
- data/spec/benchmarks/flattening_and_serialization.rb +0 -100
- data/spec/benchmarks/json_serialization.rb +0 -85
- data/spec/benchmarks/metrics_overhead.rb +0 -48
- data/spec/benchmarks/set_session_level_serverhost_on_events.rb +0 -107
- data/spec/benchmarks/util.rb +0 -24
- data/spec/logstash/outputs/fixtures/example_com.pem +0 -41
- data/spec/logstash/outputs/scalyr_integration_spec.rb +0 -318
- data/spec/logstash/outputs/scalyr_spec.rb +0 -1262
- data/spec/scalyr/common/util_spec.rb +0 -543
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da91d80d418081307dc1a6b756f17024712c0eab9a6d527a140288976a58b76f
|
4
|
+
data.tar.gz: 2ac792b42939edb371614c73c4dbb50fcc835f199d03685ff619d9d355c6465a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0a87ca7115a2eb3f03282046510bd51869e575eb29b8bec6ee273cbc9f68fea0167890835ffdda8bd0d571ac4299aff689642542d8dd5d88eaa2a89fe760a3f0
|
7
|
+
data.tar.gz: 7457fff5003d10b7288a9a958f3ec3f4a490c19da51a3ac3ade0402a37f74fafffc2501185857b4b3a7263753cb6af3276471724870efa0dca9e0773671b2f3f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.2.8.beta
|
4
|
+
|
5
|
+
* Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
|
6
|
+
fixtures with the actual production gem file.
|
7
|
+
|
8
|
+
* Do not retry requests that will never be accepted by the server.
|
9
|
+
Specifically, any request that returns HTTP Status code 413 is too large, and
|
10
|
+
will never be accepted. Instead of simply retrying for 10 minutes before
|
11
|
+
sending the request to the DLQ, skip the retries go directly to sending the
|
12
|
+
request to the DLQ.
|
13
|
+
|
14
|
+
To be notified when an event fails to be ingested for whatever reason, create
|
15
|
+
an alert using the query: ``parser='logstash_plugin_metrics'
|
16
|
+
failed_events_processed > 0``. Instructions on how to create an alert can be
|
17
|
+
found in our docs here: https://scalyr.com/help/alerts
|
18
|
+
|
3
19
|
## 0.2.7.beta
|
4
20
|
|
5
21
|
* SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
|
data/README.md
CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
|
|
505
505
|
|
506
506
|
```
|
507
507
|
rm -rf vendor/
|
508
|
-
bundle
|
508
|
+
bundle install
|
509
509
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
510
510
|
chmod 0600 ~/.gem/credentials
|
511
|
-
bundle exec rake vendor
|
512
511
|
bundle exec rspec
|
513
512
|
bundle exec rake publish_gem
|
514
513
|
```
|
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
|
|
518
517
|
|
519
518
|
```
|
520
519
|
rm -rf vendor/
|
521
|
-
bundle
|
520
|
+
bundle install
|
522
521
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
523
522
|
chmod 0600 ~/.gem/credentials
|
524
|
-
bundle exec rake vendor
|
525
523
|
bundle exec rspec
|
526
524
|
rvm use jruby
|
527
525
|
bundle exec gem build logstash-output-scalyr.gemspec
|
@@ -500,6 +500,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
500
500
|
result.push(multi_event_request)
|
501
501
|
end
|
502
502
|
|
503
|
+
rescue Scalyr::Common::Client::PayloadTooLargeError => e
|
504
|
+
# if the payload is too large, we do not retry. we send to DLQ or drop it.
|
505
|
+
exc_data = {
|
506
|
+
:error_class => e.e_class,
|
507
|
+
:url => e.url.to_s,
|
508
|
+
:message => e.message,
|
509
|
+
:batch_num => batch_num,
|
510
|
+
:total_batches => total_batches,
|
511
|
+
:record_count => multi_event_request[:record_count],
|
512
|
+
:payload_size => multi_event_request[:body].bytesize,
|
513
|
+
}
|
514
|
+
exc_data[:code] = e.code if e.code
|
515
|
+
if defined?(e.body) and e.body
|
516
|
+
exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
|
517
|
+
end
|
518
|
+
exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
|
519
|
+
log_retry_failure(multi_event_request, exc_data, 0, 0)
|
520
|
+
next
|
503
521
|
rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
|
504
522
|
sleep_interval = sleep_for(sleep_interval)
|
505
523
|
exc_sleep += sleep_interval
|
@@ -634,18 +652,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
634
652
|
@multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
|
635
653
|
@multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
|
636
654
|
end
|
637
|
-
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
638
655
|
sample_events = Array.new
|
639
656
|
multi_event_request[:logstash_events][0,5].each {|l_event|
|
640
657
|
sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
|
641
658
|
}
|
642
|
-
|
659
|
+
if exc_data[:code] == 413
|
660
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
|
661
|
+
# For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
|
662
|
+
# to include redundant sample Logstash event objects
|
663
|
+
@logger.error(message, :error_data => exc_data)
|
664
|
+
else
|
665
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
666
|
+
@logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
|
667
|
+
end
|
643
668
|
if @dlq_writer
|
644
669
|
multi_event_request[:logstash_events].each {|l_event|
|
645
670
|
@dlq_writer.write(l_event, "#{exc_data[:message]}")
|
646
671
|
}
|
647
672
|
else
|
648
|
-
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events
|
673
|
+
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
|
649
674
|
end
|
650
675
|
end
|
651
676
|
|
@@ -1050,7 +1075,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1050
1075
|
serialized_request_size = serialized_body.bytesize
|
1051
1076
|
|
1052
1077
|
# We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
|
1053
|
-
if
|
1078
|
+
if serialized_request_size >= @max_request_buffer + 5000
|
1054
1079
|
# TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
|
1055
1080
|
@logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
|
1056
1081
|
end
|
data/lib/scalyr/common/client.rb
CHANGED
@@ -27,6 +27,18 @@ end
|
|
27
27
|
# An exception that signifies the Scalyr server received the upload request but dropped it
|
28
28
|
#---------------------------------------------------------------------------------------------------------------------
|
29
29
|
class RequestDroppedError < ServerError;
|
30
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
|
31
|
+
super(msg, code, url, body, e_class)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
#---------------------------------------------------------------------------------------------------------------------
|
36
|
+
# An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
|
37
|
+
#---------------------------------------------------------------------------------------------------------------------
|
38
|
+
class PayloadTooLargeError < ServerError;
|
39
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
|
40
|
+
super(msg, code, url, body, e_class)
|
41
|
+
end
|
30
42
|
end
|
31
43
|
|
32
44
|
#---------------------------------------------------------------------------------------------------------------------
|
@@ -308,15 +320,17 @@ class ClientSession
|
|
308
320
|
end
|
309
321
|
|
310
322
|
status = response_hash["status"]
|
323
|
+
code = response.code.to_s.strip.to_i
|
311
324
|
|
312
325
|
if status != "success"
|
313
|
-
if
|
326
|
+
if code == 413
|
327
|
+
raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
|
328
|
+
elsif status =~ /discardBuffer/
|
314
329
|
raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
|
315
330
|
else
|
316
331
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
317
332
|
end
|
318
333
|
else
|
319
|
-
code = response.code.to_s.strip.to_i
|
320
334
|
if code < 200 or code > 299
|
321
335
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
322
336
|
end
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.2.
|
3
|
+
s.version = '0.2.8.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files = Dir['lib/**/*','
|
13
|
+
s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
14
|
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.8.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -165,16 +165,6 @@ files:
|
|
165
165
|
- lib/scalyr/common/util.rb
|
166
166
|
- lib/scalyr/constants.rb
|
167
167
|
- logstash-output-scalyr.gemspec
|
168
|
-
- spec/benchmarks/bignum_fixing.rb
|
169
|
-
- spec/benchmarks/flattening_and_serialization.rb
|
170
|
-
- spec/benchmarks/json_serialization.rb
|
171
|
-
- spec/benchmarks/metrics_overhead.rb
|
172
|
-
- spec/benchmarks/set_session_level_serverhost_on_events.rb
|
173
|
-
- spec/benchmarks/util.rb
|
174
|
-
- spec/logstash/outputs/fixtures/example_com.pem
|
175
|
-
- spec/logstash/outputs/scalyr_integration_spec.rb
|
176
|
-
- spec/logstash/outputs/scalyr_spec.rb
|
177
|
-
- spec/scalyr/common/util_spec.rb
|
178
168
|
homepage: https://www.scalyr.com/help/data-sources#logstash
|
179
169
|
licenses:
|
180
170
|
- Apache-2.0
|
@@ -201,14 +191,4 @@ rubygems_version: 2.7.10
|
|
201
191
|
signing_key:
|
202
192
|
specification_version: 4
|
203
193
|
summary: Scalyr output plugin for Logstash
|
204
|
-
test_files:
|
205
|
-
- spec/benchmarks/bignum_fixing.rb
|
206
|
-
- spec/benchmarks/flattening_and_serialization.rb
|
207
|
-
- spec/benchmarks/json_serialization.rb
|
208
|
-
- spec/benchmarks/metrics_overhead.rb
|
209
|
-
- spec/benchmarks/set_session_level_serverhost_on_events.rb
|
210
|
-
- spec/benchmarks/util.rb
|
211
|
-
- spec/logstash/outputs/fixtures/example_com.pem
|
212
|
-
- spec/logstash/outputs/scalyr_integration_spec.rb
|
213
|
-
- spec/logstash/outputs/scalyr_spec.rb
|
214
|
-
- spec/scalyr/common/util_spec.rb
|
194
|
+
test_files: []
|
@@ -1,87 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
require_relative './util'
|
6
|
-
|
7
|
-
# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
|
8
|
-
|
9
|
-
ITERATIONS = 500
|
10
|
-
|
11
|
-
def rand_bignum()
|
12
|
-
return 200004000020304050300 + rand(999999)
|
13
|
-
end
|
14
|
-
|
15
|
-
def generate_hash(widths)
|
16
|
-
result = {}
|
17
|
-
if widths.empty?
|
18
|
-
return rand_bignum()
|
19
|
-
else
|
20
|
-
widths[0].times do
|
21
|
-
result[rand_str(9)] = generate_hash(widths[1..widths.length])
|
22
|
-
end
|
23
|
-
return result
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def generate_data_array_for_spec(spec)
|
28
|
-
data = []
|
29
|
-
ITERATIONS.times do
|
30
|
-
data << generate_hash(spec)
|
31
|
-
end
|
32
|
-
|
33
|
-
data
|
34
|
-
end
|
35
|
-
|
36
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
37
|
-
puts ""
|
38
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
39
|
-
puts ""
|
40
|
-
|
41
|
-
result = []
|
42
|
-
ITERATIONS.times do |i|
|
43
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
44
|
-
end
|
45
|
-
|
46
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
47
|
-
avg = sum / result.size
|
48
|
-
|
49
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
50
|
-
[sum, avg]
|
51
|
-
end
|
52
|
-
puts ""
|
53
|
-
end
|
54
|
-
|
55
|
-
|
56
|
-
puts "Using %s iterations" % [ITERATIONS]
|
57
|
-
puts ""
|
58
|
-
|
59
|
-
@value = Quantile::Estimator.new
|
60
|
-
@prng = Random.new
|
61
|
-
|
62
|
-
def convert_bignums(record)
|
63
|
-
Scalyr::Common::Util.convert_bignums(record)
|
64
|
-
end
|
65
|
-
|
66
|
-
puts "Util.convert_bignums()"
|
67
|
-
puts "==============================="
|
68
|
-
|
69
|
-
# Around ~200 keys in a hash
|
70
|
-
data = generate_data_array_for_spec([4, 4, 3, 4])
|
71
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
72
|
-
|
73
|
-
# Around ~200 keys in a hash (single level)
|
74
|
-
data = generate_data_array_for_spec([200])
|
75
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
76
|
-
|
77
|
-
# Around ~512 keys in a hash
|
78
|
-
data = generate_data_array_for_spec([8, 4, 4, 4])
|
79
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
80
|
-
|
81
|
-
# Around ~960 keys in a hash
|
82
|
-
data = generate_data_array_for_spec([12, 5, 4, 4])
|
83
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
84
|
-
|
85
|
-
# Around ~2700 keys in a hash
|
86
|
-
data = generate_data_array_for_spec([14, 8, 6, 4])
|
87
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
@@ -1,100 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'json'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
require_relative './util'
|
6
|
-
|
7
|
-
# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
|
8
|
-
# sense since it will just use JITed version of the code which will be very fast. If we
|
9
|
-
# wanted to accurately measure using multiple iterations we would need te different
|
10
|
-
# input data for each iteration.
|
11
|
-
ITERATIONS = 500
|
12
|
-
|
13
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
14
|
-
puts ""
|
15
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
16
|
-
puts ""
|
17
|
-
|
18
|
-
result = []
|
19
|
-
ITERATIONS.times do |i|
|
20
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
21
|
-
end
|
22
|
-
|
23
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
24
|
-
avg = sum / result.size
|
25
|
-
|
26
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
27
|
-
[sum, avg]
|
28
|
-
end
|
29
|
-
puts ""
|
30
|
-
end
|
31
|
-
|
32
|
-
def flatten_data_func(data)
|
33
|
-
Scalyr::Common::Util.flatten(data)
|
34
|
-
end
|
35
|
-
|
36
|
-
def json_serialize_data(data)
|
37
|
-
data.to_json
|
38
|
-
end
|
39
|
-
|
40
|
-
DATASETS = {
|
41
|
-
:keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
|
42
|
-
:keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
|
43
|
-
:keys_200_flat => generate_data_array_for_spec([200]),
|
44
|
-
:keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
|
45
|
-
:keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
|
46
|
-
:keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
|
47
|
-
}
|
48
|
-
|
49
|
-
puts "Using %s iterations" % [ITERATIONS]
|
50
|
-
puts ""
|
51
|
-
|
52
|
-
puts "Scalyr::Common::Util.flatten()"
|
53
|
-
puts "==============================="
|
54
|
-
|
55
|
-
# Around ~50 keys in a hash
|
56
|
-
data = DATASETS[:keys_50]
|
57
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
58
|
-
|
59
|
-
# Around ~200 keys in a hash
|
60
|
-
data = DATASETS[:keys_200]
|
61
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
62
|
-
|
63
|
-
# Around ~200 keys in a hash (single level)
|
64
|
-
data = DATASETS[:keys_200_flat]
|
65
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
66
|
-
|
67
|
-
# Around ~512 keys in a hash
|
68
|
-
data = DATASETS[:keys_512]
|
69
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
70
|
-
|
71
|
-
# Around ~960 keys in a hash
|
72
|
-
data = DATASETS[:keys_960]
|
73
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
74
|
-
|
75
|
-
# Around ~2700 keys in a hash
|
76
|
-
data = DATASETS[:keys_2700]
|
77
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
78
|
-
|
79
|
-
puts "JSON.dumps (hash.to_dict)"
|
80
|
-
puts "==============================="
|
81
|
-
|
82
|
-
# Around ~200 keys in a hash
|
83
|
-
data = generate_data_array_for_spec([4, 4, 3, 4])
|
84
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
85
|
-
|
86
|
-
# Around ~200 keys in a hash (single level)
|
87
|
-
data = DATASETS[:keys_200_flat]
|
88
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
89
|
-
|
90
|
-
# Around ~512 keys in a hash
|
91
|
-
data = generate_data_array_for_spec([8, 4, 4, 4])
|
92
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
93
|
-
|
94
|
-
# Around ~960 keys in a hash
|
95
|
-
data = generate_data_array_for_spec([12, 5, 4, 4])
|
96
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
97
|
-
|
98
|
-
# Around ~2700 keys in a hash
|
99
|
-
data = generate_data_array_for_spec([14, 8, 6, 4])
|
100
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
@@ -1,85 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'json'
|
3
|
-
require 'jrjackson'
|
4
|
-
|
5
|
-
require_relative '../../lib/scalyr/common/util'
|
6
|
-
require_relative './util'
|
7
|
-
|
8
|
-
ITERATIONS = 500
|
9
|
-
|
10
|
-
def json_serialize_data_native(data)
|
11
|
-
data.to_json
|
12
|
-
end
|
13
|
-
|
14
|
-
def json_serialize_data_jrjackson(data)
|
15
|
-
JrJackson::Json.dump(data)
|
16
|
-
end
|
17
|
-
|
18
|
-
DATASETS = {
|
19
|
-
:keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
|
20
|
-
:keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
|
21
|
-
:keys_200_flat => generate_data_array_for_spec([200]),
|
22
|
-
:keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
|
23
|
-
:keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
|
24
|
-
:keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
|
25
|
-
}
|
26
|
-
|
27
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
28
|
-
puts ""
|
29
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
30
|
-
puts ""
|
31
|
-
|
32
|
-
result = []
|
33
|
-
ITERATIONS.times do |i|
|
34
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
35
|
-
end
|
36
|
-
|
37
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
38
|
-
avg = sum / result.size
|
39
|
-
|
40
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
41
|
-
[sum, avg]
|
42
|
-
end
|
43
|
-
puts ""
|
44
|
-
end
|
45
|
-
|
46
|
-
puts "Using %s iterations" % [ITERATIONS]
|
47
|
-
puts ""
|
48
|
-
|
49
|
-
puts "native"
|
50
|
-
puts "==============================="
|
51
|
-
|
52
|
-
# Around ~50 keys in a hash
|
53
|
-
data = DATASETS[:keys_50]
|
54
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
55
|
-
|
56
|
-
# Around ~200 keys in a hash
|
57
|
-
data = DATASETS[:keys_200]
|
58
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
59
|
-
|
60
|
-
# Around ~200 keys in a hash (single level)
|
61
|
-
data = DATASETS[:keys_200_flat]
|
62
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
63
|
-
|
64
|
-
# Around ~2700 keys in a hash
|
65
|
-
data = DATASETS[:keys_2700]
|
66
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
67
|
-
|
68
|
-
puts "jrjackson"
|
69
|
-
puts "==============================="
|
70
|
-
|
71
|
-
# Around ~50 keys in a hash
|
72
|
-
data = DATASETS[:keys_50]
|
73
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
74
|
-
|
75
|
-
# Around ~200 keys in a hash
|
76
|
-
data = DATASETS[:keys_200]
|
77
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
78
|
-
|
79
|
-
# Around ~200 keys in a hash (single level)
|
80
|
-
data = DATASETS[:keys_200_flat]
|
81
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
82
|
-
|
83
|
-
# Around ~2700 keys in a hash
|
84
|
-
data = DATASETS[:keys_2700]
|
85
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
|
6
|
-
# Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
|
7
|
-
# where making sampling (e.g. on event level metrics) is desired
|
8
|
-
|
9
|
-
ITERATIONS = 10000
|
10
|
-
|
11
|
-
def run_benchmark_and_print_results(run_benchmark_func)
|
12
|
-
result = []
|
13
|
-
ITERATIONS.times do |i|
|
14
|
-
result << Benchmark.measure { run_benchmark_func.() }
|
15
|
-
end
|
16
|
-
|
17
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
18
|
-
avg = sum / result.size
|
19
|
-
|
20
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
21
|
-
[sum, avg]
|
22
|
-
end
|
23
|
-
puts ""
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
|
-
puts "Using %s iterations" % [ITERATIONS]
|
28
|
-
puts ""
|
29
|
-
|
30
|
-
@value = Quantile::Estimator.new
|
31
|
-
@prng = Random.new
|
32
|
-
|
33
|
-
def quantile_observe()
|
34
|
-
@value.observe(5)
|
35
|
-
end
|
36
|
-
|
37
|
-
def random_sample()
|
38
|
-
return @prng.rand(0.0..1.0) < 0.5
|
39
|
-
end
|
40
|
-
|
41
|
-
puts "Quartile.observe()"
|
42
|
-
puts "==============================="
|
43
|
-
|
44
|
-
run_benchmark_and_print_results(method(:quantile_observe))
|
45
|
-
|
46
|
-
puts "random sample"
|
47
|
-
puts "==============================="
|
48
|
-
run_benchmark_and_print_results(method(:random_sample))
|
@@ -1,107 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/constants'
|
5
|
-
require_relative '../../lib/scalyr/common/util'
|
6
|
-
require_relative './util'
|
7
|
-
|
8
|
-
# Micro benchmark which measures how long "set_session_level_serverhost_on_events" takes
|
9
|
-
|
10
|
-
ITERATIONS = 100
|
11
|
-
|
12
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
13
|
-
puts ""
|
14
|
-
puts "Using %s total events in a batch" % [data[0].size]
|
15
|
-
puts ""
|
16
|
-
|
17
|
-
result = []
|
18
|
-
ITERATIONS.times do |i|
|
19
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
20
|
-
end
|
21
|
-
|
22
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
23
|
-
avg = sum / result.size
|
24
|
-
|
25
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
26
|
-
[sum, avg]
|
27
|
-
end
|
28
|
-
puts ""
|
29
|
-
end
|
30
|
-
|
31
|
-
# Generate random event with only single event having special server host attribute set which
|
32
|
-
# represents a worst case scenario since we need to backfill rest of the events.
|
33
|
-
def generate_events(count)
|
34
|
-
result = []
|
35
|
-
|
36
|
-
ITERATIONS.times do |iteration|
|
37
|
-
events = []
|
38
|
-
|
39
|
-
count.times do |index|
|
40
|
-
event = generate_hash([2])
|
41
|
-
event[:attrs] = Hash.new
|
42
|
-
event[:log] = 1
|
43
|
-
|
44
|
-
if index == count - 1
|
45
|
-
event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME] = format("test-host-%s", index)
|
46
|
-
end
|
47
|
-
|
48
|
-
events << event
|
49
|
-
end
|
50
|
-
|
51
|
-
raise "Assertion failed" unless events.size == count
|
52
|
-
|
53
|
-
result << events
|
54
|
-
end
|
55
|
-
|
56
|
-
raise "Assertion failed" unless result.size == ITERATIONS
|
57
|
-
result
|
58
|
-
end
|
59
|
-
|
60
|
-
def run_func(events)
|
61
|
-
# NOTE: This function manipulates events in place
|
62
|
-
events.each_with_index do |event, index|
|
63
|
-
if index < events.size - 1
|
64
|
-
# Last event will have _origServerHost set, but others won't
|
65
|
-
raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil?
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
Scalyr::Common::Util.set_session_level_serverhost_on_events("session-server-host-dummy", events, {}, true)
|
70
|
-
|
71
|
-
events.each do |event|
|
72
|
-
raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil? == false
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
|
77
|
-
puts "Using %s iterations" % [ITERATIONS]
|
78
|
-
puts ""
|
79
|
-
|
80
|
-
@value = Quantile::Estimator.new
|
81
|
-
|
82
|
-
puts "Util.set_session_level_serverhost_on_events()"
|
83
|
-
puts "==============================="
|
84
|
-
|
85
|
-
# 100 events in a batch
|
86
|
-
data = generate_events(100)
|
87
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
88
|
-
|
89
|
-
# 500 events in a batch
|
90
|
-
data = generate_events(500)
|
91
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
92
|
-
|
93
|
-
# 1000 events in a batch
|
94
|
-
data = generate_events(1000)
|
95
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
96
|
-
|
97
|
-
# 2000 events in a batch
|
98
|
-
data = generate_events(2000)
|
99
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
100
|
-
|
101
|
-
# 3000 events in a batch
|
102
|
-
data = generate_events(3000)
|
103
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
104
|
-
|
105
|
-
# 5000 events in a batch
|
106
|
-
data = generate_events(5000)
|
107
|
-
run_benchmark_and_print_results(data, method(:run_func))
|