logstash-output-scalyr 0.2.7.beta → 0.2.8.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +2 -4
- data/lib/logstash/outputs/scalyr.rb +29 -4
- data/lib/scalyr/common/client.rb +16 -2
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +2 -2
- metadata +3 -23
- data/spec/benchmarks/bignum_fixing.rb +0 -87
- data/spec/benchmarks/flattening_and_serialization.rb +0 -100
- data/spec/benchmarks/json_serialization.rb +0 -85
- data/spec/benchmarks/metrics_overhead.rb +0 -48
- data/spec/benchmarks/set_session_level_serverhost_on_events.rb +0 -107
- data/spec/benchmarks/util.rb +0 -24
- data/spec/logstash/outputs/fixtures/example_com.pem +0 -41
- data/spec/logstash/outputs/scalyr_integration_spec.rb +0 -318
- data/spec/logstash/outputs/scalyr_spec.rb +0 -1262
- data/spec/scalyr/common/util_spec.rb +0 -543
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da91d80d418081307dc1a6b756f17024712c0eab9a6d527a140288976a58b76f
|
4
|
+
data.tar.gz: 2ac792b42939edb371614c73c4dbb50fcc835f199d03685ff619d9d355c6465a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0a87ca7115a2eb3f03282046510bd51869e575eb29b8bec6ee273cbc9f68fea0167890835ffdda8bd0d571ac4299aff689642542d8dd5d88eaa2a89fe760a3f0
|
7
|
+
data.tar.gz: 7457fff5003d10b7288a9a958f3ec3f4a490c19da51a3ac3ade0402a37f74fafffc2501185857b4b3a7263753cb6af3276471724870efa0dca9e0773671b2f3f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.2.8.beta
|
4
|
+
|
5
|
+
* Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
|
6
|
+
fixtures with the actual production gem file.
|
7
|
+
|
8
|
+
* Do not retry requests that will never be accepted by the server.
|
9
|
+
Specifically, any request that returns HTTP Status code 413 is too large, and
|
10
|
+
will never be accepted. Instead of simply retrying for 10 minutes before
|
11
|
+
sending the request to the DLQ, skip the retries go directly to sending the
|
12
|
+
request to the DLQ.
|
13
|
+
|
14
|
+
To be notified when an event fails to be ingested for whatever reason, create
|
15
|
+
an alert using the query: ``parser='logstash_plugin_metrics'
|
16
|
+
failed_events_processed > 0``. Instructions on how to create an alert can be
|
17
|
+
found in our docs here: https://scalyr.com/help/alerts
|
18
|
+
|
3
19
|
## 0.2.7.beta
|
4
20
|
|
5
21
|
* SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
|
data/README.md
CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
|
|
505
505
|
|
506
506
|
```
|
507
507
|
rm -rf vendor/
|
508
|
-
bundle
|
508
|
+
bundle install
|
509
509
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
510
510
|
chmod 0600 ~/.gem/credentials
|
511
|
-
bundle exec rake vendor
|
512
511
|
bundle exec rspec
|
513
512
|
bundle exec rake publish_gem
|
514
513
|
```
|
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
|
|
518
517
|
|
519
518
|
```
|
520
519
|
rm -rf vendor/
|
521
|
-
bundle
|
520
|
+
bundle install
|
522
521
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
523
522
|
chmod 0600 ~/.gem/credentials
|
524
|
-
bundle exec rake vendor
|
525
523
|
bundle exec rspec
|
526
524
|
rvm use jruby
|
527
525
|
bundle exec gem build logstash-output-scalyr.gemspec
|
@@ -500,6 +500,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
500
500
|
result.push(multi_event_request)
|
501
501
|
end
|
502
502
|
|
503
|
+
rescue Scalyr::Common::Client::PayloadTooLargeError => e
|
504
|
+
# if the payload is too large, we do not retry. we send to DLQ or drop it.
|
505
|
+
exc_data = {
|
506
|
+
:error_class => e.e_class,
|
507
|
+
:url => e.url.to_s,
|
508
|
+
:message => e.message,
|
509
|
+
:batch_num => batch_num,
|
510
|
+
:total_batches => total_batches,
|
511
|
+
:record_count => multi_event_request[:record_count],
|
512
|
+
:payload_size => multi_event_request[:body].bytesize,
|
513
|
+
}
|
514
|
+
exc_data[:code] = e.code if e.code
|
515
|
+
if defined?(e.body) and e.body
|
516
|
+
exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
|
517
|
+
end
|
518
|
+
exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
|
519
|
+
log_retry_failure(multi_event_request, exc_data, 0, 0)
|
520
|
+
next
|
503
521
|
rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
|
504
522
|
sleep_interval = sleep_for(sleep_interval)
|
505
523
|
exc_sleep += sleep_interval
|
@@ -634,18 +652,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
634
652
|
@multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
|
635
653
|
@multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
|
636
654
|
end
|
637
|
-
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
638
655
|
sample_events = Array.new
|
639
656
|
multi_event_request[:logstash_events][0,5].each {|l_event|
|
640
657
|
sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
|
641
658
|
}
|
642
|
-
|
659
|
+
if exc_data[:code] == 413
|
660
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
|
661
|
+
# For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
|
662
|
+
# to include redundant sample Logstash event objects
|
663
|
+
@logger.error(message, :error_data => exc_data)
|
664
|
+
else
|
665
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
666
|
+
@logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
|
667
|
+
end
|
643
668
|
if @dlq_writer
|
644
669
|
multi_event_request[:logstash_events].each {|l_event|
|
645
670
|
@dlq_writer.write(l_event, "#{exc_data[:message]}")
|
646
671
|
}
|
647
672
|
else
|
648
|
-
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events
|
673
|
+
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
|
649
674
|
end
|
650
675
|
end
|
651
676
|
|
@@ -1050,7 +1075,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1050
1075
|
serialized_request_size = serialized_body.bytesize
|
1051
1076
|
|
1052
1077
|
# We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
|
1053
|
-
if
|
1078
|
+
if serialized_request_size >= @max_request_buffer + 5000
|
1054
1079
|
# TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
|
1055
1080
|
@logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
|
1056
1081
|
end
|
data/lib/scalyr/common/client.rb
CHANGED
@@ -27,6 +27,18 @@ end
|
|
27
27
|
# An exception that signifies the Scalyr server received the upload request but dropped it
|
28
28
|
#---------------------------------------------------------------------------------------------------------------------
|
29
29
|
class RequestDroppedError < ServerError;
|
30
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
|
31
|
+
super(msg, code, url, body, e_class)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
#---------------------------------------------------------------------------------------------------------------------
|
36
|
+
# An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
|
37
|
+
#---------------------------------------------------------------------------------------------------------------------
|
38
|
+
class PayloadTooLargeError < ServerError;
|
39
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
|
40
|
+
super(msg, code, url, body, e_class)
|
41
|
+
end
|
30
42
|
end
|
31
43
|
|
32
44
|
#---------------------------------------------------------------------------------------------------------------------
|
@@ -308,15 +320,17 @@ class ClientSession
|
|
308
320
|
end
|
309
321
|
|
310
322
|
status = response_hash["status"]
|
323
|
+
code = response.code.to_s.strip.to_i
|
311
324
|
|
312
325
|
if status != "success"
|
313
|
-
if
|
326
|
+
if code == 413
|
327
|
+
raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
|
328
|
+
elsif status =~ /discardBuffer/
|
314
329
|
raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
|
315
330
|
else
|
316
331
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
317
332
|
end
|
318
333
|
else
|
319
|
-
code = response.code.to_s.strip.to_i
|
320
334
|
if code < 200 or code > 299
|
321
335
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
322
336
|
end
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.2.
|
3
|
+
s.version = '0.2.8.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files = Dir['lib/**/*','
|
13
|
+
s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
14
|
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.8.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -165,16 +165,6 @@ files:
|
|
165
165
|
- lib/scalyr/common/util.rb
|
166
166
|
- lib/scalyr/constants.rb
|
167
167
|
- logstash-output-scalyr.gemspec
|
168
|
-
- spec/benchmarks/bignum_fixing.rb
|
169
|
-
- spec/benchmarks/flattening_and_serialization.rb
|
170
|
-
- spec/benchmarks/json_serialization.rb
|
171
|
-
- spec/benchmarks/metrics_overhead.rb
|
172
|
-
- spec/benchmarks/set_session_level_serverhost_on_events.rb
|
173
|
-
- spec/benchmarks/util.rb
|
174
|
-
- spec/logstash/outputs/fixtures/example_com.pem
|
175
|
-
- spec/logstash/outputs/scalyr_integration_spec.rb
|
176
|
-
- spec/logstash/outputs/scalyr_spec.rb
|
177
|
-
- spec/scalyr/common/util_spec.rb
|
178
168
|
homepage: https://www.scalyr.com/help/data-sources#logstash
|
179
169
|
licenses:
|
180
170
|
- Apache-2.0
|
@@ -201,14 +191,4 @@ rubygems_version: 2.7.10
|
|
201
191
|
signing_key:
|
202
192
|
specification_version: 4
|
203
193
|
summary: Scalyr output plugin for Logstash
|
204
|
-
test_files:
|
205
|
-
- spec/benchmarks/bignum_fixing.rb
|
206
|
-
- spec/benchmarks/flattening_and_serialization.rb
|
207
|
-
- spec/benchmarks/json_serialization.rb
|
208
|
-
- spec/benchmarks/metrics_overhead.rb
|
209
|
-
- spec/benchmarks/set_session_level_serverhost_on_events.rb
|
210
|
-
- spec/benchmarks/util.rb
|
211
|
-
- spec/logstash/outputs/fixtures/example_com.pem
|
212
|
-
- spec/logstash/outputs/scalyr_integration_spec.rb
|
213
|
-
- spec/logstash/outputs/scalyr_spec.rb
|
214
|
-
- spec/scalyr/common/util_spec.rb
|
194
|
+
test_files: []
|
@@ -1,87 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
require_relative './util'
|
6
|
-
|
7
|
-
# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
|
8
|
-
|
9
|
-
ITERATIONS = 500
|
10
|
-
|
11
|
-
def rand_bignum()
|
12
|
-
return 200004000020304050300 + rand(999999)
|
13
|
-
end
|
14
|
-
|
15
|
-
def generate_hash(widths)
|
16
|
-
result = {}
|
17
|
-
if widths.empty?
|
18
|
-
return rand_bignum()
|
19
|
-
else
|
20
|
-
widths[0].times do
|
21
|
-
result[rand_str(9)] = generate_hash(widths[1..widths.length])
|
22
|
-
end
|
23
|
-
return result
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def generate_data_array_for_spec(spec)
|
28
|
-
data = []
|
29
|
-
ITERATIONS.times do
|
30
|
-
data << generate_hash(spec)
|
31
|
-
end
|
32
|
-
|
33
|
-
data
|
34
|
-
end
|
35
|
-
|
36
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
37
|
-
puts ""
|
38
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
39
|
-
puts ""
|
40
|
-
|
41
|
-
result = []
|
42
|
-
ITERATIONS.times do |i|
|
43
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
44
|
-
end
|
45
|
-
|
46
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
47
|
-
avg = sum / result.size
|
48
|
-
|
49
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
50
|
-
[sum, avg]
|
51
|
-
end
|
52
|
-
puts ""
|
53
|
-
end
|
54
|
-
|
55
|
-
|
56
|
-
puts "Using %s iterations" % [ITERATIONS]
|
57
|
-
puts ""
|
58
|
-
|
59
|
-
@value = Quantile::Estimator.new
|
60
|
-
@prng = Random.new
|
61
|
-
|
62
|
-
def convert_bignums(record)
|
63
|
-
Scalyr::Common::Util.convert_bignums(record)
|
64
|
-
end
|
65
|
-
|
66
|
-
puts "Util.convert_bignums()"
|
67
|
-
puts "==============================="
|
68
|
-
|
69
|
-
# Around ~200 keys in a hash
|
70
|
-
data = generate_data_array_for_spec([4, 4, 3, 4])
|
71
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
72
|
-
|
73
|
-
# Around ~200 keys in a hash (single level)
|
74
|
-
data = generate_data_array_for_spec([200])
|
75
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
76
|
-
|
77
|
-
# Around ~512 keys in a hash
|
78
|
-
data = generate_data_array_for_spec([8, 4, 4, 4])
|
79
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
80
|
-
|
81
|
-
# Around ~960 keys in a hash
|
82
|
-
data = generate_data_array_for_spec([12, 5, 4, 4])
|
83
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
84
|
-
|
85
|
-
# Around ~2700 keys in a hash
|
86
|
-
data = generate_data_array_for_spec([14, 8, 6, 4])
|
87
|
-
run_benchmark_and_print_results(data, method(:convert_bignums))
|
@@ -1,100 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'json'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
require_relative './util'
|
6
|
-
|
7
|
-
# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
|
8
|
-
# sense since it will just use JITed version of the code which will be very fast. If we
|
9
|
-
# wanted to accurately measure using multiple iterations we would need te different
|
10
|
-
# input data for each iteration.
|
11
|
-
ITERATIONS = 500
|
12
|
-
|
13
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
14
|
-
puts ""
|
15
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
16
|
-
puts ""
|
17
|
-
|
18
|
-
result = []
|
19
|
-
ITERATIONS.times do |i|
|
20
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
21
|
-
end
|
22
|
-
|
23
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
24
|
-
avg = sum / result.size
|
25
|
-
|
26
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
27
|
-
[sum, avg]
|
28
|
-
end
|
29
|
-
puts ""
|
30
|
-
end
|
31
|
-
|
32
|
-
def flatten_data_func(data)
|
33
|
-
Scalyr::Common::Util.flatten(data)
|
34
|
-
end
|
35
|
-
|
36
|
-
def json_serialize_data(data)
|
37
|
-
data.to_json
|
38
|
-
end
|
39
|
-
|
40
|
-
DATASETS = {
|
41
|
-
:keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
|
42
|
-
:keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
|
43
|
-
:keys_200_flat => generate_data_array_for_spec([200]),
|
44
|
-
:keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
|
45
|
-
:keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
|
46
|
-
:keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
|
47
|
-
}
|
48
|
-
|
49
|
-
puts "Using %s iterations" % [ITERATIONS]
|
50
|
-
puts ""
|
51
|
-
|
52
|
-
puts "Scalyr::Common::Util.flatten()"
|
53
|
-
puts "==============================="
|
54
|
-
|
55
|
-
# Around ~50 keys in a hash
|
56
|
-
data = DATASETS[:keys_50]
|
57
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
58
|
-
|
59
|
-
# Around ~200 keys in a hash
|
60
|
-
data = DATASETS[:keys_200]
|
61
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
62
|
-
|
63
|
-
# Around ~200 keys in a hash (single level)
|
64
|
-
data = DATASETS[:keys_200_flat]
|
65
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
66
|
-
|
67
|
-
# Around ~512 keys in a hash
|
68
|
-
data = DATASETS[:keys_512]
|
69
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
70
|
-
|
71
|
-
# Around ~960 keys in a hash
|
72
|
-
data = DATASETS[:keys_960]
|
73
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
74
|
-
|
75
|
-
# Around ~2700 keys in a hash
|
76
|
-
data = DATASETS[:keys_2700]
|
77
|
-
run_benchmark_and_print_results(data, method(:flatten_data_func))
|
78
|
-
|
79
|
-
puts "JSON.dumps (hash.to_dict)"
|
80
|
-
puts "==============================="
|
81
|
-
|
82
|
-
# Around ~200 keys in a hash
|
83
|
-
data = generate_data_array_for_spec([4, 4, 3, 4])
|
84
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
85
|
-
|
86
|
-
# Around ~200 keys in a hash (single level)
|
87
|
-
data = DATASETS[:keys_200_flat]
|
88
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
89
|
-
|
90
|
-
# Around ~512 keys in a hash
|
91
|
-
data = generate_data_array_for_spec([8, 4, 4, 4])
|
92
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
93
|
-
|
94
|
-
# Around ~960 keys in a hash
|
95
|
-
data = generate_data_array_for_spec([12, 5, 4, 4])
|
96
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
97
|
-
|
98
|
-
# Around ~2700 keys in a hash
|
99
|
-
data = generate_data_array_for_spec([14, 8, 6, 4])
|
100
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data))
|
@@ -1,85 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'json'
|
3
|
-
require 'jrjackson'
|
4
|
-
|
5
|
-
require_relative '../../lib/scalyr/common/util'
|
6
|
-
require_relative './util'
|
7
|
-
|
8
|
-
ITERATIONS = 500
|
9
|
-
|
10
|
-
def json_serialize_data_native(data)
|
11
|
-
data.to_json
|
12
|
-
end
|
13
|
-
|
14
|
-
def json_serialize_data_jrjackson(data)
|
15
|
-
JrJackson::Json.dump(data)
|
16
|
-
end
|
17
|
-
|
18
|
-
DATASETS = {
|
19
|
-
:keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
|
20
|
-
:keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
|
21
|
-
:keys_200_flat => generate_data_array_for_spec([200]),
|
22
|
-
:keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
|
23
|
-
:keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
|
24
|
-
:keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
|
25
|
-
}
|
26
|
-
|
27
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
28
|
-
puts ""
|
29
|
-
puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
|
30
|
-
puts ""
|
31
|
-
|
32
|
-
result = []
|
33
|
-
ITERATIONS.times do |i|
|
34
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
35
|
-
end
|
36
|
-
|
37
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
38
|
-
avg = sum / result.size
|
39
|
-
|
40
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
41
|
-
[sum, avg]
|
42
|
-
end
|
43
|
-
puts ""
|
44
|
-
end
|
45
|
-
|
46
|
-
puts "Using %s iterations" % [ITERATIONS]
|
47
|
-
puts ""
|
48
|
-
|
49
|
-
puts "native"
|
50
|
-
puts "==============================="
|
51
|
-
|
52
|
-
# Around ~50 keys in a hash
|
53
|
-
data = DATASETS[:keys_50]
|
54
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
55
|
-
|
56
|
-
# Around ~200 keys in a hash
|
57
|
-
data = DATASETS[:keys_200]
|
58
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
59
|
-
|
60
|
-
# Around ~200 keys in a hash (single level)
|
61
|
-
data = DATASETS[:keys_200_flat]
|
62
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
63
|
-
|
64
|
-
# Around ~2700 keys in a hash
|
65
|
-
data = DATASETS[:keys_2700]
|
66
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_native))
|
67
|
-
|
68
|
-
puts "jrjackson"
|
69
|
-
puts "==============================="
|
70
|
-
|
71
|
-
# Around ~50 keys in a hash
|
72
|
-
data = DATASETS[:keys_50]
|
73
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
74
|
-
|
75
|
-
# Around ~200 keys in a hash
|
76
|
-
data = DATASETS[:keys_200]
|
77
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
78
|
-
|
79
|
-
# Around ~200 keys in a hash (single level)
|
80
|
-
data = DATASETS[:keys_200_flat]
|
81
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
82
|
-
|
83
|
-
# Around ~2700 keys in a hash
|
84
|
-
data = DATASETS[:keys_2700]
|
85
|
-
run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
|
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/common/util'
|
5
|
-
|
6
|
-
# Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
|
7
|
-
# where making sampling (e.g. on event level metrics) is desired
|
8
|
-
|
9
|
-
ITERATIONS = 10000
|
10
|
-
|
11
|
-
def run_benchmark_and_print_results(run_benchmark_func)
|
12
|
-
result = []
|
13
|
-
ITERATIONS.times do |i|
|
14
|
-
result << Benchmark.measure { run_benchmark_func.() }
|
15
|
-
end
|
16
|
-
|
17
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
18
|
-
avg = sum / result.size
|
19
|
-
|
20
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
21
|
-
[sum, avg]
|
22
|
-
end
|
23
|
-
puts ""
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
|
-
puts "Using %s iterations" % [ITERATIONS]
|
28
|
-
puts ""
|
29
|
-
|
30
|
-
@value = Quantile::Estimator.new
|
31
|
-
@prng = Random.new
|
32
|
-
|
33
|
-
def quantile_observe()
|
34
|
-
@value.observe(5)
|
35
|
-
end
|
36
|
-
|
37
|
-
def random_sample()
|
38
|
-
return @prng.rand(0.0..1.0) < 0.5
|
39
|
-
end
|
40
|
-
|
41
|
-
puts "Quartile.observe()"
|
42
|
-
puts "==============================="
|
43
|
-
|
44
|
-
run_benchmark_and_print_results(method(:quantile_observe))
|
45
|
-
|
46
|
-
puts "random sample"
|
47
|
-
puts "==============================="
|
48
|
-
run_benchmark_and_print_results(method(:random_sample))
|
@@ -1,107 +0,0 @@
|
|
1
|
-
require 'benchmark'
|
2
|
-
require 'quantile'
|
3
|
-
|
4
|
-
require_relative '../../lib/scalyr/constants'
|
5
|
-
require_relative '../../lib/scalyr/common/util'
|
6
|
-
require_relative './util'
|
7
|
-
|
8
|
-
# Micro benchmark which measures how long "set_session_level_serverhost_on_events" takes
|
9
|
-
|
10
|
-
ITERATIONS = 100
|
11
|
-
|
12
|
-
def run_benchmark_and_print_results(data, run_benchmark_func)
|
13
|
-
puts ""
|
14
|
-
puts "Using %s total events in a batch" % [data[0].size]
|
15
|
-
puts ""
|
16
|
-
|
17
|
-
result = []
|
18
|
-
ITERATIONS.times do |i|
|
19
|
-
result << Benchmark.measure { run_benchmark_func.(data[i]) }
|
20
|
-
end
|
21
|
-
|
22
|
-
sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
|
23
|
-
avg = sum / result.size
|
24
|
-
|
25
|
-
Benchmark.bm(7, "sum:", "avg:") do |b|
|
26
|
-
[sum, avg]
|
27
|
-
end
|
28
|
-
puts ""
|
29
|
-
end
|
30
|
-
|
31
|
-
# Generate random event with only single event having special server host attribute set which
|
32
|
-
# represents a worst case scenario since we need to backfill rest of the events.
|
33
|
-
def generate_events(count)
|
34
|
-
result = []
|
35
|
-
|
36
|
-
ITERATIONS.times do |iteration|
|
37
|
-
events = []
|
38
|
-
|
39
|
-
count.times do |index|
|
40
|
-
event = generate_hash([2])
|
41
|
-
event[:attrs] = Hash.new
|
42
|
-
event[:log] = 1
|
43
|
-
|
44
|
-
if index == count - 1
|
45
|
-
event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME] = format("test-host-%s", index)
|
46
|
-
end
|
47
|
-
|
48
|
-
events << event
|
49
|
-
end
|
50
|
-
|
51
|
-
raise "Assertion failed" unless events.size == count
|
52
|
-
|
53
|
-
result << events
|
54
|
-
end
|
55
|
-
|
56
|
-
raise "Assertion failed" unless result.size == ITERATIONS
|
57
|
-
result
|
58
|
-
end
|
59
|
-
|
60
|
-
def run_func(events)
|
61
|
-
# NOTE: This function manipulates events in place
|
62
|
-
events.each_with_index do |event, index|
|
63
|
-
if index < events.size - 1
|
64
|
-
# Last event will have _origServerHost set, but others won't
|
65
|
-
raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil?
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
Scalyr::Common::Util.set_session_level_serverhost_on_events("session-server-host-dummy", events, {}, true)
|
70
|
-
|
71
|
-
events.each do |event|
|
72
|
-
raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil? == false
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
|
77
|
-
puts "Using %s iterations" % [ITERATIONS]
|
78
|
-
puts ""
|
79
|
-
|
80
|
-
@value = Quantile::Estimator.new
|
81
|
-
|
82
|
-
puts "Util.set_session_level_serverhost_on_events()"
|
83
|
-
puts "==============================="
|
84
|
-
|
85
|
-
# 100 events in a batch
|
86
|
-
data = generate_events(100)
|
87
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
88
|
-
|
89
|
-
# 500 events in a batch
|
90
|
-
data = generate_events(500)
|
91
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
92
|
-
|
93
|
-
# 1000 events in a batch
|
94
|
-
data = generate_events(1000)
|
95
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
96
|
-
|
97
|
-
# 2000 events in a batch
|
98
|
-
data = generate_events(2000)
|
99
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
100
|
-
|
101
|
-
# 3000 events in a batch
|
102
|
-
data = generate_events(3000)
|
103
|
-
run_benchmark_and_print_results(data, method(:run_func))
|
104
|
-
|
105
|
-
# 5000 events in a batch
|
106
|
-
data = generate_events(5000)
|
107
|
-
run_benchmark_and_print_results(data, method(:run_func))
|